diff --git a/.gitignore b/.gitignore index e277bd8c..4acc7987 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ bin dist/ config/ builds/ +hack/generated/ # Test binary, build with `go test -c` *.test diff --git a/.golangci.yaml b/.golangci.yaml index 6fdb9c34..fe61c502 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -5,6 +5,7 @@ run: linters: default: all disable: + - goconst - godoclint - depguard - err113 diff --git a/Makefile b/Makefile index 46676d53..370454e2 100644 --- a/Makefile +++ b/Makefile @@ -17,9 +17,9 @@ IMG_BASE ?= $(REPOSITORY) IMG ?= $(IMG_BASE):$(VERSION) CAPSULE_IMG ?= $(REGISTRY)/$(IMG_BASE) CLUSTER_NAME ?= capsule -FILTER ?= --label-filter="!skip" +FILTER ?= && !skip ## Kubernetes Version Support -KUBERNETES_SUPPORTED_VERSION ?= "v1.35.0" +KUBERNETES_SUPPORTED_VERSION ?= "v1.32.0" ## Openshift Version Support OS_SUPPORTED_VERSION ?= "4.22.0-okd-scos.ec.10" @@ -109,7 +109,7 @@ helm-test-exec: ct helm-controller-version ko-build-all # Setup development env dev-build: kind $(KIND) create cluster --wait=60s --name $(CLUSTER_NAME) --image kindest/node:$(KUBERNETES_SUPPORTED_VERSION) --config ./hack/kind-cluster.yaml - $(MAKE) dev-install-deps + $(MAKE) dev-install-gw-api-crds .PHONY: dev-destroy dev-destroy: kind @@ -158,6 +158,8 @@ subjectAltName = @alt_names IP.1 = $(LAPTOP_HOST_IP) endef export TLS_CNF +CHART ?= "./charts/capsule" +CHART_VERSION ?= "./charts/capsule" dev-setup: $(KUBECTL) -n capsule-system scale deployment capsule-controller-manager --replicas=0 || true mkdir -p /tmp/k8s-webhook-server/serving-certs @@ -175,22 +177,44 @@ dev-setup: export WEBHOOK_URL="https://$${LAPTOP_HOST_IP}:9443"; \ export CA_BUNDLE=`openssl base64 -in /tmp/k8s-webhook-server/serving-certs/tls.crt | tr -d '\n'`; \ $(HELM) upgrade \ - --dependency-update \ + --dependency-update \ --force-conflicts \ + --take-ownership \ --debug \ --install \ --namespace capsule-system \ --create-namespace \ + --version=$(CHART_VERSION) \ + --set 'proxy.enabled=true' \ --set 'crds.install=true' \ --set 'crds.exclusive=true'\ - --set 'crds.createConfig=true'\ - --set "tls.enableController=false"\ + --set 'crds.createConfig=true'\ + --set 'crds.createRBAC=true'\ + --set 'crds.createDiagnostics=true'\ + --set "monitoring.diagnostics.enabled=true"\ + --set "manager.rbac.minimal=true"\ + --set 'certManager.generateCertificates=false' \ + --set 'tls.enableController=true' \ + --set 'tls.create=true' \ --set "webhooks.exclusive=true"\ - --set "webhooks.hooks.nodes.enabled=true"\ --set "webhooks.service.url=$${WEBHOOK_URL}" \ --set "webhooks.service.caBundle=$${CA_BUNDLE}" \ + --set "webhooks.hooks.nodes.enabled=true"\ + --set 'webhooks.hooks.calculations.enabled=true' \ + --set-string 'webhooks.hooks.calculations.rules[0].apiGroups[0]=' \ + --set 'webhooks.hooks.calculations.rules[0].apiVersions[0]=v1' \ + --set 'webhooks.hooks.calculations.rules[0].operations[0]=CREATE' \ + --set 'webhooks.hooks.calculations.rules[0].operations[1]=UPDATE' \ + --set 'webhooks.hooks.calculations.rules[0].operations[2]=DELETE' \ + --set 'webhooks.hooks.calculations.rules[0].resources[0]=pods' \ + --set 'webhooks.hooks.calculations.rules[0].resources[1]=persistentvolumeclaims' \ + --set 'webhooks.hooks.calculations.rules[0].scope=Namespaced' \ + --set 'webhooks.hooks.calculations.namespaceSelector.matchLabels.env=e2e' \ capsule \ - ./charts/capsule || true + $(CHART) + mkdir -p ./hack/generated/ || true + $(KUBECTL) label clusterrole admin projectcapsule.dev/aggregate-to-controller=true + bash ./hack/kubeconfig-for-sa.sh $(CLUSTER_NAME) "capsule-system" "capsule" "./hack/generated/kubeconfig.yaml" setup-monitoring: dev-setup-fluxcd @@ -232,12 +256,24 @@ dev-setup-capsule: dev-setup-fluxcd dev-setup-capsule-example: dev-setup-fluxcd @$(KUBECTL) kustomize --load-restrictor='LoadRestrictionsNone' hack/distro/capsule/example-setup | envsubst | kubectl apply -f - + @$(KUBECTL) create ns wind-uat --as joe --as-group projectcapsule.dev || true + @$(KUBECTL) label ns wind-uat env=test @$(KUBECTL) create ns wind-test --as joe --as-group projectcapsule.dev || true + @$(KUBECTL) label ns wind-test env=test @$(KUBECTL) create ns wind-prod --as joe --as-group projectcapsule.dev || true + @$(KUBECTL) label ns wind-prod env=prod + @$(KUBECTL) create ns green-uat --as bob --as-group projectcapsule.dev || true + @$(KUBECTL) label ns green-uat env=test @$(KUBECTL) create ns green-test --as bob --as-group projectcapsule.dev || true + @$(KUBECTL) label ns green-test env=test @$(KUBECTL) create ns green-prod --as bob --as-group projectcapsule.dev || true + @$(KUBECTL) label ns green-prod env=prod + @$(KUBECTL) create ns solar-uat --as alice --as-group projectcapsule.dev || true + @$(KUBECTL) label ns solar-uat env=test @$(KUBECTL) create ns solar-test --as alice --as-group projectcapsule.dev || true + @$(KUBECTL) label ns solar-test env=test @$(KUBECTL) create ns solar-prod --as alice --as-group projectcapsule.dev || true + @$(KUBECTL) label ns solar-prod env=prod @$(KUBECTL) apply -f hack/distro/capsule/example-setup/claims.yaml @@ -247,19 +283,53 @@ wait-for-helmreleases: sleep 5; \ done +#################### +# -- Enterprise Release +#################### -ENTERPRISE_VERSION ?= "0.13.0-rc.2" -ENTERPRISE_REGISTRY ?= "oci.peakscale.ch" +ENTERPRISE_VERSION ?= "dirty" +ENTERPRISE_REGISTRY ?= "registry.projectcapsule.dev" -enterprise-prerelease: +enterprise-release: mkdir -p ./builds - $(MAKE) CAPSULE_IMG=$(ENTERPRISE_REGISTRY)/prereleases/images/capsule VERSION=$(ENTERPRISE_VERSION) ko-publish-capsule + $(MAKE) CAPSULE_IMG=$(ENTERPRISE_REGISTRY)/enterprise/capsule VERSION=v$(ENTERPRISE_VERSION) ko-publish-capsule $(HELM) package ./charts/capsule --app-version=$(ENTERPRISE_VERSION) --version=$(ENTERPRISE_VERSION) --destination ./builds/ - $(HELM) push ./builds/capsule-$(ENTERPRISE_VERSION).tgz oci://$(ENTERPRISE_REGISTRY)/prereleases/charts/ + $(HELM) push ./builds/capsule-$(ENTERPRISE_VERSION).tgz oci://$(ENTERPRISE_REGISTRY)/charts/ $(MAKE) deploy-enterprise rm -rf ./builds deploy-enterprise: + @echo "" + @echo "Deploying Capsule (Enterprise) $(ENTERPRISE_VERSION)" + @echo "" + @echo "1) Create image pull secret (Change the credentials with the ones provided to you):" + @echo "" + @echo "kubectl create secret docker-registry capsule-enterprise -n capsule-system \\" + @echo " --docker-username='robot\$$name' \\" + @echo " --docker-password='serviceaccount-password' \\" + @echo " --docker-server='$(ENTERPRISE_REGISTRY)'" + @echo "" + @echo "2) Deploy Capsule:" + @echo "" + @echo "helm upgrade --install capsule \\" + @echo " oci://$(ENTERPRISE_REGISTRY)/charts/capsule \\" + @echo " --namespace capsule-system \\" + @echo " --version $(ENTERPRISE_VERSION) \\" + @echo " --reuse-values \\" + @echo " --set manager.image.registry=$(ENTERPRISE_REGISTRY) \\" + @echo " --set manager.image.repository=enterprise/capsule \\" + @echo " --set 'serviceAccount.imagePullSecrets={capsule-enterprise}'" + @echo "" + +enterprise-prerelease: + mkdir -p ./builds + $(MAKE) CAPSULE_IMG=$(ENTERPRISE_REGISTRY)/prereleases/capsule VERSION=v$(ENTERPRISE_VERSION) ko-publish-capsule + $(HELM) package ./charts/capsule --app-version=$(ENTERPRISE_VERSION) --version=$(ENTERPRISE_VERSION) --destination ./builds/ + $(HELM) push ./builds/capsule-$(ENTERPRISE_VERSION).tgz oci://$(ENTERPRISE_REGISTRY)/charts/prereleases/ + $(MAKE) deploy-enterprise-prerelease + rm -rf ./builds + +deploy-enterprise-prerelease: @echo "" @echo "Deploying Capsule Prerelease (Enterprise) $(ENTERPRISE_VERSION)" @echo "" @@ -273,17 +343,18 @@ deploy-enterprise: @echo "2) Deploy Capsule:" @echo "" @echo "helm upgrade --install capsule \\" - @echo " oci://$(ENTERPRISE_REGISTRY)/prereleases/charts/capsule \\" + @echo " oci://$(ENTERPRISE_REGISTRY)/charts/prereleases/capsule \\" @echo " --namespace capsule-system \\" @echo " --version $(ENTERPRISE_VERSION) \\" @echo " --reuse-values \\" @echo " --set manager.image.registry=$(ENTERPRISE_REGISTRY) \\" - @echo " --set manager.image.repository=prereleases/images/capsule \\" - @echo " --set manager.image.tag=$(ENTERPRISE_VERSION) \\" + @echo " --set manager.image.repository=prereleases/capsule \\" + @echo " --set manager.image.tag=v$(ENTERPRISE_VERSION) \\" @echo " --set manager.image.pullPolicy=Always \\" @echo " --set 'serviceAccount.imagePullSecrets={capsule-enterprise}'" @echo "" + #################### # -- Docker #################### @@ -357,7 +428,7 @@ golint-fix: golangci-lint .PHONY: e2e-openshift e2e-openshift: ginkgo - $(MAKE) e2e-build-openshift && $(MAKE) e2e-exec FILTER='--label-filter="!skip && !skip-on-openshift"' && $(MAKE) e2e-destroy-openshift + $(MAKE) e2e-build-openshift && $(MAKE) e2e-exec FILTER='&& !skip && !skip-on-openshift' && $(MAKE) e2e-destroy-openshift e2e-build-openshift: minc $(MINC) config set provider docker @@ -384,6 +455,7 @@ e2e-build: kind .PHONY: e2e-install e2e-install: helm-controller-version ko-build-all $(MAKE) e2e-load-image CLUSTER_NAME=$(CLUSTER_NAME) IMAGE=$(CAPSULE_IMG) VERSION=$(VERSION) + $(KUBECTL) label clusterrole admin projectcapsule.dev/aggregate-to-controller=true $(HELM) upgrade \ --dependency-update \ --debug \ @@ -391,13 +463,30 @@ e2e-install: helm-controller-version ko-build-all --namespace capsule-system \ --create-namespace \ --set 'replicaCount=2'\ + --set 'certManager.generateCertificates=false' \ + --set 'tls.enableController=true' \ + --set 'tls.create=true' \ --set 'manager.image.pullPolicy=Never' \ --set 'manager.resources=null'\ --set "manager.image.tag=$(VERSION)" \ --set 'manager.livenessProbe.failureThreshold=10' \ + --set 'manager.options.logLevel=debug' \ + --set 'manager.options.workers=4' \ + --set 'manager.options.clientConnectionQPS=2000' \ + --set 'manager.options.clientConnectionQPS=1000' \ + --set 'manager.rbac.minimal=true' \ --set 'webhooks.hooks.nodes.enabled=true' \ --set "webhooks.exclusive=true"\ - --set "manager.options.logLevel=debug"\ + --set 'webhooks.hooks.calculations.enabled=true' \ + --set-string 'webhooks.hooks.calculations.rules[0].apiGroups[0]=' \ + --set 'webhooks.hooks.calculations.rules[0].apiVersions[0]=v1' \ + --set 'webhooks.hooks.calculations.rules[0].operations[0]=CREATE' \ + --set 'webhooks.hooks.calculations.rules[0].operations[1]=UPDATE' \ + --set 'webhooks.hooks.calculations.rules[0].operations[2]=DELETE' \ + --set 'webhooks.hooks.calculations.rules[0].resources[0]=pods' \ + --set 'webhooks.hooks.calculations.rules[0].resources[1]=persistentvolumeclaims' \ + --set 'webhooks.hooks.calculations.rules[0].scope=Namespaced' \ + --set 'webhooks.hooks.calculations.namespaceSelector.matchLabels.env=e2e' \ capsule \ ./charts/capsule @@ -470,7 +559,12 @@ e2e-load-image-openshift: minc .PHONY: e2e-exec e2e-exec: ginkgo - $(GINKGO) -v -tags e2e $(FILTER) ./e2e + $(GINKGO) -v -p -tags e2e --label-filter="!config $(FILTER)" ./e2e + $(MAKE) e2e-exec-config + +.PHONY: e2e-exec-config +e2e-exec-config: ginkgo + $(GINKGO) -v -tags e2e --label-filter="config $(FILTER)" ./e2e .PHONY: e2e-destroy e2e-destroy: dev-destroy @@ -556,7 +650,7 @@ nwa: $(call go-install-tool,$(NWA),github.com/$(NWA_LOOKUP)@$(NWA_VERSION)) GOLANGCI_LINT := $(LOCALBIN)/golangci-lint -GOLANGCI_LINT_VERSION := v2.8.0 +GOLANGCI_LINT_VERSION := v2.12.2 GOLANGCI_LINT_LOOKUP := golangci/golangci-lint golangci-lint: ## Download golangci-lint locally if necessary. @test -s $(GOLANGCI_LINT) && $(GOLANGCI_LINT) -h | grep -q $(GOLANGCI_LINT_VERSION) || \ diff --git a/PROJECT b/PROJECT index 11b2b886..533dc637 100644 --- a/PROJECT +++ b/PROJECT @@ -71,4 +71,13 @@ resources: kind: TenantOwner path: github.com/projectcapsule/capsule/api/v1beta2 version: v1beta2 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: clastix.io + group: capsule + kind: QuantityLedger + path: github.com/projectcapsule/capsule/api/v1beta2 + version: v1beta2 version: "3" diff --git a/README.md b/README.md index 74773bfa..4d838206 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,10 @@ Capsule takes a different approach. In a single cluster, the Capsule Controller On the other side, the Capsule Policy Engine keeps the different tenants isolated from each other. _Network and Security Policies_, _Resource Quota_, _Limit Ranges_, _RBAC_, and other policies defined at the tenant level are automatically inherited by all the namespaces in the tenant. Then users are free to operate their tenants in autonomy, without the intervention of the cluster administrator. + +# Project Status +This project is stable. [We follow the Kubernetes definition of stable](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-stages). + # Features ## Self-Service diff --git a/api/v1beta1/tenant_types.go b/api/v1beta1/tenant_types.go index af636df7..a605a597 100644 --- a/api/v1beta1/tenant_types.go +++ b/api/v1beta1/tenant_types.go @@ -7,6 +7,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) // TenantSpec defines the desired state of Tenant. @@ -36,7 +37,7 @@ type TenantSpec struct { // +optional ResourceQuota api.ResourceQuotaSpec `json:"resourceQuotas,omitzero"` // Specifies additional RoleBindings assigned to the Tenant. Capsule will ensure that all namespaces in the Tenant always contain the RoleBinding for the given ClusterRole. Optional. - AdditionalRoleBindings []api.AdditionalRoleBindingsSpec `json:"additionalRoleBindings,omitempty"` + AdditionalRoleBindings []rbac.AdditionalRoleBindingsSpec `json:"additionalRoleBindings,omitempty"` // Specify the allowed values for the imagePullPolicies option in Pod resources. Capsule assures that all Pod resources created in the Tenant can use only one of the allowed policy. Optional. ImagePullPolicies []api.ImagePullPolicySpec `json:"imagePullPolicies,omitempty"` // Specifies the allowed priorityClasses assigned to the Tenant. Capsule assures that all Pods resources created in the Tenant can use only one of the allowed PriorityClasses. Optional. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 72c9b30d..5baf39fc 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -9,6 +9,7 @@ package v1beta1 import ( "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -323,7 +324,7 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) { in.ResourceQuota.DeepCopyInto(&out.ResourceQuota) if in.AdditionalRoleBindings != nil { in, out := &in.AdditionalRoleBindings, &out.AdditionalRoleBindings - *out = make([]api.AdditionalRoleBindingsSpec, len(*in)) + *out = make([]rbac.AdditionalRoleBindingsSpec, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/api/v1beta2/capsuleconfiguration_status.go b/api/v1beta2/capsuleconfiguration_status.go index 15c6e1f4..d2c3ffd1 100644 --- a/api/v1beta2/capsuleconfiguration_status.go +++ b/api/v1beta2/capsuleconfiguration_status.go @@ -4,16 +4,11 @@ package v1beta2 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) // CapsuleConfigurationStatus defines the Capsule configuration status. type CapsuleConfigurationStatus struct { - // Last time all caches were invalided - LastCacheInvalidation metav1.Time `json:"lastCacheInvalidation,omitempty"` - // Users which are considered Capsule Users and are bound to the Capsule Tenant construct. - Users api.UserListSpec `json:"users,omitempty"` + Users rbac.UserListSpec `json:"users,omitempty"` } diff --git a/api/v1beta2/capsuleconfiguration_types.go b/api/v1beta2/capsuleconfiguration_types.go index 1f81dcba..3e592f82 100644 --- a/api/v1beta2/capsuleconfiguration_types.go +++ b/api/v1beta2/capsuleconfiguration_types.go @@ -4,26 +4,19 @@ package v1beta2 import ( - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" + "github.com/projectcapsule/capsule/pkg/runtime/admission" ) // CapsuleConfigurationSpec defines the Capsule configuration. type CapsuleConfigurationSpec struct { // Define entities which are considered part of the Capsule construct // Users not mentioned here will be ignored by Capsule - Users api.UserListSpec `json:"users,omitempty"` - // Deprecated: use users property instead (https://projectcapsule.dev/docs/operating/setup/configuration/#users) - // - // Names of the users considered as Capsule users. - UserNames []string `json:"userNames,omitempty"` - // Deprecated: use users property instead (https://projectcapsule.dev/docs/operating/setup/configuration/#users) - // - // Names of the groups considered as Capsule users. - UserGroups []string `json:"userGroups,omitempty"` + Users rbac.UserListSpec `json:"users,omitempty"` // Define groups which when found in the request of a user will be ignored by the Capsule // this might be useful if you have one group where all the users are in, but you want to separate administrators from normal users with additional groups. IgnoreUserWithGroups []string `json:"ignoreUserWithGroups,omitempty"` @@ -54,7 +47,7 @@ type CapsuleConfigurationSpec struct { // These entities are automatically owners for all existing tenants. Meaning they can add namespaces to any tenant. However they must be specific by using the capsule label // for interacting with namespaces. Because if that label is not defined, it's assumed that namespace interaction was not targeted towards a tenant and will therefor // be ignored by capsule. - Administrators api.UserListSpec `json:"administrators,omitempty"` + Administrators rbac.UserListSpec `json:"administrators,omitempty"` // Configuration for dynamic Validating and Mutating Admission webhooks managed by Capsule. Admission DynamicAdmission `json:"admission,omitempty"` // Define Properties for managed ClusterRoles by Capsule @@ -63,6 +56,18 @@ type CapsuleConfigurationSpec struct { // Define the period of time upon a cache invalidation is executed for all caches. // +kubebuilder:default="24h" CacheInvalidation metav1.Duration `json:"cacheInvalidation"` + // Service Account Client configuration for impersonation properties + // +optional + Impersonation ServiceAccountClient `json:"impersonation,omitzero"` + + // Deprecated: use users property instead (https://projectcapsule.dev/docs/operating/setup/configuration/#users) + // + // Names of the users considered as Capsule users. + UserNames []string `json:"userNames,omitempty"` + // Deprecated: use users property instead (https://projectcapsule.dev/docs/operating/setup/configuration/#users) + // + // Names of the groups considered as Capsule users. + UserGroups []string `json:"userGroups,omitempty"` } type RBACConfiguration struct { @@ -81,24 +86,31 @@ type RBACConfiguration struct { } type DynamicAdmission struct { + // Service Name of the Admission Service + // +kubebuilder:default=capsule-webhook-service + ServiceName string `json:"serviceName,omitempty"` + // Configure dynamic Mutating Admission for Capsule - Mutating DynamicAdmissionConfig `json:"mutating,omitempty"` + Mutating *DynamicMutatingAdmissionConfig `json:"mutating,omitempty"` // Configure dynamic Validating Admission for Capsule - Validating DynamicAdmissionConfig `json:"validating,omitempty"` + Validating *DynamicValidatingAdmissionConfig `json:"validating,omitempty"` } -type DynamicAdmissionConfig struct { - // Name the Admission Webhook - Name meta.RFC1123Name `json:"name,omitempty"` - // Labels added to the Admission Webhook +type DynamicValidatingAdmissionConfig struct { + admission.DynamicAdmissionConfig `json:",inline"` + + // Define Dynamic Admission Webhooks // +optional - Labels map[string]string `json:"labels,omitempty"` - // Annotations added to the Admission Webhook + Webhooks []*admission.ValidatingWebhook `json:"webhooks,omitempty"` +} + +type DynamicMutatingAdmissionConfig struct { + admission.DynamicAdmissionConfig `json:",inline"` + + // Define Dynamic Admission Webhooks // +optional - Annotations map[string]string `json:"annotations,omitempty"` - // From the upstram struct - Client admissionregistrationv1.WebhookClientConfig `json:"client"` + Webhooks []*admission.MutatingWebhook `json:"webhooks,omitempty"` } type NodeMetadata struct { @@ -115,14 +127,47 @@ type CapsuleResources struct { // Must be in the same Namespace where the Capsule Deployment is deployed. // +kubebuilder:default=capsule-tls TLSSecretName string `json:"TLSSecretName"` //nolint:tagliatelle + // Deprecated: use dynamic admission instead + // // Name of the MutatingWebhookConfiguration which contains the dynamic admission controller paths and resources. // +kubebuilder:default=capsule-mutating-webhook-configuration MutatingWebhookConfigurationName string `json:"mutatingWebhookConfigurationName"` + // Deprecated: use dynamic admission instead + // // Name of the ValidatingWebhookConfiguration which contains the dynamic admission controller paths and resources. // +kubebuilder:default=capsule-validating-webhook-configuration ValidatingWebhookConfigurationName string `json:"validatingWebhookConfigurationName"` } +// +kubebuilder:object:generate=true +type ServiceAccountClient struct { + // Kubernetes API Endpoint to use for impersonation + Endpoint string `json:"endpoint,omitempty"` + // Namespace where the CA certificate secret is located + CASecretNamespace meta.RFC1123SubdomainName `json:"caSecretNamespace,omitempty"` + // Name of the secret containing the CA certificate + CASecretName meta.RFC1123Name `json:"caSecretName,omitempty"` + // Key in the secret that holds the CA certificate (e.g., "ca.crt") + // +kubebuilder:default=ca.crt + CASecretKey string `json:"caSecretKey,omitempty"` + // If true, TLS certificate verification is skipped (not recommended for production) + // +kubebuilder:default=false + SkipTLSVerify bool `json:"skipTlsVerify,omitempty"` + // Default ServiceAccount for global resources (GlobalTenantResource) + // When defined, users are required to use this ServiceAccount anywhere in the cluster + // unless they explicitly provide their own. + GlobalDefaultServiceAccount meta.RFC1123Name `json:"globalDefaultServiceAccount,omitempty"` + // Default ServiceAccount for global resources (GlobalTenantResource) + // When defined, users are required to use this ServiceAccount anywhere in the cluster + // unless they explicitly provide their own. + // +optional + GlobalDefaultServiceAccountNamespace meta.RFC1123SubdomainName `json:"globalDefaultServiceAccountNamespace,omitempty"` + // Default ServiceAccount for namespaced resources (TenantResource) + // When defined, users are required to use this ServiceAccount within the namespace + // where they deploy the resource, unless they explicitly provide their own. + TenantDefaultServiceAccount meta.RFC1123Name `json:"tenantDefaultServiceAccount,omitempty"` +} + // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster diff --git a/api/v1beta2/customquota_func.go b/api/v1beta2/customquota_func.go new file mode 100644 index 00000000..592791ef --- /dev/null +++ b/api/v1beta2/customquota_func.go @@ -0,0 +1,28 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +func (c *CustomQuotaSpec) CollectJSONPathExpressions() (expressions []string) { + set := map[string]struct{}{} + + for _, source := range c.Sources { + if source.Path != "" { + set[source.Path] = struct{}{} + } + + for _, sel := range source.Selectors { + for _, fs := range sel.FieldSelectors { + if fs != "" { + set[fs] = struct{}{} + } + } + } + } + + for e := range set { + expressions = append(expressions, e) + } + + return expressions +} diff --git a/api/v1beta2/customquota_status.go b/api/v1beta2/customquota_status.go new file mode 100644 index 00000000..09b6d4c2 --- /dev/null +++ b/api/v1beta2/customquota_status.go @@ -0,0 +1,62 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +import ( + k8smeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +// CustomQuotaStatus defines the observed state of GlobalResourceQuota. +type CustomQuotaStatus struct { + // Usage measurements + // +optional + Usage CustomQuotaStatusUsage `json:"usage,omitempty"` + // Objects regarding this policy + Claims []CustomQuotaClaimItem `json:"claims,omitempty"` + // Targeting GVK + Targets []CustomQuotaStatusTarget `json:"targets"` + // Conditions + Conditions meta.ConditionList `json:"conditions"` +} + +func (s *CustomQuotaStatus) HasClaimUID(uid types.UID) bool { + for i := range s.Claims { + if s.Claims[i].UID == uid { + return true + } + } + + return false +} + +type CustomQuotaClaimItem struct { + metav1.GroupVersionKind `json:",inline"` + meta.NamespacedObjectWithUIDReference `json:",inline"` + + // Resource Quantity for given item + Usage resource.Quantity `json:"usage"` +} + +type CustomQuotaStatusTarget struct { + metav1.GroupVersionKind `json:",inline"` + CustomQuotaSpecSourceConfig `json:",inline"` + + // Path on GVK where usage is evaluated + Scope k8smeta.RESTScopeName `json:"scope,omitempty"` +} + +// CustomQuotaStatus defines the observed state of GlobalResourceQuota. +type CustomQuotaStatusUsage struct { + // Used is the current observed total usage of the resource. + // +optional + Used resource.Quantity `json:"used"` + // Used is the current observed total available of the resource (limit - used). + // +optional + Available resource.Quantity `json:"available"` +} diff --git a/api/v1beta2/customquota_types.go b/api/v1beta2/customquota_types.go new file mode 100644 index 00000000..2a37fa9b --- /dev/null +++ b/api/v1beta2/customquota_types.go @@ -0,0 +1,94 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/projectcapsule/capsule/pkg/runtime/gvk" + "github.com/projectcapsule/capsule/pkg/runtime/quota" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +// CustomQuotaSpec. +type CustomQuotaSpec struct { + // Select items governed by this quota + ScopeSelectors []metav1.LabelSelector `json:"scopeSelectors,omitempty"` + // Resource Quantity as limit + Limit resource.Quantity `json:"limit"` + // Target resource + Sources []CustomQuotaSpecSource `json:"sources,omitzero"` + // Additional Options for the CustomQuotaSpecification + // +kubebuilder:default:={emitMetricPerClaimUsage:false} + Options *CustomQuotaOptionsSpec `json:"options,omitzero"` +} + +// CustomQuotaOptionsSpec. +type CustomQuotaOptionsSpec struct { + // Additionally expose usage metrics for each claim contributing to the quota. + // This is disabled by default to avoid high cardinality in the metrics, but can be enabled for more granular monitoring and alerting. + // +kubebuilder:default:=false + EmitPerClaimMetrics bool `json:"emitMetricPerClaimUsage,omitempty"` +} + +// +kubebuilder:validation:XValidation:rule="self.op == 'count' ? !has(self.path) || size(self.path) == 0 : has(self.path) && size(self.path) > 0",message="path must be empty when op is 'count'; otherwise path must be set and non-empty" +type CustomQuotaSpecSource struct { + gvk.VersionKind `json:",inline"` + CustomQuotaSpecSourceConfig `json:",inline"` +} + +type CustomQuotaSpecSourceConfig struct { + // Path on GVK where usage is evaluated. + // Must be empty when op is "count". + // Required and non-empty for all other operations. + // +optional + Path string `json:"path,omitempty"` + + // Operation used to evaluate usage. + // +kubebuilder:default:=add + Operation quota.Operation `json:"op,omitempty"` + + // Provide more granular selectors for these sources + // The ScopeSelector and NamespaceSelector are always applied + // Allowing these selectors to make further selecting on the resulting subset. + Selectors []selectors.SelectorWithFields `json:"selectors,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced,shortName=cq +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Limit",type="string",JSONPath=".spec.limit",description="The total limit available" +// +kubebuilder:printcolumn:name="Used",type="string",JSONPath=".status.usage.used",description="The total used amount" +// +kubebuilder:printcolumn:name="Available",type="string",JSONPath=".status.usage.available",description="The total amount available" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="Reconcile Status" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="Reconcile Message" + +type CustomQuota struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ObjectMeta `json:"metadata,omitzero"` + + Spec CustomQuotaSpec `json:"spec"` + + // +optional + Status CustomQuotaStatus `json:"status,omitzero"` +} + +// +kubebuilder:object:root=true + +// CustomQuotaList contains a list of CustomQuota. +type CustomQuotaList struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ListMeta `json:"metadata,omitzero"` + + Items []CustomQuota `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CustomQuota{}, &CustomQuotaList{}) +} diff --git a/api/v1beta2/globalcustomquota_status.go b/api/v1beta2/globalcustomquota_status.go new file mode 100644 index 00000000..c8585d43 --- /dev/null +++ b/api/v1beta2/globalcustomquota_status.go @@ -0,0 +1,18 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +import "slices" + +// CustomQuotaStatus defines the observed state of GlobalResourceQuota. +type GlobalCustomQuotaStatus struct { + CustomQuotaStatus `json:",inline"` + + // Observed Namespaces + Namespaces []string `json:"namespaces,omitempty"` +} + +func (g *GlobalCustomQuotaStatus) NamespacePresent(ns string) bool { + return slices.Contains(g.Namespaces, ns) +} diff --git a/api/v1beta2/globalcustomquota_types.go b/api/v1beta2/globalcustomquota_types.go new file mode 100644 index 00000000..d379e597 --- /dev/null +++ b/api/v1beta2/globalcustomquota_types.go @@ -0,0 +1,55 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +// ClusterCustomQuotaSpec. +type GlobalCustomQuotaSpec struct { + CustomQuotaSpec `json:",inline"` + + // Select specifc namespaces where this Quota selects items. + NamespaceSelectors []selectors.NamespaceSelector `json:"namespaceSelectors,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,shortName=gcq +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Limit",type="string",JSONPath=".spec.limit",description="The total limit available" +// +kubebuilder:printcolumn:name="Used",type="string",JSONPath=".status.usage.used",description="The total used amount" +// +kubebuilder:printcolumn:name="Available",type="string",JSONPath=".status.usage.available",description="The total amount available" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="Reconcile Status" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="Reconcile Message" + +type GlobalCustomQuota struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ObjectMeta `json:"metadata,omitzero"` + + Spec GlobalCustomQuotaSpec `json:"spec"` + + // +optional + Status GlobalCustomQuotaStatus `json:"status,omitzero"` +} + +// +kubebuilder:object:root=true + +// ClusterCustomQuotaList contains a list of ClusterCustomQuota. +type GlobalCustomQuotaList struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ListMeta `json:"metadata,omitzero"` + + Items []GlobalCustomQuota `json:"items"` +} + +func init() { + SchemeBuilder.Register(&GlobalCustomQuota{}, &GlobalCustomQuotaList{}) +} diff --git a/api/v1beta2/namespace_rule_type.go b/api/v1beta2/namespace_rule_type.go deleted file mode 100644 index 79a8632c..00000000 --- a/api/v1beta2/namespace_rule_type.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package v1beta2 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/projectcapsule/capsule/pkg/api" -) - -// +kubebuilder:object:generate=true -type NamespaceRule struct { - // Enforce these properties via Rules - NamespaceRuleBody `json:",inline"` - - // Select namespaces which are going to usese - NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"` -} - -// +kubebuilder:object:generate=true -type NamespaceRuleBody struct { - // Enforcement Rules applied - //+optional - Enforce NamespaceRuleEnforceBody `json:"enforce,omitzero"` -} - -// +kubebuilder:object:generate=true -type NamespaceRuleEnforceBody struct { - // Define registries which are allowed to be used within this tenant - // The rules are aggregated, since you can use Regular Expressions the match registry endpoints - Registries []api.OCIRegistry `json:"registries,omitempty"` -} diff --git a/api/v1beta2/quantityledgers_status.go b/api/v1beta2/quantityledgers_status.go new file mode 100644 index 00000000..6d53d666 --- /dev/null +++ b/api/v1beta2/quantityledgers_status.go @@ -0,0 +1,68 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +// QuantityLedgerReservation represents one active inflight reservation. +// ID should be stable for retries of the same admission request. +// In practice, admission.Request.UID is a good default. +type QuantityLedgerReservation struct { + // Unique reservation identifier. + // +kubebuilder:validation:MinLength=1 + ID string `json:"id"` + + // Amount reserved for this request. + Usage resource.Quantity `json:"usage"` + + // Object that this reservation is intended to create/update. + ObjectRef QuantityLedgerObjectRef `json:"objectRef"` + + // Time the reservation was first created. + CreatedAt metav1.Time `json:"createdAt"` + + // Time the reservation was last refreshed or updated. + UpdatedAt metav1.Time `json:"updatedAt"` + + // Time after which the reservation may be considered stale. + // +optional + ExpiresAt *metav1.Time `json:"expiresAt,omitempty"` +} + +// QuantityLedgerPendingDelete tracks objects that are expected to disappear from claims +// soon, but may still temporarily appear during rebuild due to propagation delay. +type QuantityLedgerPendingDelete struct { + ObjectRef QuantityLedgerObjectRef `json:"objectRef"` + CreatedAt metav1.Time `json:"createdAt"` +} + +// QuantityLedgerStatus contains the mutable coordination state used by admission +// and quota controllers. +type QuantityLedgerStatus struct { + // Reserved is the aggregate sum of all active reservations. + // Controllers/webhooks should treat this as derived data from Reservations. + // +optional + Reserved resource.Quantity `json:"reserved,omitempty"` + + // Active inflight reservations for this quota. + // +optional + Reservations []QuantityLedgerReservation `json:"reservations,omitempty"` + + // Pending delete hints carried over from admission delete handling. + // +optional + PendingDeletes []QuantityLedgerPendingDelete `json:"pendingDeletes,omitempty"` + + // Conditions for the resource claim + // +optional + Conditions meta.ConditionList `json:"conditions,omitzero"` + + // Allocated is the admission-owned total that has been accepted by the webhook. + // It must be updated only through optimistic concurrency on QuantityLedger. + Allocated resource.Quantity `json:"allocated,omitempty"` +} diff --git a/api/v1beta2/quantityledgers_types.go b/api/v1beta2/quantityledgers_types.go new file mode 100644 index 00000000..2719c956 --- /dev/null +++ b/api/v1beta2/quantityledgers_types.go @@ -0,0 +1,98 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// QuotaLedgerTargetRef identifies the quota object that owns this ledger. +// Namespace is optional for cluster-scoped targets such as GlobalCustomQuota. +type QuantityLedgerTargetRef struct { + // APIGroup of the target quota resource, for example "capsule.clastix.io". + // +optional + APIGroup string `json:"apiGroup,omitempty"` + + // Kind of the target quota resource, for example "CustomQuota" or "GlobalCustomQuota". + // +kubebuilder:validation:MinLength=1 + Kind string `json:"kind"` + + // Namespace of the target quota resource. + // Must be empty for cluster-scoped targets. + // +optional + Namespace string `json:"namespace,omitempty"` + + // Name of the target quota resource. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // UID of the target quota resource. + // Optional, but useful for stale reference detection. + // +optional + UID types.UID `json:"uid,omitempty"` +} + +// QuotaLedgerObjectRef identifies the object for which a reservation exists. +// UID may be empty for CREATE admission before the object is persisted. +type QuantityLedgerObjectRef struct { + // APIGroup of the tracked object. + // +optional + APIGroup string `json:"apiGroup,omitempty"` + + // APIVersion of the tracked object, for example "v1". + // +kubebuilder:validation:MinLength=1 + APIVersion string `json:"apiVersion"` + + // Kind of the tracked object, for example "Pod". + // +kubebuilder:validation:MinLength=1 + Kind string `json:"kind"` + + // Namespace of the tracked object. + // +optional + Namespace string `json:"namespace,omitempty"` + + // Name of the tracked object. + // +optional + Name string `json:"name,omitempty"` + + // UID of the tracked object. + // +optional + UID types.UID `json:"uid,omitempty"` +} + +// QuotaLedgerSpec contains the immutable target reference. +type QuantityLedgerSpec struct { + // TargetRef points to the quota object that this ledger belongs to. + TargetRef QuantityLedgerTargetRef `json:"targetRef"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:path=quantityledgers,scope=Namespaced,shortName=ql +// +kubebuilder:printcolumn:name="TargetKind",type=string,JSONPath=`.spec.targetRef.kind` +// +kubebuilder:printcolumn:name="TargetNamespace",type=string,JSONPath=`.spec.targetRef.namespace` +// +kubebuilder:printcolumn:name="TargetName",type=string,JSONPath=`.spec.targetRef.name` +// +kubebuilder:printcolumn:name="Reserved",type=string,JSONPath=`.status.reserved` +// +kubebuilder:printcolumn:name="Reservations",type=integer,JSONPath=`.status.reservations.size()` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +type QuantityLedger struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec QuantityLedgerSpec `json:"spec,omitempty"` + Status QuantityLedgerStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +type QuantityLedgerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []QuantityLedger `json:"items"` +} + +func init() { + SchemeBuilder.Register(&QuantityLedger{}, &QuantityLedgerList{}) +} diff --git a/api/v1beta2/resourcepool_func.go b/api/v1beta2/resourcepool_func.go index 523f78fe..dea7b52b 100644 --- a/api/v1beta2/resourcepool_func.go +++ b/api/v1beta2/resourcepool_func.go @@ -5,7 +5,6 @@ package v1beta2 import ( "errors" - "fmt" "sort" corev1 "k8s.io/api/core/v1" @@ -15,7 +14,7 @@ import ( ) func (r *ResourcePool) GetQuotaName() string { - return fmt.Sprintf("capsule-pool-%s", r.GetName()) + return meta.NameForManagedPoolResourceQuota(r.GetName()) } func (r *ResourcePool) AssignNamespaces(namespaces []corev1.Namespace) { diff --git a/api/v1beta2/resourcepool_func_test.go b/api/v1beta2/resourcepool_func_test.go index bcc98b69..a1825276 100644 --- a/api/v1beta2/resourcepool_func_test.go +++ b/api/v1beta2/resourcepool_func_test.go @@ -1,18 +1,19 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package v1beta2 +package v1beta2_test import ( "testing" + "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api/meta" - "github.com/stretchr/testify/assert" ) func TestGetClaimFromStatus(t *testing.T) { @@ -20,7 +21,7 @@ func TestGetClaimFromStatus(t *testing.T) { testUID := types.UID("test-uid") otherUID := types.UID("wrong-uid") - claim := &ResourcePoolClaim{ + claim := &v1beta2.ResourcePoolClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "claim-a", Namespace: ns, @@ -28,11 +29,11 @@ func TestGetClaimFromStatus(t *testing.T) { }, } - pool := &ResourcePool{ - Status: ResourcePoolStatus{ - Claims: ResourcePoolNamespaceClaimsStatus{ + pool := &v1beta2.ResourcePool{ + Status: v1beta2.ResourcePoolStatus{ + Claims: v1beta2.ResourcePoolNamespaceClaimsStatus{ ns: { - &ResourcePoolClaimsItem{ + &v1beta2.ResourcePoolClaimsItem{ NamespacedRFC1123ObjectReferenceWithNamespaceWithUID: meta.NamespacedRFC1123ObjectReferenceWithNamespaceWithUID{ UID: testUID, }, @@ -76,21 +77,21 @@ func makeResourceList(cpu, memory string) corev1.ResourceList { } } -func makeClaim(name, ns string, uid types.UID, res corev1.ResourceList) *ResourcePoolClaim { - return &ResourcePoolClaim{ +func makeClaim(name, ns string, uid types.UID, res corev1.ResourceList) *v1beta2.ResourcePoolClaim { + return &v1beta2.ResourcePoolClaim{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: ns, UID: uid, }, - Spec: ResourcePoolClaimSpec{ + Spec: v1beta2.ResourcePoolClaimSpec{ ResourceClaims: res, }, } } func TestAssignNamespaces(t *testing.T) { - pool := &ResourcePool{} + pool := &v1beta2.ResourcePool{} namespaces := []corev1.Namespace{ {ObjectMeta: metav1.ObjectMeta{Name: "active-ns"}, Status: corev1.NamespaceStatus{Phase: corev1.NamespaceActive}}, @@ -104,12 +105,12 @@ func TestAssignNamespaces(t *testing.T) { } func TestAssignClaims(t *testing.T) { - pool := &ResourcePool{ - Status: ResourcePoolStatus{ - Claims: ResourcePoolNamespaceClaimsStatus{ + pool := &v1beta2.ResourcePool{ + Status: v1beta2.ResourcePoolStatus{ + Claims: v1beta2.ResourcePoolNamespaceClaimsStatus{ "ns": { - &ResourcePoolClaimsItem{}, - &ResourcePoolClaimsItem{}, + &v1beta2.ResourcePoolClaimsItem{}, + &v1beta2.ResourcePoolClaimsItem{}, }, }, }, @@ -120,7 +121,7 @@ func TestAssignClaims(t *testing.T) { } func TestAddRemoveClaimToStatus(t *testing.T) { - pool := &ResourcePool{} + pool := &v1beta2.ResourcePool{} claim := makeClaim("claim-1", "ns", "uid-1", makeResourceList("1", "1Gi")) pool.AddClaimToStatus(claim) @@ -135,16 +136,16 @@ func TestAddRemoveClaimToStatus(t *testing.T) { } func TestCalculateResources(t *testing.T) { - pool := &ResourcePool{ - Status: ResourcePoolStatus{ - Allocation: ResourcePoolQuotaStatus{ + pool := &v1beta2.ResourcePool{ + Status: v1beta2.ResourcePoolStatus{ + Allocation: v1beta2.ResourcePoolQuotaStatus{ Hard: corev1.ResourceList{ corev1.ResourceLimitsCPU: resource.MustParse("2"), }, }, - Claims: ResourcePoolNamespaceClaimsStatus{ + Claims: v1beta2.ResourcePoolNamespaceClaimsStatus{ "ns": { - &ResourcePoolClaimsItem{ + &v1beta2.ResourcePoolClaimsItem{ Claims: corev1.ResourceList{ corev1.ResourceLimitsCPU: resource.MustParse("1"), }, @@ -164,9 +165,9 @@ func TestCalculateResources(t *testing.T) { } func TestCanClaimFromPool(t *testing.T) { - pool := &ResourcePool{ - Status: ResourcePoolStatus{ - Allocation: ResourcePoolQuotaStatus{ + pool := &v1beta2.ResourcePool{ + Status: v1beta2.ResourcePoolStatus{ + Allocation: v1beta2.ResourcePoolQuotaStatus{ Hard: corev1.ResourceList{ corev1.ResourceLimitsMemory: resource.MustParse("1Gi"), }, @@ -189,16 +190,16 @@ func TestCanClaimFromPool(t *testing.T) { } func TestGetResourceQuotaHardResources(t *testing.T) { - pool := &ResourcePool{ - Spec: ResourcePoolSpec{ + pool := &v1beta2.ResourcePool{ + Spec: v1beta2.ResourcePoolSpec{ Defaults: corev1.ResourceList{ corev1.ResourceLimitsCPU: resource.MustParse("1"), }, }, - Status: ResourcePoolStatus{ - Claims: ResourcePoolNamespaceClaimsStatus{ + Status: v1beta2.ResourcePoolStatus{ + Claims: v1beta2.ResourcePoolNamespaceClaimsStatus{ "ns": { - &ResourcePoolClaimsItem{ + &v1beta2.ResourcePoolClaimsItem{ Claims: corev1.ResourceList{ corev1.ResourceLimitsCPU: resource.MustParse("1"), }, @@ -214,11 +215,11 @@ func TestGetResourceQuotaHardResources(t *testing.T) { } func TestGetNamespaceClaims(t *testing.T) { - pool := &ResourcePool{ - Status: ResourcePoolStatus{ - Claims: ResourcePoolNamespaceClaimsStatus{ + pool := &v1beta2.ResourcePool{ + Status: v1beta2.ResourcePoolStatus{ + Claims: v1beta2.ResourcePoolNamespaceClaimsStatus{ "ns": { - &ResourcePoolClaimsItem{ + &v1beta2.ResourcePoolClaimsItem{ NamespacedRFC1123ObjectReferenceWithNamespaceWithUID: meta.NamespacedRFC1123ObjectReferenceWithNamespaceWithUID{UID: "uid1"}, Claims: corev1.ResourceList{ corev1.ResourceLimitsCPU: resource.MustParse("1"), @@ -236,11 +237,11 @@ func TestGetNamespaceClaims(t *testing.T) { } func TestGetClaimedByNamespaceClaims(t *testing.T) { - pool := &ResourcePool{ - Status: ResourcePoolStatus{ - Claims: ResourcePoolNamespaceClaimsStatus{ + pool := &v1beta2.ResourcePool{ + Status: v1beta2.ResourcePoolStatus{ + Claims: v1beta2.ResourcePoolNamespaceClaimsStatus{ "ns1": { - &ResourcePoolClaimsItem{ + &v1beta2.ResourcePoolClaimsItem{ Claims: makeResourceList("1", "1Gi"), }, }, @@ -258,8 +259,8 @@ func TestGetClaimedByNamespaceClaims(t *testing.T) { func TestIsBoundToResourcePool_2(t *testing.T) { t.Run("bound to resource pool (Assigned=True)", func(t *testing.T) { - claim := &ResourcePoolClaim{ - Status: ResourcePoolClaimStatus{ + claim := &v1beta2.ResourcePoolClaim{ + Status: v1beta2.ResourcePoolClaimStatus{ Conditions: meta.ConditionList{}, }, } @@ -268,8 +269,8 @@ func TestIsBoundToResourcePool_2(t *testing.T) { }) t.Run("not bound - wrong condition type", func(t *testing.T) { - claim := &ResourcePoolClaim{ - Status: ResourcePoolClaimStatus{ + claim := &v1beta2.ResourcePoolClaim{ + Status: v1beta2.ResourcePoolClaimStatus{ Conditions: meta.ConditionList{ meta.Condition{}, }, @@ -284,8 +285,8 @@ func TestIsBoundToResourcePool_2(t *testing.T) { }) t.Run("not bound - condition not true", func(t *testing.T) { - claim := &ResourcePoolClaim{ - Status: ResourcePoolClaimStatus{ + claim := &v1beta2.ResourcePoolClaim{ + Status: v1beta2.ResourcePoolClaimStatus{ Conditions: meta.ConditionList{ meta.Condition{}, }, @@ -300,8 +301,8 @@ func TestIsBoundToResourcePool_2(t *testing.T) { }) t.Run("not bound - condition not true", func(t *testing.T) { - claim := &ResourcePoolClaim{ - Status: ResourcePoolClaimStatus{ + claim := &v1beta2.ResourcePoolClaim{ + Status: v1beta2.ResourcePoolClaimStatus{ Conditions: meta.ConditionList{ meta.Condition{}, }, diff --git a/api/v1beta2/resourcepoolclaim_func_test.go b/api/v1beta2/resourcepoolclaim_func_test.go index 402b6799..16065a16 100644 --- a/api/v1beta2/resourcepoolclaim_func_test.go +++ b/api/v1beta2/resourcepoolclaim_func_test.go @@ -1,7 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package v1beta2 +package v1beta2_test import ( "testing" @@ -9,36 +9,34 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api/meta" ) func TestIsBoundToResourcePool(t *testing.T) { tests := []struct { name string - claim ResourcePoolClaim + claim v1beta2.ResourcePoolClaim expected bool }{ { name: "bound to resource pool (Assigned=True)", - claim: ResourcePoolClaim{ - Status: ResourcePoolClaimStatus{ - Conditions: meta.ConditionList{ - meta.Condition{ - Type: meta.BoundCondition, - Status: metav1.ConditionTrue, - Reason: meta.SucceededReason, - Message: "reconciled", - LastTransitionTime: metav1.Now(), - }, + claim: v1beta2.ResourcePoolClaim{ + Status: v1beta2.ResourcePoolClaimStatus{ + Condition: metav1.Condition{ + Type: meta.BoundCondition, + Status: metav1.ConditionTrue, + Reason: meta.SucceededReason, + Message: "reconciled", + LastTransitionTime: metav1.Now(), }, }, }, - expected: true, }, { name: "not bound - wrong condition type", - claim: ResourcePoolClaim{ - Status: ResourcePoolClaimStatus{ + claim: v1beta2.ResourcePoolClaim{ + Status: v1beta2.ResourcePoolClaimStatus{ Conditions: meta.ConditionList{ meta.Condition{ Type: meta.AssignedCondition, @@ -54,8 +52,8 @@ func TestIsBoundToResourcePool(t *testing.T) { }, { name: "not bound - status not true", - claim: ResourcePoolClaim{ - Status: ResourcePoolClaimStatus{ + claim: v1beta2.ResourcePoolClaim{ + Status: v1beta2.ResourcePoolClaimStatus{ Conditions: meta.ConditionList{ meta.Condition{ Type: meta.AssignedCondition, @@ -71,8 +69,8 @@ func TestIsBoundToResourcePool(t *testing.T) { }, { name: "not bound - empty condition", - claim: ResourcePoolClaim{ - Status: ResourcePoolClaimStatus{}, + claim: v1beta2.ResourcePoolClaim{ + Status: v1beta2.ResourcePoolClaimStatus{}, }, expected: false, }, @@ -89,16 +87,16 @@ func TestIsBoundToResourcePool(t *testing.T) { func TestGetPool(t *testing.T) { tests := []struct { name string - claim ResourcePoolClaim + claim v1beta2.ResourcePoolClaim expected string }{ { name: "returns status pool name when set", - claim: ResourcePoolClaim{ - Spec: ResourcePoolClaimSpec{ + claim: v1beta2.ResourcePoolClaim{ + Spec: v1beta2.ResourcePoolClaimSpec{ Pool: "spec-pool", }, - Status: ResourcePoolClaimStatus{ + Status: v1beta2.ResourcePoolClaimStatus{ Pool: meta.LocalRFC1123ObjectReferenceWithUID{ Name: meta.RFC1123Name("status-pool"), }, @@ -108,11 +106,11 @@ func TestGetPool(t *testing.T) { }, { name: "falls back to spec pool when status pool name is empty", - claim: ResourcePoolClaim{ - Spec: ResourcePoolClaimSpec{ + claim: v1beta2.ResourcePoolClaim{ + Spec: v1beta2.ResourcePoolClaimSpec{ Pool: "spec-pool", }, - Status: ResourcePoolClaimStatus{ + Status: v1beta2.ResourcePoolClaimStatus{ Pool: meta.LocalRFC1123ObjectReferenceWithUID{ Name: meta.RFC1123Name(""), }, @@ -122,11 +120,11 @@ func TestGetPool(t *testing.T) { }, { name: "falls back to spec pool when status pool struct is zero-value", - claim: ResourcePoolClaim{ - Spec: ResourcePoolClaimSpec{ + claim: v1beta2.ResourcePoolClaim{ + Spec: v1beta2.ResourcePoolClaimSpec{ Pool: "spec-pool", }, - Status: ResourcePoolClaimStatus{ + Status: v1beta2.ResourcePoolClaimStatus{ Pool: meta.LocalRFC1123ObjectReferenceWithUID{}, }, }, @@ -134,11 +132,11 @@ func TestGetPool(t *testing.T) { }, { name: "returns empty when both status and spec are empty", - claim: ResourcePoolClaim{ - Spec: ResourcePoolClaimSpec{ + claim: v1beta2.ResourcePoolClaim{ + Spec: v1beta2.ResourcePoolClaimSpec{ Pool: "", }, - Status: ResourcePoolClaimStatus{ + Status: v1beta2.ResourcePoolClaimStatus{ Pool: meta.LocalRFC1123ObjectReferenceWithUID{ Name: meta.RFC1123Name(""), }, @@ -148,11 +146,11 @@ func TestGetPool(t *testing.T) { }, { name: "status wins even if spec differs", - claim: ResourcePoolClaim{ - Spec: ResourcePoolClaimSpec{ + claim: v1beta2.ResourcePoolClaim{ + Spec: v1beta2.ResourcePoolClaimSpec{ Pool: "spec-pool", }, - Status: ResourcePoolClaimStatus{ + Status: v1beta2.ResourcePoolClaimStatus{ Pool: meta.LocalRFC1123ObjectReferenceWithUID{ Name: meta.RFC1123Name("status-pool"), }, diff --git a/api/v1beta2/rule_status_type.go b/api/v1beta2/rule_status_type.go index e121c3fc..94f3a106 100644 --- a/api/v1beta2/rule_status_type.go +++ b/api/v1beta2/rule_status_type.go @@ -5,8 +5,21 @@ package v1beta2 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" ) +// RuleStatus contains the accumulated rules applying to namespace it's deployed in. +// +kubebuilder:object:generate=true +type RuleStatusSpec struct { + // Managed Enforcement properties per Namespace (aggregated from rules) + //+optional + Rule api.NamespaceRuleBodyNamespace `json:"rule,omitzero"` + // Conditions + Conditions meta.ConditionList `json:"conditions"` +} + // +kubebuilder:object:root=true // +kubebuilder:storageversion // +kubebuilder:subresource:status @@ -17,6 +30,9 @@ type RuleStatus struct { // +optional metav1.ObjectMeta `json:"metadata,omitzero"` + // +optional + Spec []*api.NamespaceRuleBodyNamespace `json:"spec,omitzero"` + // +optional Status RuleStatusSpec `json:"status,omitzero"` } @@ -34,11 +50,3 @@ type RuleStatusList struct { func init() { SchemeBuilder.Register(&RuleStatus{}, &RuleStatusList{}) } - -// RuleStatus contains the accumulated rules applying to namespace it's deployed in. -// +kubebuilder:object:generate=true -type RuleStatusSpec struct { - // Managed Enforcement properties per Namespace (aggregated from rules) - //+optional - Rule NamespaceRuleBody `json:"rule,omitzero"` -} diff --git a/api/v1beta2/tenant_conversion_hub.go b/api/v1beta2/tenant_conversion_hub.go index dc12dcb6..927f33b3 100644 --- a/api/v1beta2/tenant_conversion_hub.go +++ b/api/v1beta2/tenant_conversion_hub.go @@ -13,6 +13,7 @@ import ( capsulev1beta1 "github.com/projectcapsule/capsule/api/v1beta1" "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) func (in *Tenant) ConvertFrom(raw conversion.Hub) error { @@ -27,28 +28,28 @@ func (in *Tenant) ConvertFrom(raw conversion.Hub) error { } in.ObjectMeta = src.ObjectMeta - in.Spec.Owners = make(api.OwnerListSpec, 0, len(src.Spec.Owners)) + in.Spec.Owners = make(rbac.OwnerListSpec, 0, len(src.Spec.Owners)) for index, owner := range src.Spec.Owners { - proxySettings := make([]api.ProxySettings, 0, len(owner.ProxyOperations)) + proxySettings := make([]rbac.ProxySettings, 0, len(owner.ProxyOperations)) for _, proxyOp := range owner.ProxyOperations { - ops := make([]api.ProxyOperation, 0, len(proxyOp.Operations)) + ops := make([]rbac.ProxyOperation, 0, len(proxyOp.Operations)) for _, op := range proxyOp.Operations { - ops = append(ops, api.ProxyOperation(op)) + ops = append(ops, rbac.ProxyOperation(op)) } - proxySettings = append(proxySettings, api.ProxySettings{ - Kind: api.ProxyServiceKind(proxyOp.Kind), + proxySettings = append(proxySettings, rbac.ProxySettings{ + Kind: rbac.ProxyServiceKind(proxyOp.Kind), Operations: ops, }) } - in.Spec.Owners = append(in.Spec.Owners, api.OwnerSpec{ - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.OwnerKind(owner.Kind), + in.Spec.Owners = append(in.Spec.Owners, rbac.OwnerSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.OwnerKind(owner.Kind), Name: owner.Name, }, ClusterRoles: owner.GetRoles(*src, index), @@ -281,6 +282,7 @@ func (in *Tenant) ConvertTo(raw conversion.Hub) error { dst.Status.Size = in.Status.Size dst.Status.Namespaces = in.Status.Namespaces + //nolint:exhaustive switch in.Status.State { case TenantStateActive: dst.Status.State = capsulev1beta1.TenantStateActive diff --git a/api/v1beta2/tenant_func.go b/api/v1beta2/tenant_func.go index 65bdd261..1fdf19da 100644 --- a/api/v1beta2/tenant_func.go +++ b/api/v1beta2/tenant_func.go @@ -4,17 +4,20 @@ package v1beta2 import ( - "slices" + "context" "sort" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -func (in *Tenant) GetRoleBindings() []api.AdditionalRoleBindingsSpec { - roleBindings := make([]api.AdditionalRoleBindingsSpec, 0, len(in.Spec.AdditionalRoleBindings)) +func (in *Tenant) GetRoleBindings() []rbac.AdditionalRoleBindingsSpec { + roleBindings := make([]rbac.AdditionalRoleBindingsSpec, 0, len(in.Spec.AdditionalRoleBindings)) for _, owner := range in.Status.Owners { roleBindings = append(roleBindings, owner.ToAdditionalRolebindings()...) @@ -25,6 +28,16 @@ func (in *Tenant) GetRoleBindings() []api.AdditionalRoleBindingsSpec { return roleBindings } +func (in *Tenant) GetPromotionRoleBindings() []rbac.AdditionalRoleBindingsWithNamespaceSpec { + roleBindings := make([]rbac.AdditionalRoleBindingsWithNamespaceSpec, 0, len(in.Status.Promotions)) + + for _, promotion := range in.Status.Promotions { + roleBindings = append(roleBindings, promotion.ToAdditionalRolebindings()...) + } + + return roleBindings +} + func (in *Tenant) IsFull() bool { // we don't have limits on assigned Namespaces if in.Spec.NamespaceOptions == nil || in.Spec.NamespaceOptions.Quota == nil { @@ -35,12 +48,10 @@ func (in *Tenant) IsFull() bool { } func (in *Tenant) AssignNamespaces(namespaces []corev1.Namespace) { - var l []string + l := make([]string, 0, len(namespaces)) for _, ns := range namespaces { - if ns.Status.Phase == corev1.NamespaceActive { - l = append(l, ns.GetName()) - } + l = append(l, ns.GetName()) } sort.Strings(l) @@ -49,14 +60,44 @@ func (in *Tenant) AssignNamespaces(namespaces []corev1.Namespace) { in.Status.Size = uint(len(l)) } -func (in *Tenant) GetOwnerProxySettings(name string, kind api.OwnerKind) []api.ProxySettings { +func (in *Tenant) GetNamespaces() (res []string) { + return in.Status.Namespaces +} + +// Fetch all namespaces defined in the status. +func (in *Tenant) GetNamespaceObjects(ctx context.Context, c client.Reader) (namespaces []corev1.Namespace, err error) { + nsList := &corev1.NamespaceList{} + + if len(in.Status.Namespaces) == 0 { + return nsList.Items, nil + } + + req, err := labels.NewRequirement( + corev1.LabelMetadataName, + selection.In, + in.Status.Namespaces, + ) + if err != nil { + return nil, err + } + + selector := labels.NewSelector().Add(*req) + + if err := c.List(ctx, nsList, client.MatchingLabelsSelector{Selector: selector}); err != nil { + return nil, err + } + + return nsList.Items, nil +} + +func (in *Tenant) GetOwnerProxySettings(name string, kind rbac.OwnerKind) []rbac.ProxySettings { return in.Spec.Owners.FindOwner(name, kind).ProxyOperations } // GetClusterRolePermissions returns a map where the clusterRole is the key // and the value is a list of permission subjects (kind and name) that reference that role. // These mappings are gathered from the owners and additionalRolebindings spec. -func (in *Tenant) GetSubjectsByClusterRoles(ignoreOwnerKind []api.OwnerKind) (rolePerms map[string][]rbacv1.Subject) { +func (in *Tenant) GetSubjectsByClusterRoles(ignoreOwnerKind []rbac.OwnerKind) (rolePerms map[string][]rbacv1.Subject) { rolePerms = make(map[string][]rbacv1.Subject) // Helper to add permissions for a given clusterRole @@ -80,7 +121,7 @@ func (in *Tenant) GetSubjectsByClusterRoles(ignoreOwnerKind []api.OwnerKind) (ro } // Process owners - for _, owner := range in.Spec.Owners { + for _, owner := range in.Status.Owners { if !isIgnoredKind(owner.Kind.String()) { for _, clusterRole := range owner.ClusterRoles { perm := rbacv1.Subject{ @@ -108,72 +149,87 @@ func (in *Tenant) GetSubjectsByClusterRoles(ignoreOwnerKind []api.OwnerKind) (ro return rolePerms } -// Get the permissions for a tenant ordered by groups and users. -func (in *Tenant) GetClusterRolesBySubject(ignoreOwnerKind []api.OwnerKind) (maps map[string]map[string]api.TenantSubjectRoles) { - maps = make(map[string]map[string]api.TenantSubjectRoles) +func (in *Tenant) GetClusterRolesBySubject(ignoreOwnerKind []rbac.OwnerKind) []rbac.SubjectRoles { + ignore := make(map[string]struct{}, len(ignoreOwnerKind)) + for _, k := range ignoreOwnerKind { + ignore[k.String()] = struct{}{} + } - // Initialize a nested map for kind ("User", "Group") and name - initNestedMap := func(kind string) { - if _, exists := maps[kind]; !exists { - maps[kind] = make(map[string]api.TenantSubjectRoles) + roleSet := map[string]map[string]map[string]struct{}{} + + ensure := func(kind, name string) map[string]struct{} { + km, ok := roleSet[kind] + if !ok { + km = map[string]map[string]struct{}{} + roleSet[kind] = km + } + + ns, ok := km[name] + if !ok { + ns = map[string]struct{}{} + km[name] = ns + } + + return ns + } + + for _, owner := range in.Status.Owners { + kind := owner.Kind.String() + if _, skip := ignore[kind]; skip { + continue + } + + s := ensure(kind, owner.Name) + for _, r := range owner.ClusterRoles { + s[r] = struct{}{} } } - // Helper to check if a kind is in the ignoreOwnerKind list - isIgnoredKind := func(kind string) bool { - for _, ignored := range ignoreOwnerKind { - if kind == ignored.String() { - return true + + for _, rb := range in.Spec.AdditionalRoleBindings { + for _, subj := range rb.Subjects { + if _, skip := ignore[subj.Kind]; skip { + continue } - } - return false + s := ensure(subj.Kind, subj.Name) + s[rb.ClusterRoleName] = struct{}{} + } } - // Process owners - for _, owner := range in.Spec.Owners { - if !isIgnoredKind(owner.Kind.String()) { - initNestedMap(owner.Kind.String()) + // Flatten deterministically: sort kinds, names, roles + kinds := make([]string, 0, len(roleSet)) + for k := range roleSet { + kinds = append(kinds, k) + } - if perm, exists := maps[owner.Kind.String()][owner.Name]; exists { - // If the permission entry already exists, append cluster roles - perm.ClusterRoles = append(perm.ClusterRoles, owner.ClusterRoles...) - maps[owner.Kind.String()][owner.Name] = perm - } else { - // Create a new permission entry - maps[owner.Kind.String()][owner.Name] = api.TenantSubjectRoles{ - ClusterRoles: owner.ClusterRoles, - } + sort.Strings(kinds) + + totalSubjects := 0 + for _, byName := range roleSet { + totalSubjects += len(byName) + } + + out := make([]rbac.SubjectRoles, 0, totalSubjects) + + for _, kind := range kinds { + names := make([]string, 0, len(roleSet[kind])) + for n := range roleSet[kind] { + names = append(names, n) + } + + sort.Strings(names) + + for _, name := range names { + roles := make([]string, 0, len(roleSet[kind][name])) + for r := range roleSet[kind][name] { + roles = append(roles, r) } + + sort.Strings(roles) + + out = append(out, rbac.SubjectRoles{Kind: kind, Name: name, Roles: roles}) } } - // Process additional role bindings - for _, role := range in.Spec.AdditionalRoleBindings { - for _, subject := range role.Subjects { - if !isIgnoredKind(subject.Kind) { - initNestedMap(subject.Kind) - - if perm, exists := maps[subject.Kind][subject.Name]; exists { - // If the permission entry already exists, append cluster roles - perm.ClusterRoles = append(perm.ClusterRoles, role.ClusterRoleName) - maps[subject.Kind][subject.Name] = perm - } else { - // Create a new permission entry - maps[subject.Kind][subject.Name] = api.TenantSubjectRoles{ - ClusterRoles: []string{role.ClusterRoleName}, - } - } - } - } - } - - // Remove duplicates from cluster roles in both maps - for kind, nameMap := range maps { - for name, perm := range nameMap { - perm.ClusterRoles = slices.Compact(perm.ClusterRoles) - maps[kind][name] = perm - } - } - - return maps + return out } diff --git a/api/v1beta2/tenant_func_test.go b/api/v1beta2/tenant_func_test.go index 0857dfa5..2c1b256d 100644 --- a/api/v1beta2/tenant_func_test.go +++ b/api/v1beta2/tenant_func_test.go @@ -1,83 +1,78 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package v1beta2 +package v1beta2_test import ( "reflect" "testing" - "github.com/projectcapsule/capsule/pkg/api" rbacv1 "k8s.io/api/rbac/v1" + + "github.com/projectcapsule/capsule/api/v1beta2" + capsulerbac "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var tenant = &Tenant{ - Spec: TenantSpec{ - Owners: []api.OwnerSpec{ - { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: "User", +func testTenant() *v1beta2.Tenant { + return &v1beta2.Tenant{ + Spec: v1beta2.TenantSpec{ + AdditionalRoleBindings: []capsulerbac.AdditionalRoleBindingsSpec{ + { + ClusterRoleName: "developer", + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "user2"}, + {Kind: "Group", Name: "group1"}, + }, + }, + { + ClusterRoleName: "cluster-admin", + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "user3"}, + {Kind: "Group", Name: "group1"}, + }, + }, + { + ClusterRoleName: "deployer", + Subjects: []rbacv1.Subject{ + {Kind: "ServiceAccount", Name: "system:serviceaccount:argocd:argo-operator"}, + }, + }, + }, + }, + Status: v1beta2.TenantStatus{ + Owners: capsulerbac.OwnerStatusListSpec{ + { + UserSpec: capsulerbac.UserSpec{ + Kind: capsulerbac.UserOwner, Name: "user1", }, ClusterRoles: []string{"cluster-admin", "read-only"}, }, - }, - { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: "Group", + { + UserSpec: capsulerbac.UserSpec{ + Kind: capsulerbac.GroupOwner, Name: "group1", }, ClusterRoles: []string{"edit"}, }, - }, - { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, + { + UserSpec: capsulerbac.UserSpec{ + Kind: capsulerbac.ServiceAccountOwner, Name: "service", }, ClusterRoles: []string{"read-only"}, }, }, }, - AdditionalRoleBindings: []api.AdditionalRoleBindingsSpec{ - { - ClusterRoleName: "developer", - Subjects: []rbacv1.Subject{ - {Kind: "User", Name: "user2"}, - {Kind: "Group", Name: "group1"}, - }, - }, - { - ClusterRoleName: "cluster-admin", - Subjects: []rbacv1.Subject{ - { - Kind: "User", - Name: "user3", - }, - { - Kind: "Group", - Name: "group1", - }, - }, - }, - { - ClusterRoleName: "deployer", - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: "system:serviceaccount:argocd:argo-operator", - }, - }, - }, - }, - }, + } + } -// TestGetClusterRolePermissions tests the GetClusterRolePermissions function func TestGetSubjectsByClusterRoles(t *testing.T) { + t.Parallel() + + tenant := testTenant() + expected := map[string][]rbacv1.Subject{ "cluster-admin": { {Kind: "User", Name: "user1"}, @@ -100,15 +95,11 @@ func TestGetSubjectsByClusterRoles(t *testing.T) { }, } - // Call the function to test - permissions := tenant.GetSubjectsByClusterRoles(nil) - - if !reflect.DeepEqual(permissions, expected) { - t.Errorf("Expected %v, but got %v", expected, permissions) + got := tenant.GetSubjectsByClusterRoles(nil) + if !reflect.DeepEqual(got, expected) { + t.Fatalf("expected %#v\n got %#v", expected, got) } - // Ignore SubjectTypes (Ignores ServiceAccounts) - ignored := tenant.GetSubjectsByClusterRoles([]api.OwnerKind{"ServiceAccount"}) expectedIgnored := map[string][]rbacv1.Subject{ "cluster-admin": { {Kind: "User", Name: "user1"}, @@ -127,78 +118,121 @@ func TestGetSubjectsByClusterRoles(t *testing.T) { }, } - if !reflect.DeepEqual(ignored, expectedIgnored) { - t.Errorf("Expected %v, but got %v", expectedIgnored, ignored) + gotIgnored := tenant.GetSubjectsByClusterRoles([]capsulerbac.OwnerKind{capsulerbac.ServiceAccountOwner}) + if !reflect.DeepEqual(gotIgnored, expectedIgnored) { + t.Fatalf("expected %#v\n got %#v", expectedIgnored, gotIgnored) } - } -func TestGetClusterRolesBySubject(t *testing.T) { +func TestGetClusterRolesBySubjectSorted(t *testing.T) { + t.Parallel() - expected := map[string]map[string]api.TenantSubjectRoles{ - "User": { - "user1": { - ClusterRoles: []string{"cluster-admin", "read-only"}, - }, - "user2": { - ClusterRoles: []string{"developer"}, - }, - "user3": { - ClusterRoles: []string{"cluster-admin"}, + tenant := &v1beta2.Tenant{ + Spec: v1beta2.TenantSpec{ + AdditionalRoleBindings: []capsulerbac.AdditionalRoleBindingsSpec{ + { + ClusterRoleName: "deployer", + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "system:serviceaccount:argocd:argo-operator", + }, + }, + }, + { + ClusterRoleName: "developer", + Subjects: []rbacv1.Subject{ + { + Kind: "Group", + Name: "group1", + }, + }, + }, }, }, - "Group": { - "group1": { - ClusterRoles: []string{"edit", "developer", "cluster-admin"}, - }, - }, - "ServiceAccount": { - "service": { - ClusterRoles: []string{"read-only"}, - }, - "system:serviceaccount:argocd:argo-operator": { - ClusterRoles: []string{"deployer"}, + Status: v1beta2.TenantStatus{ + Owners: capsulerbac.OwnerStatusListSpec{ + { + UserSpec: capsulerbac.UserSpec{ + Kind: capsulerbac.UserOwner, + Name: "user1", + }, + ClusterRoles: []string{"cluster-admin", "read-only"}, + }, + { + UserSpec: capsulerbac.UserSpec{ + Kind: capsulerbac.UserOwner, + Name: "user2", + }, + ClusterRoles: []string{"developer"}, + }, + { + UserSpec: capsulerbac.UserSpec{ + Kind: capsulerbac.UserOwner, + Name: "user3", + }, + ClusterRoles: []string{"cluster-admin"}, + }, + { + UserSpec: capsulerbac.UserSpec{ + Kind: capsulerbac.GroupOwner, + Name: "group1", + }, + ClusterRoles: []string{"edit", "developer", "cluster-admin"}, + }, + { + UserSpec: capsulerbac.UserSpec{ + Kind: capsulerbac.ServiceAccountOwner, + Name: "service", + }, + ClusterRoles: []string{"read-only"}, + }, }, }, } - permissions := tenant.GetClusterRolesBySubject(nil) - if !reflect.DeepEqual(permissions, expected) { - t.Errorf("Expected %v, but got %v", expected, permissions) + expected := []capsulerbac.SubjectRoles{ + {Kind: "Group", Name: "group1", Roles: []string{"cluster-admin", "developer", "edit"}}, + {Kind: "ServiceAccount", Name: "service", Roles: []string{"read-only"}}, + {Kind: "ServiceAccount", Name: "system:serviceaccount:argocd:argo-operator", Roles: []string{"deployer"}}, + {Kind: "User", Name: "user1", Roles: []string{"cluster-admin", "read-only"}}, + {Kind: "User", Name: "user2", Roles: []string{"developer"}}, + {Kind: "User", Name: "user3", Roles: []string{"cluster-admin"}}, } - delete(expected, "ServiceAccount") - ignored := tenant.GetClusterRolesBySubject([]api.OwnerKind{"ServiceAccount"}) + t.Run("includes all kinds and is deterministic, deduped and sorted", func(t *testing.T) { + t.Parallel() - if !reflect.DeepEqual(ignored, expected) { - t.Errorf("Expected %v, but got %v", expected, ignored) - } -} - -// Helper function to run tests -func TestMain(t *testing.M) { - t.Run() -} - -// permissionsEqual checks the equality of two TenantPermission structs. -func permissionsEqual(a, b api.TenantSubjectRoles) bool { - if a.Kind != b.Kind { - return false - } - if len(a.ClusterRoles) != len(b.ClusterRoles) { - return false - } - - // Create a map to count occurrences of cluster roles - counts := make(map[string]int) - for _, role := range a.ClusterRoles { - counts[role]++ - } - for _, role := range b.ClusterRoles { - counts[role]-- - if counts[role] < 0 { - return false // More occurrences in b than in a + got := tenant.GetClusterRolesBySubject(nil) + if !reflect.DeepEqual(got, expected) { + t.Fatalf("expected %#v\n got %#v", expected, got) } - } - return true + }) + + t.Run("ignores ServiceAccount kind", func(t *testing.T) { + t.Parallel() + + got := tenant.GetClusterRolesBySubject([]capsulerbac.OwnerKind{capsulerbac.ServiceAccountOwner}) + + expectedIgnored := []capsulerbac.SubjectRoles{ + {Kind: "Group", Name: "group1", Roles: []string{"cluster-admin", "developer", "edit"}}, + {Kind: "User", Name: "user1", Roles: []string{"cluster-admin", "read-only"}}, + {Kind: "User", Name: "user2", Roles: []string{"developer"}}, + {Kind: "User", Name: "user3", Roles: []string{"cluster-admin"}}, + } + + if !reflect.DeepEqual(got, expectedIgnored) { + t.Fatalf("expected %#v\n got %#v", expectedIgnored, got) + } + }) + + t.Run("empty tenant yields empty slice", func(t *testing.T) { + t.Parallel() + + empty := &v1beta2.Tenant{} + got := empty.GetClusterRolesBySubject(nil) + if len(got) != 0 { + t.Fatalf("expected empty, got %#v", got) + } + }) } diff --git a/api/v1beta2/tenant_status.go b/api/v1beta2/tenant_status.go index 0ef14caf..7e05f877 100644 --- a/api/v1beta2/tenant_status.go +++ b/api/v1beta2/tenant_status.go @@ -8,14 +8,16 @@ import ( "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -// +kubebuilder:validation:Enum=Cordoned;Active +// +kubebuilder:validation:Enum=Cordoned;Active;Terminating type tenantState string const ( - TenantStateActive tenantState = "Active" - TenantStateCordoned tenantState = "Cordoned" + TenantStateActive tenantState = "Active" + TenantStateCordoned tenantState = "Cordoned" + TenantStateTerminating tenantState = "Terminating" ) // Returns the observed state of the Tenant. @@ -24,9 +26,11 @@ type TenantStatus struct { TenantAvailableStatus `json:",inline"` // Collected owners for this tenant - Owners api.OwnerStatusListSpec `json:"owners,omitempty"` + Owners rbac.OwnerStatusListSpec `json:"owners,omitempty"` + // Promoted ServiceAccounts across the Tenant + Promotions rbac.PromotionStatusListSpec `json:"promotions,omitempty"` // +kubebuilder:default=Active - // The operational state of the Tenant. Possible values are "Active", "Cordoned". + // The operational state of the Tenant. Possible values are "Active", "Cordoned" or "Terminating". State tenantState `json:"state"` // How many namespaces are assigned to the Tenant. Size uint `json:"size"` @@ -52,6 +56,16 @@ type TenantStatusNamespaceItem struct { Enforce TenantStatusNamespaceEnforcement `json:"enforce,omitzero"` } +// RuleStatus contains the accumulated rules applying to namespace it's deployed in. +// +kubebuilder:object:generate=true +type TenantStatusRuleStatusItem struct { + // Promotions originating from this namespace + Promotions rbac.OwnerStatusListSpec `json:"promotions,omitempty"` + + // Target Namespaces for this rule + TargetNamespaces []meta.RFC1123SubdomainName `json:"namespaces,omitempty"` +} + type TenantStatusNamespaceEnforcement struct { // Registries which are allowed within this namespace Registries []api.OCIRegistry `json:"registry,omitempty"` diff --git a/api/v1beta2/tenant_types.go b/api/v1beta2/tenant_types.go index cb740d3a..c80b6134 100644 --- a/api/v1beta2/tenant_types.go +++ b/api/v1beta2/tenant_types.go @@ -6,16 +6,23 @@ package v1beta2 import ( "context" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/runtime/selectors" ) // TenantSpec defines the desired state of Tenant. type TenantSpec struct { + // Specify additional data relating to the tenant. + // Mainly useable in templating and more accessible than labels/annotations. + // +optional + Data apiextensionsv1.JSON `json:"data"` + // Specify Permissions for the Tenant. // +optional Permissions Permissions `json:"permissions,omitzero"` @@ -25,11 +32,11 @@ type TenantSpec struct { // // Read More: https://projectcapsule.dev/docs/tenants/rules/ //+optional - Rules []*NamespaceRule `json:"rules,omitzero"` + Rules []*api.NamespaceRuleBodyTenant `json:"rules,omitzero"` // Specifies the owners of the Tenant. // Optional - Owners api.OwnerListSpec `json:"owners,omitempty"` + Owners rbac.OwnerListSpec `json:"owners,omitempty"` // Specifies options for the Namespaces, such as additional metadata or maximum number of namespaces allowed for that Tenant. Once the namespace quota assigned to the Tenant has been reached, the Tenant owner cannot create further namespaces. Optional. NamespaceOptions *NamespaceOptions `json:"namespaceOptions,omitempty"` // Specifies options for the Service, such as additional metadata or block of certain type of Services. Optional. @@ -50,7 +57,7 @@ type TenantSpec struct { // +optional ResourceQuota api.ResourceQuotaSpec `json:"resourceQuotas,omitzero"` // Specifies additional RoleBindings assigned to the Tenant. Capsule will ensure that all namespaces in the Tenant always contain the RoleBinding for the given ClusterRole. Optional. - AdditionalRoleBindings []api.AdditionalRoleBindingsSpec `json:"additionalRoleBindings,omitempty"` + AdditionalRoleBindings []rbac.AdditionalRoleBindingsSpec `json:"additionalRoleBindings,omitempty"` // Specifies the allowed RuntimeClasses assigned to the Tenant. // Capsule assures that all Pods resources created in the Tenant can use only one of the allowed RuntimeClasses. // Optional. @@ -108,11 +115,15 @@ type Permissions struct { // The elements are OR operations and independent. You can see the resulting Tenant Owners // in the Status.Owners specification of the Tenant. MatchOwners []*metav1.LabelSelector `json:"matchOwners,omitempty"` + + // ClusterRoles granted to the promoted ServiceAccounts across the Tenant + //+kubebuilder:default:=true + AllowOwnerPromotion bool `json:"allowOwnerPromotion,omitempty"` } func (p *Permissions) ListMatchingOwners( ctx context.Context, - c client.Client, + c client.Reader, tnt string, opts ...client.ListOption, ) ([]*TenantOwner, error) { @@ -150,10 +161,6 @@ type Tenant struct { Status TenantStatus `json:"status,omitzero"` } -func (in *Tenant) GetNamespaces() (res []string) { - return in.Status.Namespaces -} - // +kubebuilder:object:root=true // TenantList contains a list of Tenant. diff --git a/api/v1beta2/tenantowner_types.go b/api/v1beta2/tenantowner_types.go index 906b38c8..55b3d0da 100644 --- a/api/v1beta2/tenantowner_types.go +++ b/api/v1beta2/tenantowner_types.go @@ -6,13 +6,13 @@ package v1beta2 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) // TenantOwnerSpec defines the desired state of TenantOwner. type TenantOwnerSpec struct { // Subject - api.CoreOwnerSpec `json:",inline"` + rbac.CoreOwnerSpec `json:",inline"` // Adds the given subject as capsule user. When enabled this subject does not have to be // mentioned in the CapsuleConfiguration as Capsule User. In almost all scenarios Tenant Owners @@ -26,7 +26,7 @@ type TenantOwnerStatus struct{} // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:resource:scope=Cluster +// +kubebuilder:resource:scope=Cluster,shortName=to // TenantOwner is the Schema for the tenantowners API. type TenantOwner struct { diff --git a/api/v1beta2/tenantresource_global.go b/api/v1beta2/tenantresource_global.go index 6d102236..068d8732 100644 --- a/api/v1beta2/tenantresource_global.go +++ b/api/v1beta2/tenantresource_global.go @@ -5,13 +5,25 @@ package v1beta2 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" + + "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" ) // GlobalTenantResourceSpec defines the desired state of GlobalTenantResource. type GlobalTenantResourceSpec struct { - TenantResourceSpec `json:",inline"` + TenantResourceCommonSpec `json:",inline"` + // Local ServiceAccount which will perform all the actions defined in the TenantResource + // You must provide permissions accordingly to that ServiceAccount + //+optional + ServiceAccount *meta.NamespacedRFC1123ObjectReferenceWithNamespace `json:"serviceAccount,omitzero"` + // Resource Scope, Can either be + // - Tenant: Create Resources for each tenant in selected Tenants + // - Namespace: Create Resources for each namespace in selected Tenants + // +kubebuilder:default:=Namespace + // +optional + Scope api.ResourceScope `json:"scope"` // Defines the Tenant selector used target the tenants on which resources must be propagated. // +optional TenantSelector metav1.LabelSelector `json:"tenantSelector,omitzero"` @@ -19,27 +31,19 @@ type GlobalTenantResourceSpec struct { // GlobalTenantResourceStatus defines the observed state of GlobalTenantResource. type GlobalTenantResourceStatus struct { + TenantResourceCommonStatus `json:",inline"` + // List of Tenants addressed by the GlobalTenantResource. - SelectedTenants []string `json:"selectedTenants"` - // List of the replicated resources for the given TenantResource. - ProcessedItems ProcessedItems `json:"processedItems,omitzero"` -} - -type ProcessedItems []ObjectReferenceStatus - -func (p *ProcessedItems) AsSet() sets.Set[string] { - set := sets.New[string]() - - for _, i := range *p { - set.Insert(i.String()) - } - - return set + SelectedTenants []string `json:"selectedTenants,omitempty"` } // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:resource:scope=Cluster +// +kubebuilder:resource:scope=Cluster,shortName=gtr +// +kubebuilder:printcolumn:name="Items",type="integer",JSONPath=".status.size",description="The total amount of items being replicated" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="Reconcile Status for the tenant" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="Reconcile Message for the tenant" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age" // GlobalTenantResource allows to propagate resource replications to a specific subset of Tenant resources. type GlobalTenantResource struct { diff --git a/api/v1beta2/tenantresource_global_func.go b/api/v1beta2/tenantresource_global_func.go new file mode 100644 index 00000000..c56e1409 --- /dev/null +++ b/api/v1beta2/tenantresource_global_func.go @@ -0,0 +1,20 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +import ( + "sort" +) + +func (in *GlobalTenantResource) AssignTenants(tnts []Tenant) { + l := make([]string, 0, len(tnts)) + + for _, tnt := range tnts { + l = append(l, tnt.GetName()) + } + + sort.Strings(l) + + in.Status.SelectedTenants = l +} diff --git a/api/v1beta2/tenantresource_namespaced.go b/api/v1beta2/tenantresource_namespaced.go index a9cd1d65..98925834 100644 --- a/api/v1beta2/tenantresource_namespaced.go +++ b/api/v1beta2/tenantresource_namespaced.go @@ -5,52 +5,31 @@ package v1beta2 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" ) // TenantResourceSpec defines the desired state of TenantResource. type TenantResourceSpec struct { - // Define the period of time upon a second reconciliation must be invoked. - // Keep in mind that any change to the manifests will trigger a new reconciliation. - // +kubebuilder:default="60s" - ResyncPeriod metav1.Duration `json:"resyncPeriod"` - // When the replicated resource manifest is deleted, all the objects replicated so far will be automatically deleted. - // Disable this to keep replicated resources although the deletion of the replication manifest. - // +kubebuilder:default=true - PruningOnDelete *bool `json:"pruningOnDelete,omitempty"` - // Defines the rules to select targeting Namespace, along with the objects that must be replicated. - Resources []ResourceSpec `json:"resources"` -} + TenantResourceCommonSpec `json:",inline"` -type ResourceSpec struct { - // Defines the Namespace selector to select the Tenant Namespaces on which the resources must be propagated. - // In case of nil value, all the Tenant Namespaces are targeted. - NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"` - // List of the resources already existing in other Namespaces that must be replicated. - NamespacedItems []ObjectReference `json:"namespacedItems,omitempty"` - // List of raw resources that must be replicated. - RawItems []RawExtension `json:"rawItems,omitempty"` - // Besides the Capsule metadata required by TenantResource controller, defines additional metadata that must be - // added to the replicated resources. - AdditionalMetadata *api.AdditionalMetadataSpec `json:"additionalMetadata,omitempty"` -} - -// +kubebuilder:validation:XEmbeddedResource -// +kubebuilder:validation:XPreserveUnknownFields -type RawExtension struct { - runtime.RawExtension `json:",inline"` + // Local ServiceAccount which will perform all the actions defined in the TenantResource + // You must provide permissions accordingly to that ServiceAccount + //+optional + ServiceAccount *meta.LocalRFC1123ObjectReference `json:"serviceAccount,omitzero"` } // TenantResourceStatus defines the observed state of TenantResource. type TenantResourceStatus struct { - // List of the replicated resources for the given TenantResource. - ProcessedItems ProcessedItems `json:"processedItems"` + TenantResourceCommonStatus `json:",inline"` } // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Items",type="integer",JSONPath=".status.size",description="The total amount of items being replicated" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="Reconcile Status for the tenant" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="Reconcile Message for the tenant" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age" // TenantResource allows a Tenant Owner, if enabled with proper RBAC, to propagate resources in its Namespace. // The object must be deployed in a Tenant Namespace, and cannot reference object living in non-Tenant namespaces. diff --git a/api/v1beta2/tenantresource_types.go b/api/v1beta2/tenantresource_types.go index 0cd6f8e9..401857ad 100644 --- a/api/v1beta2/tenantresource_types.go +++ b/api/v1beta2/tenantresource_types.go @@ -4,71 +4,98 @@ package v1beta2 import ( - "fmt" - "strings" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + tpl "github.com/projectcapsule/capsule/pkg/template" ) -type ObjectReferenceAbstract struct { - // Kind of the referent. - // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - Kind string `json:"kind"` - // Namespace of the referent. - // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - Namespace string `json:"namespace"` - // API version of the referent. - APIVersion string `json:"apiVersion,omitempty"` +type TenantResourceCommonStatus struct { + // Condition of the GlobalTenantResource. + Conditions meta.ConditionList `json:"conditions,omitempty"` + + // List of the replicated resources for the given TenantResource. + //+optional + ProcessedItems meta.ProcessedItems `json:"processedItems,omitzero"` + + // How many items are being replicated by the TenantResource. + Size uint `json:"size"` + + // Serviceaccount used for impersonation + //+optional + ServiceAccount *meta.NamespacedRFC1123ObjectReferenceWithNamespace `json:"serviceAccount,omitzero"` } -type ObjectReferenceStatus struct { - ObjectReferenceAbstract `json:",inline"` - - // Name of the referent. - // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - Name string `json:"name"` +func (s *TenantResourceCommonStatus) UpdateStats() { + s.Size = uint(len(s.ProcessedItems)) } -type ObjectReference struct { - ObjectReferenceAbstract `json:",inline"` - - // Label selector used to select the given resources in the given Namespace. - Selector metav1.LabelSelector `json:"selector"` +type TenantResourceCommonSpec struct { + // Provide additional settings + // +kubebuilder:default={} + Settings TenantResourceCommonSpecSettings `json:"settings,omitzero"` + // DependsOn may contain a meta.NamespacedObjectReference slice + // with references to TenantResource resources that must be ready before this + // TenantResource can be reconciled. + // +optional + DependsOn []meta.LocalRFC1123ObjectReference `json:"dependsOn,omitempty"` + // Define the period of time upon a second reconciliation must be invoked. + // Keep in mind that any change to the manifests will trigger a new reconciliation. + // +kubebuilder:default="60s" + ResyncPeriod metav1.Duration `json:"resyncPeriod"` + // When the replicated resource manifest is deleted, all the objects replicated so far will be automatically deleted. + // Disable this to keep replicated resources although the deletion of the replication manifest. + // +kubebuilder:default=true + PruningOnDelete *bool `json:"pruningOnDelete,omitempty"` + // When cordoning a replication it will no longer execute any applies or deletions (paused). + // This is useful for maintenances + // +kubebuilder:default=false + Cordoned *bool `json:"cordoned,omitempty"` + // Defines the rules to select targeting Namespace, along with the objects that must be replicated. + Resources []ResourceSpec `json:"resources"` } -func (in *ObjectReferenceStatus) String() string { - return fmt.Sprintf("Kind=%s,APIVersion=%s,Namespace=%s,Name=%s", in.Kind, in.APIVersion, in.Namespace, in.Name) +type TenantResourceCommonSpecSettings struct { + // Enabling this allows TenanResources to interact with objects which were not created by a TenantResource. In this case on prune no deletion of the entire object is made. + // +kubebuilder:default=false + Adopt *bool `json:"adopt,omitempty"` + // Force indicates that in case of conflicts with server-side apply, the client should acquire ownership of the conflicting field. + // You may create collisions with this. + // +kubebuilder:default=false + Force *bool `json:"force,omitempty"` } -func (in *ObjectReferenceStatus) ParseFromString(value string) error { - rawParts := strings.Split(value, ",") +type ResourceSpec struct { + // Defines the Namespace selector to select the Tenant Namespaces on which the resources must be propagated. + // In case of nil value, all the Tenant Namespaces are targeted. + NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"` + // List of the resources already existing in other Namespaces that must be replicated. + NamespacedItems []tpl.ResourceReference `json:"namespacedItems,omitempty"` + // List of raw resources that must be replicated. + RawItems []RawExtension `json:"rawItems,omitempty"` + // Besides the Capsule metadata required by TenantResource controller, defines additional metadata that must be + // added to the replicated resources. + AdditionalMetadata *api.AdditionalMetadataSpec `json:"additionalMetadata,omitempty"` + // Templates for advanced use cases + Generators []TemplateItemSpec `json:"generators,omitempty"` + // Provide additional template context, which can be used throughout all + // the declared items for the replication + // +optional + Context *tpl.TemplateContext `json:"context,omitempty"` +} - if len(rawParts) != 4 { - return fmt.Errorf("unexpected raw parts") - } +// +kubebuilder:validation:XPreserveUnknownFields +type RawExtension struct { + runtime.RawExtension `json:",inline"` +} - for _, i := range rawParts { - parts := strings.Split(i, "=") - - if len(parts) != 2 { - return fmt.Errorf("unrecognized separator") - } - - k, v := parts[0], parts[1] - - switch k { - case "Kind": - in.Kind = v - case "APIVersion": - in.APIVersion = v - case "Namespace": - in.Namespace = v - case "Name": - in.Name = v - default: - return fmt.Errorf("unrecognized marker: %s", k) - } - } - - return nil +type TemplateItemSpec struct { + // Template contains any amount of yaml which is applied to Kubernetes. + // This can be a single resource or multiple resources + Template string `json:"template,omitempty"` + // Missing Key Option for templating + // +kubebuilder:default=zero + MissingKey tpl.MissingKeyOption `json:"missingKey,omitempty"` } diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 3ecdc35d..34279fb6 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -10,7 +10,10 @@ package v1beta2 import ( "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" + "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/selectors" + "github.com/projectcapsule/capsule/pkg/template" corev1 "k8s.io/api/core/v1" "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -101,17 +104,7 @@ func (in *CapsuleConfigurationSpec) DeepCopyInto(out *CapsuleConfigurationSpec) *out = *in if in.Users != nil { in, out := &in.Users, &out.Users - *out = make(api.UserListSpec, len(*in)) - copy(*out, *in) - } - if in.UserNames != nil { - in, out := &in.UserNames, &out.UserNames - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.UserGroups != nil { - in, out := &in.UserGroups, &out.UserGroups - *out = make([]string, len(*in)) + *out = make(rbac.UserListSpec, len(*in)) copy(*out, *in) } if in.IgnoreUserWithGroups != nil { @@ -127,7 +120,7 @@ func (in *CapsuleConfigurationSpec) DeepCopyInto(out *CapsuleConfigurationSpec) } if in.Administrators != nil { in, out := &in.Administrators, &out.Administrators - *out = make(api.UserListSpec, len(*in)) + *out = make(rbac.UserListSpec, len(*in)) copy(*out, *in) } in.Admission.DeepCopyInto(&out.Admission) @@ -137,6 +130,17 @@ func (in *CapsuleConfigurationSpec) DeepCopyInto(out *CapsuleConfigurationSpec) (*in).DeepCopyInto(*out) } out.CacheInvalidation = in.CacheInvalidation + out.Impersonation = in.Impersonation + if in.UserNames != nil { + in, out := &in.UserNames, &out.UserNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.UserGroups != nil { + in, out := &in.UserGroups, &out.UserGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CapsuleConfigurationSpec. @@ -152,10 +156,9 @@ func (in *CapsuleConfigurationSpec) DeepCopy() *CapsuleConfigurationSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CapsuleConfigurationStatus) DeepCopyInto(out *CapsuleConfigurationStatus) { *out = *in - in.LastCacheInvalidation.DeepCopyInto(&out.LastCacheInvalidation) if in.Users != nil { in, out := &in.Users, &out.Users - *out = make(api.UserListSpec, len(*in)) + *out = make(rbac.UserListSpec, len(*in)) copy(*out, *in) } } @@ -185,11 +188,256 @@ func (in *CapsuleResources) DeepCopy() *CapsuleResources { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomQuota) DeepCopyInto(out *CustomQuota) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomQuota. +func (in *CustomQuota) DeepCopy() *CustomQuota { + if in == nil { + return nil + } + out := new(CustomQuota) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CustomQuota) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomQuotaClaimItem) DeepCopyInto(out *CustomQuotaClaimItem) { + *out = *in + out.GroupVersionKind = in.GroupVersionKind + out.NamespacedObjectWithUIDReference = in.NamespacedObjectWithUIDReference + out.Usage = in.Usage.DeepCopy() +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomQuotaClaimItem. +func (in *CustomQuotaClaimItem) DeepCopy() *CustomQuotaClaimItem { + if in == nil { + return nil + } + out := new(CustomQuotaClaimItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomQuotaList) DeepCopyInto(out *CustomQuotaList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CustomQuota, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomQuotaList. +func (in *CustomQuotaList) DeepCopy() *CustomQuotaList { + if in == nil { + return nil + } + out := new(CustomQuotaList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CustomQuotaList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomQuotaOptionsSpec) DeepCopyInto(out *CustomQuotaOptionsSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomQuotaOptionsSpec. +func (in *CustomQuotaOptionsSpec) DeepCopy() *CustomQuotaOptionsSpec { + if in == nil { + return nil + } + out := new(CustomQuotaOptionsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomQuotaSpec) DeepCopyInto(out *CustomQuotaSpec) { + *out = *in + if in.ScopeSelectors != nil { + in, out := &in.ScopeSelectors, &out.ScopeSelectors + *out = make([]metav1.LabelSelector, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.Limit = in.Limit.DeepCopy() + if in.Sources != nil { + in, out := &in.Sources, &out.Sources + *out = make([]CustomQuotaSpecSource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Options != nil { + in, out := &in.Options, &out.Options + *out = new(CustomQuotaOptionsSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomQuotaSpec. +func (in *CustomQuotaSpec) DeepCopy() *CustomQuotaSpec { + if in == nil { + return nil + } + out := new(CustomQuotaSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomQuotaSpecSource) DeepCopyInto(out *CustomQuotaSpecSource) { + *out = *in + out.VersionKind = in.VersionKind + in.CustomQuotaSpecSourceConfig.DeepCopyInto(&out.CustomQuotaSpecSourceConfig) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomQuotaSpecSource. +func (in *CustomQuotaSpecSource) DeepCopy() *CustomQuotaSpecSource { + if in == nil { + return nil + } + out := new(CustomQuotaSpecSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomQuotaSpecSourceConfig) DeepCopyInto(out *CustomQuotaSpecSourceConfig) { + *out = *in + if in.Selectors != nil { + in, out := &in.Selectors, &out.Selectors + *out = make([]selectors.SelectorWithFields, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomQuotaSpecSourceConfig. +func (in *CustomQuotaSpecSourceConfig) DeepCopy() *CustomQuotaSpecSourceConfig { + if in == nil { + return nil + } + out := new(CustomQuotaSpecSourceConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomQuotaStatus) DeepCopyInto(out *CustomQuotaStatus) { + *out = *in + in.Usage.DeepCopyInto(&out.Usage) + if in.Claims != nil { + in, out := &in.Claims, &out.Claims + *out = make([]CustomQuotaClaimItem, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Targets != nil { + in, out := &in.Targets, &out.Targets + *out = make([]CustomQuotaStatusTarget, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(meta.ConditionList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomQuotaStatus. +func (in *CustomQuotaStatus) DeepCopy() *CustomQuotaStatus { + if in == nil { + return nil + } + out := new(CustomQuotaStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomQuotaStatusTarget) DeepCopyInto(out *CustomQuotaStatusTarget) { + *out = *in + out.GroupVersionKind = in.GroupVersionKind + in.CustomQuotaSpecSourceConfig.DeepCopyInto(&out.CustomQuotaSpecSourceConfig) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomQuotaStatusTarget. +func (in *CustomQuotaStatusTarget) DeepCopy() *CustomQuotaStatusTarget { + if in == nil { + return nil + } + out := new(CustomQuotaStatusTarget) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomQuotaStatusUsage) DeepCopyInto(out *CustomQuotaStatusUsage) { + *out = *in + out.Used = in.Used.DeepCopy() + out.Available = in.Available.DeepCopy() +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomQuotaStatusUsage. +func (in *CustomQuotaStatusUsage) DeepCopy() *CustomQuotaStatusUsage { + if in == nil { + return nil + } + out := new(CustomQuotaStatusUsage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DynamicAdmission) DeepCopyInto(out *DynamicAdmission) { *out = *in - in.Mutating.DeepCopyInto(&out.Mutating) - in.Validating.DeepCopyInto(&out.Validating) + if in.Mutating != nil { + in, out := &in.Mutating, &out.Mutating + *out = new(DynamicMutatingAdmissionConfig) + (*in).DeepCopyInto(*out) + } + if in.Validating != nil { + in, out := &in.Validating, &out.Validating + *out = new(DynamicValidatingAdmissionConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamicAdmission. @@ -203,31 +451,55 @@ func (in *DynamicAdmission) DeepCopy() *DynamicAdmission { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DynamicAdmissionConfig) DeepCopyInto(out *DynamicAdmissionConfig) { +func (in *DynamicMutatingAdmissionConfig) DeepCopyInto(out *DynamicMutatingAdmissionConfig) { *out = *in - if in.Labels != nil { - in, out := &in.Labels, &out.Labels - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val + in.DynamicAdmissionConfig.DeepCopyInto(&out.DynamicAdmissionConfig) + if in.Webhooks != nil { + in, out := &in.Webhooks, &out.Webhooks + *out = make([]*admission.MutatingWebhook, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(admission.MutatingWebhook) + (*in).DeepCopyInto(*out) + } } } - if in.Annotations != nil { - in, out := &in.Annotations, &out.Annotations - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - in.Client.DeepCopyInto(&out.Client) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamicAdmissionConfig. -func (in *DynamicAdmissionConfig) DeepCopy() *DynamicAdmissionConfig { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamicMutatingAdmissionConfig. +func (in *DynamicMutatingAdmissionConfig) DeepCopy() *DynamicMutatingAdmissionConfig { if in == nil { return nil } - out := new(DynamicAdmissionConfig) + out := new(DynamicMutatingAdmissionConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DynamicValidatingAdmissionConfig) DeepCopyInto(out *DynamicValidatingAdmissionConfig) { + *out = *in + in.DynamicAdmissionConfig.DeepCopyInto(&out.DynamicAdmissionConfig) + if in.Webhooks != nil { + in, out := &in.Webhooks, &out.Webhooks + *out = make([]*admission.ValidatingWebhook, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(admission.ValidatingWebhook) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamicValidatingAdmissionConfig. +func (in *DynamicValidatingAdmissionConfig) DeepCopy() *DynamicValidatingAdmissionConfig { + if in == nil { + return nil + } + out := new(DynamicValidatingAdmissionConfig) in.DeepCopyInto(out) return out } @@ -252,6 +524,109 @@ func (in *GatewayOptions) DeepCopy() *GatewayOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalCustomQuota) DeepCopyInto(out *GlobalCustomQuota) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalCustomQuota. +func (in *GlobalCustomQuota) DeepCopy() *GlobalCustomQuota { + if in == nil { + return nil + } + out := new(GlobalCustomQuota) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GlobalCustomQuota) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalCustomQuotaList) DeepCopyInto(out *GlobalCustomQuotaList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GlobalCustomQuota, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalCustomQuotaList. +func (in *GlobalCustomQuotaList) DeepCopy() *GlobalCustomQuotaList { + if in == nil { + return nil + } + out := new(GlobalCustomQuotaList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GlobalCustomQuotaList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalCustomQuotaSpec) DeepCopyInto(out *GlobalCustomQuotaSpec) { + *out = *in + in.CustomQuotaSpec.DeepCopyInto(&out.CustomQuotaSpec) + if in.NamespaceSelectors != nil { + in, out := &in.NamespaceSelectors, &out.NamespaceSelectors + *out = make([]selectors.NamespaceSelector, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalCustomQuotaSpec. +func (in *GlobalCustomQuotaSpec) DeepCopy() *GlobalCustomQuotaSpec { + if in == nil { + return nil + } + out := new(GlobalCustomQuotaSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalCustomQuotaStatus) DeepCopyInto(out *GlobalCustomQuotaStatus) { + *out = *in + in.CustomQuotaStatus.DeepCopyInto(&out.CustomQuotaStatus) + if in.Namespaces != nil { + in, out := &in.Namespaces, &out.Namespaces + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalCustomQuotaStatus. +func (in *GlobalCustomQuotaStatus) DeepCopy() *GlobalCustomQuotaStatus { + if in == nil { + return nil + } + out := new(GlobalCustomQuotaStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GlobalTenantResource) DeepCopyInto(out *GlobalTenantResource) { *out = *in @@ -314,7 +689,12 @@ func (in *GlobalTenantResourceList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GlobalTenantResourceSpec) DeepCopyInto(out *GlobalTenantResourceSpec) { *out = *in - in.TenantResourceSpec.DeepCopyInto(&out.TenantResourceSpec) + in.TenantResourceCommonSpec.DeepCopyInto(&out.TenantResourceCommonSpec) + if in.ServiceAccount != nil { + in, out := &in.ServiceAccount, &out.ServiceAccount + *out = new(meta.NamespacedRFC1123ObjectReferenceWithNamespace) + **out = **in + } in.TenantSelector.DeepCopyInto(&out.TenantSelector) } @@ -331,16 +711,12 @@ func (in *GlobalTenantResourceSpec) DeepCopy() *GlobalTenantResourceSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GlobalTenantResourceStatus) DeepCopyInto(out *GlobalTenantResourceStatus) { *out = *in + in.TenantResourceCommonStatus.DeepCopyInto(&out.TenantResourceCommonStatus) if in.SelectedTenants != nil { in, out := &in.SelectedTenants, &out.SelectedTenants *out = make([]string, len(*in)) copy(*out, *in) } - if in.ProcessedItems != nil { - in, out := &in.ProcessedItems, &out.ProcessedItems - *out = make(ProcessedItems, len(*in)) - copy(*out, *in) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalTenantResourceStatus. @@ -417,65 +793,6 @@ func (in *NamespaceOptions) DeepCopy() *NamespaceOptions { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NamespaceRule) DeepCopyInto(out *NamespaceRule) { - *out = *in - in.NamespaceRuleBody.DeepCopyInto(&out.NamespaceRuleBody) - if in.NamespaceSelector != nil { - in, out := &in.NamespaceSelector, &out.NamespaceSelector - *out = new(metav1.LabelSelector) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceRule. -func (in *NamespaceRule) DeepCopy() *NamespaceRule { - if in == nil { - return nil - } - out := new(NamespaceRule) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NamespaceRuleBody) DeepCopyInto(out *NamespaceRuleBody) { - *out = *in - in.Enforce.DeepCopyInto(&out.Enforce) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceRuleBody. -func (in *NamespaceRuleBody) DeepCopy() *NamespaceRuleBody { - if in == nil { - return nil - } - out := new(NamespaceRuleBody) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NamespaceRuleEnforceBody) DeepCopyInto(out *NamespaceRuleEnforceBody) { - *out = *in - if in.Registries != nil { - in, out := &in.Registries, &out.Registries - *out = make([]api.OCIRegistry, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceRuleEnforceBody. -func (in *NamespaceRuleEnforceBody) DeepCopy() *NamespaceRuleEnforceBody { - if in == nil { - return nil - } - out := new(NamespaceRuleEnforceBody) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeMetadata) DeepCopyInto(out *NodeMetadata) { *out = *in @@ -508,54 +825,6 @@ func (in *NonLimitedResourceError) DeepCopy() *NonLimitedResourceError { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ObjectReference) DeepCopyInto(out *ObjectReference) { - *out = *in - out.ObjectReferenceAbstract = in.ObjectReferenceAbstract - in.Selector.DeepCopyInto(&out.Selector) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectReference. -func (in *ObjectReference) DeepCopy() *ObjectReference { - if in == nil { - return nil - } - out := new(ObjectReference) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ObjectReferenceAbstract) DeepCopyInto(out *ObjectReferenceAbstract) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectReferenceAbstract. -func (in *ObjectReferenceAbstract) DeepCopy() *ObjectReferenceAbstract { - if in == nil { - return nil - } - out := new(ObjectReferenceAbstract) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ObjectReferenceStatus) DeepCopyInto(out *ObjectReferenceStatus) { - *out = *in - out.ObjectReferenceAbstract = in.ObjectReferenceAbstract -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectReferenceStatus. -func (in *ObjectReferenceStatus) DeepCopy() *ObjectReferenceStatus { - if in == nil { - return nil - } - out := new(ObjectReferenceStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Permissions) DeepCopyInto(out *Permissions) { *out = *in @@ -583,22 +852,186 @@ func (in *Permissions) DeepCopy() *Permissions { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in ProcessedItems) DeepCopyInto(out *ProcessedItems) { - { - in := &in - *out = make(ProcessedItems, len(*in)) - copy(*out, *in) - } +func (in *QuantityLedger) DeepCopyInto(out *QuantityLedger) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProcessedItems. -func (in ProcessedItems) DeepCopy() ProcessedItems { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QuantityLedger. +func (in *QuantityLedger) DeepCopy() *QuantityLedger { if in == nil { return nil } - out := new(ProcessedItems) + out := new(QuantityLedger) in.DeepCopyInto(out) - return *out + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *QuantityLedger) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QuantityLedgerList) DeepCopyInto(out *QuantityLedgerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]QuantityLedger, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QuantityLedgerList. +func (in *QuantityLedgerList) DeepCopy() *QuantityLedgerList { + if in == nil { + return nil + } + out := new(QuantityLedgerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *QuantityLedgerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QuantityLedgerObjectRef) DeepCopyInto(out *QuantityLedgerObjectRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QuantityLedgerObjectRef. +func (in *QuantityLedgerObjectRef) DeepCopy() *QuantityLedgerObjectRef { + if in == nil { + return nil + } + out := new(QuantityLedgerObjectRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QuantityLedgerPendingDelete) DeepCopyInto(out *QuantityLedgerPendingDelete) { + *out = *in + out.ObjectRef = in.ObjectRef + in.CreatedAt.DeepCopyInto(&out.CreatedAt) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QuantityLedgerPendingDelete. +func (in *QuantityLedgerPendingDelete) DeepCopy() *QuantityLedgerPendingDelete { + if in == nil { + return nil + } + out := new(QuantityLedgerPendingDelete) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QuantityLedgerReservation) DeepCopyInto(out *QuantityLedgerReservation) { + *out = *in + out.Usage = in.Usage.DeepCopy() + out.ObjectRef = in.ObjectRef + in.CreatedAt.DeepCopyInto(&out.CreatedAt) + in.UpdatedAt.DeepCopyInto(&out.UpdatedAt) + if in.ExpiresAt != nil { + in, out := &in.ExpiresAt, &out.ExpiresAt + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QuantityLedgerReservation. +func (in *QuantityLedgerReservation) DeepCopy() *QuantityLedgerReservation { + if in == nil { + return nil + } + out := new(QuantityLedgerReservation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QuantityLedgerSpec) DeepCopyInto(out *QuantityLedgerSpec) { + *out = *in + out.TargetRef = in.TargetRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QuantityLedgerSpec. +func (in *QuantityLedgerSpec) DeepCopy() *QuantityLedgerSpec { + if in == nil { + return nil + } + out := new(QuantityLedgerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QuantityLedgerStatus) DeepCopyInto(out *QuantityLedgerStatus) { + *out = *in + out.Reserved = in.Reserved.DeepCopy() + if in.Reservations != nil { + in, out := &in.Reservations, &out.Reservations + *out = make([]QuantityLedgerReservation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PendingDeletes != nil { + in, out := &in.PendingDeletes, &out.PendingDeletes + *out = make([]QuantityLedgerPendingDelete, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(meta.ConditionList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.Allocated = in.Allocated.DeepCopy() +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QuantityLedgerStatus. +func (in *QuantityLedgerStatus) DeepCopy() *QuantityLedgerStatus { + if in == nil { + return nil + } + out := new(QuantityLedgerStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QuantityLedgerTargetRef) DeepCopyInto(out *QuantityLedgerTargetRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QuantityLedgerTargetRef. +func (in *QuantityLedgerTargetRef) DeepCopy() *QuantityLedgerTargetRef { + if in == nil { + return nil + } + out := new(QuantityLedgerTargetRef) + in.DeepCopyInto(out) + return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -1084,7 +1517,7 @@ func (in *ResourceSpec) DeepCopyInto(out *ResourceSpec) { } if in.NamespacedItems != nil { in, out := &in.NamespacedItems, &out.NamespacedItems - *out = make([]ObjectReference, len(*in)) + *out = make([]template.ResourceReference, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1101,6 +1534,16 @@ func (in *ResourceSpec) DeepCopyInto(out *ResourceSpec) { *out = new(api.AdditionalMetadataSpec) (*in).DeepCopyInto(*out) } + if in.Generators != nil { + in, out := &in.Generators, &out.Generators + *out = make([]TemplateItemSpec, len(*in)) + copy(*out, *in) + } + if in.Context != nil { + in, out := &in.Context, &out.Context + *out = new(template.TemplateContext) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceSpec. @@ -1118,6 +1561,17 @@ func (in *RuleStatus) DeepCopyInto(out *RuleStatus) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.Spec != nil { + in, out := &in.Spec, &out.Spec + *out = make([]*api.NamespaceRuleBodyNamespace, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(api.NamespaceRuleBodyNamespace) + (*in).DeepCopyInto(*out) + } + } + } in.Status.DeepCopyInto(&out.Status) } @@ -1175,6 +1629,13 @@ func (in *RuleStatusList) DeepCopyObject() runtime.Object { func (in *RuleStatusSpec) DeepCopyInto(out *RuleStatusSpec) { *out = *in in.Rule.DeepCopyInto(&out.Rule) + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(meta.ConditionList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuleStatusSpec. @@ -1187,6 +1648,36 @@ func (in *RuleStatusSpec) DeepCopy() *RuleStatusSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceAccountClient) DeepCopyInto(out *ServiceAccountClient) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceAccountClient. +func (in *ServiceAccountClient) DeepCopy() *ServiceAccountClient { + if in == nil { + return nil + } + out := new(ServiceAccountClient) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemplateItemSpec) DeepCopyInto(out *TemplateItemSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemplateItemSpec. +func (in *TemplateItemSpec) DeepCopy() *TemplateItemSpec { + if in == nil { + return nil + } + out := new(TemplateItemSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Tenant) DeepCopyInto(out *Tenant) { *out = *in @@ -1419,6 +1910,104 @@ func (in *TenantResource) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TenantResourceCommonSpec) DeepCopyInto(out *TenantResourceCommonSpec) { + *out = *in + in.Settings.DeepCopyInto(&out.Settings) + if in.DependsOn != nil { + in, out := &in.DependsOn, &out.DependsOn + *out = make([]meta.LocalRFC1123ObjectReference, len(*in)) + copy(*out, *in) + } + out.ResyncPeriod = in.ResyncPeriod + if in.PruningOnDelete != nil { + in, out := &in.PruningOnDelete, &out.PruningOnDelete + *out = new(bool) + **out = **in + } + if in.Cordoned != nil { + in, out := &in.Cordoned, &out.Cordoned + *out = new(bool) + **out = **in + } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = make([]ResourceSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantResourceCommonSpec. +func (in *TenantResourceCommonSpec) DeepCopy() *TenantResourceCommonSpec { + if in == nil { + return nil + } + out := new(TenantResourceCommonSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TenantResourceCommonSpecSettings) DeepCopyInto(out *TenantResourceCommonSpecSettings) { + *out = *in + if in.Adopt != nil { + in, out := &in.Adopt, &out.Adopt + *out = new(bool) + **out = **in + } + if in.Force != nil { + in, out := &in.Force, &out.Force + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantResourceCommonSpecSettings. +func (in *TenantResourceCommonSpecSettings) DeepCopy() *TenantResourceCommonSpecSettings { + if in == nil { + return nil + } + out := new(TenantResourceCommonSpecSettings) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TenantResourceCommonStatus) DeepCopyInto(out *TenantResourceCommonStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(meta.ConditionList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ProcessedItems != nil { + in, out := &in.ProcessedItems, &out.ProcessedItems + *out = make(meta.ProcessedItems, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ServiceAccount != nil { + in, out := &in.ServiceAccount, &out.ServiceAccount + *out = new(meta.NamespacedRFC1123ObjectReferenceWithNamespace) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantResourceCommonStatus. +func (in *TenantResourceCommonStatus) DeepCopy() *TenantResourceCommonStatus { + if in == nil { + return nil + } + out := new(TenantResourceCommonStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TenantResourceList) DeepCopyInto(out *TenantResourceList) { *out = *in @@ -1454,19 +2043,12 @@ func (in *TenantResourceList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TenantResourceSpec) DeepCopyInto(out *TenantResourceSpec) { *out = *in - out.ResyncPeriod = in.ResyncPeriod - if in.PruningOnDelete != nil { - in, out := &in.PruningOnDelete, &out.PruningOnDelete - *out = new(bool) + in.TenantResourceCommonSpec.DeepCopyInto(&out.TenantResourceCommonSpec) + if in.ServiceAccount != nil { + in, out := &in.ServiceAccount, &out.ServiceAccount + *out = new(meta.LocalRFC1123ObjectReference) **out = **in } - if in.Resources != nil { - in, out := &in.Resources, &out.Resources - *out = make([]ResourceSpec, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantResourceSpec. @@ -1482,11 +2064,7 @@ func (in *TenantResourceSpec) DeepCopy() *TenantResourceSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TenantResourceStatus) DeepCopyInto(out *TenantResourceStatus) { *out = *in - if in.ProcessedItems != nil { - in, out := &in.ProcessedItems, &out.ProcessedItems - *out = make(ProcessedItems, len(*in)) - copy(*out, *in) - } + in.TenantResourceCommonStatus.DeepCopyInto(&out.TenantResourceCommonStatus) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantResourceStatus. @@ -1502,21 +2080,22 @@ func (in *TenantResourceStatus) DeepCopy() *TenantResourceStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TenantSpec) DeepCopyInto(out *TenantSpec) { *out = *in + in.Data.DeepCopyInto(&out.Data) in.Permissions.DeepCopyInto(&out.Permissions) if in.Rules != nil { in, out := &in.Rules, &out.Rules - *out = make([]*NamespaceRule, len(*in)) + *out = make([]*api.NamespaceRuleBodyTenant, len(*in)) for i := range *in { if (*in)[i] != nil { in, out := &(*in)[i], &(*out)[i] - *out = new(NamespaceRule) + *out = new(api.NamespaceRuleBodyTenant) (*in).DeepCopyInto(*out) } } } if in.Owners != nil { in, out := &in.Owners, &out.Owners - *out = make(api.OwnerListSpec, len(*in)) + *out = make(rbac.OwnerListSpec, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1552,7 +2131,7 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) { in.ResourceQuota.DeepCopyInto(&out.ResourceQuota) if in.AdditionalRoleBindings != nil { in, out := &in.AdditionalRoleBindings, &out.AdditionalRoleBindings - *out = make([]api.AdditionalRoleBindingsSpec, len(*in)) + *out = make([]rbac.AdditionalRoleBindingsSpec, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1608,7 +2187,14 @@ func (in *TenantStatus) DeepCopyInto(out *TenantStatus) { in.TenantAvailableStatus.DeepCopyInto(&out.TenantAvailableStatus) if in.Owners != nil { in, out := &in.Owners, &out.Owners - *out = make(api.OwnerStatusListSpec, len(*in)) + *out = make(rbac.OwnerStatusListSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Promotions != nil { + in, out := &in.Promotions, &out.Promotions + *out = make(rbac.PromotionStatusListSpec, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1726,3 +2312,30 @@ func (in *TenantStatusNamespaceMetadata) DeepCopy() *TenantStatusNamespaceMetada in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TenantStatusRuleStatusItem) DeepCopyInto(out *TenantStatusRuleStatusItem) { + *out = *in + if in.Promotions != nil { + in, out := &in.Promotions, &out.Promotions + *out = make(rbac.OwnerStatusListSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.TargetNamespaces != nil { + in, out := &in.TargetNamespaces, &out.TargetNamespaces + *out = make([]meta.RFC1123SubdomainName, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantStatusRuleStatusItem. +func (in *TenantStatusRuleStatusItem) DeepCopy() *TenantStatusRuleStatusItem { + if in == nil { + return nil + } + out := new(TenantStatusRuleStatusItem) + in.DeepCopyInto(out) + return out +} diff --git a/charts/capsule/README.md b/charts/capsule/README.md index 44b38dff..4d1104fe 100644 --- a/charts/capsule/README.md +++ b/charts/capsule/README.md @@ -29,6 +29,8 @@ The following Values have changed key or Value: |-----|------|---------|-------------| | crds.annnotations | object | `{}` | Extra Annotations for CRDs | | crds.createConfig | bool | `false` | Create additionally CapsuleConfiguration even if CRDs are exclusive | +| crds.createDiagnostics | bool | `false` | Create Diagnostic Dashboards even if CRDs are exclusive | +| crds.createRBAC | bool | `false` | Create RBAC and Serviceaccount even if CRDs are exclusive | | crds.exclusive | bool | `false` | Only install the CRDs, no other primitives | | crds.inline | bool | `false` | | | crds.install | bool | `true` | Install the CustomResourceDefinitions (This also manages the lifecycle of the CRDs for update operations) | @@ -68,6 +70,11 @@ The following Values have changed key or Value: | affinity | object | `{}` | Set affinity rules for the Capsule pod | | certManager.additionalSANS | list | `[]` | Specify additional SANS to add to the certificate | | certManager.generateCertificates | bool | `true` | Specifies whether capsule webhooks certificates should be generated using cert-manager | +| conversions.service.caBundle | string | `""` | CABundle for the webhook service | +| conversions.service.name | string | `""` | Custom service name for the webhook service | +| conversions.service.namespace | string | `""` | Custom service namespace for the webhook service | +| conversions.service.port | string | `nil` | Custom service port for the webhook service | +| conversions.service.url | string | `""` | The URL where the capsule webhook services are running (Overwrites cluster scoped service definition) | | customAnnotations | object | `{}` | Additional annotations which will be added to all resources created by Capsule helm chart | | customLabels | object | `{}` | Additional labels which will be added to all resources created by Capsule helm chart | | extraManifests | list | `[]` | Array of additional resources to be created alongside Capsule helm chart | @@ -80,10 +87,8 @@ The following Values have changed key or Value: | ports | list | `[]` | Set additional ports for the deployment | | priorityClassName | string | `""` | Set the priority class name of the Capsule pod | | proxy.enabled | bool | `false` | Enable Installation of Capsule Proxy | -| rbac.resourcepoolclaims.create | bool | `false` | | -| rbac.resourcepoolclaims.labels."rbac.authorization.k8s.io/aggregate-to-admin" | string | `"true"` | | -| rbac.resources.create | bool | `false` | | -| rbac.resources.labels."rbac.authorization.k8s.io/aggregate-to-admin" | string | `"true"` | | +| rbac.resourcepoolclaims | object | `{"create":false,"labels":{"rbac.authorization.k8s.io/aggregate-to-admin":"true"}}` | Allow the creation of ResourcePoolClaims | +| rbac.resources | object | `{"create":false,"labels":{"rbac.authorization.k8s.io/aggregate-to-admin":"true"}}` | Allow the creation of TenantResources | | replicaCount | int | `1` | Set the replica count for capsule pod | | securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"enabled":true,"readOnlyRootFilesystem":true}` | Set the securityContext for the Capsule container | | serviceAccount.annotations | object | `{}` | Annotations to add to the service account. | @@ -116,13 +121,16 @@ The following Values have changed key or Value: | manager.options.administrators | list | `[]` | Define entities which can act as Administrators in the capsule construct These entities are automatically owners for all existing tenants. Meaning they can add namespaces to any tenant. However they must be specific by using the capsule label for interacting with namespaces. Because if that label is not defined, it's assumed that namespace interaction was not targeted towards a tenant and will therefor be ignored by capsule. May also be handy in GitOps scenarios where certain service accounts need to be able to manage namespaces for all tenants. | | manager.options.allowServiceAccountPromotion | bool | `false` | ServiceAccounts within tenant namespaces can be promoted to owners of the given tenant this can be achieved by labeling the serviceaccount and then they are considered owners. This can only be done by other owners of the tenant. However ServiceAccounts which have been promoted to owner can not promote further serviceAccounts. | | manager.options.annotations | object | `{}` | Additional annotations to add to the CapsuleConfiguration resource | -| manager.options.cacheInvalidation | string | `"24h0m0s"` | Duration after which the in-memory cache is invalidated (based on usaage) and re-fetched from the API server | +| manager.options.cacheInvalidation | string | `"0h30m0s"` | Duration after which the in-memory cache is invalidated (based on usaage) and re-fetched from the API server | | manager.options.capsuleConfiguration | string | `"default"` | Change the default name of the capsule configuration name | | manager.options.capsuleUserGroups | list | `[]` | DEPRECATED: use users properties. Names of the users considered as Capsule users. | +| manager.options.clientConnectionBurst | int | `30` | Burst to use for interacting with kubernetes apiserver | +| manager.options.clientConnectionQPS | float | `20` | QPS to use for interacting with kubernetes apiserver | | manager.options.createConfiguration | bool | `true` | Create Configuration | | manager.options.forceTenantPrefix | bool | `false` | Boolean, enforces the Tenant owner, during Namespace creation, to name it using the selected Tenant name as prefix, separated by a dash | | manager.options.generateCertificates | bool | `true` | Specifies whether capsule webhooks certificates should be generated by capsule operator | | manager.options.ignoreUserWithGroups | list | `[]` | Define groups which when found in the request of a user will be ignored by the Capsule this might be useful if you have one group where all the users are in, but you want to separate administrators from normal users with additional groups. | +| manager.options.impersonation | object | `{}` | Impersonation | | manager.options.labels | object | `{}` | Additional labels to add to the CapsuleConfiguration resource | | manager.options.logLevel | string | `"info"` | Set the log verbosity of the capsule with a value from 1 to 5 | | manager.options.nodeMetadata | object | `{"forbiddenAnnotations":{"denied":[],"deniedRegex":""},"forbiddenLabels":{"denied":[],"deniedRegex":""}}` | Allows to set the forbidden metadata for the worker nodes that could be patched by a Tenant | @@ -138,6 +146,7 @@ The following Values have changed key or Value: | manager.rbac.create | bool | `true` | Specifies whether RBAC resources should be created. | | manager.rbac.existingClusterRoles | list | `[]` | Specifies further cluster roles to be added to the Capsule manager service account. | | manager.rbac.existingRoles | list | `[]` | Specifies further cluster roles to be added to the Capsule manager service account. | +| manager.rbac.strict | bool | `false` | Strongly restrict the RBAC assigned to Capsule Controller. When set to true you must aggregate further permissions by yourself. | | manager.readinessProbe | object | `{"httpGet":{"path":"/readyz","port":10080}}` | Configure the readiness probe using Deployment probe spec | | manager.resources | object | `{}` | Set the resource requests/limits for the Capsule manager container | | manager.securityContext | object | `{}` | Set the securityContext for the Capsule container | @@ -151,7 +160,7 @@ The following Values have changed key or Value: |-----|------|---------|-------------| | monitoring.dashboards.annotations | object | `{}` | Annotations for dashboard configmaps | | monitoring.dashboards.enabled | bool | `false` | Enable Dashboards to be deployed | -| monitoring.dashboards.labels | object | `{}` | Labels for dashboard configmaps | +| monitoring.dashboards.labels | object | `{"grafana_dashboard":"1"}` | Labels for dashboard configmaps | | monitoring.dashboards.namespace | string | `""` | Custom namespace for dashboard configmaps | | monitoring.dashboards.operator.allowCrossNamespaceImport | bool | `true` | Allow the Operator to match this resource with Grafanas outside the current namespace | | monitoring.dashboards.operator.enabled | bool | `false` | Enable Operator Resources (GrafanaDashboard) | @@ -160,7 +169,7 @@ The following Values have changed key or Value: | monitoring.dashboards.operator.resyncPeriod | string | `"10m"` | How often the resource is synced, defaults to 10m0s if not set | | monitoring.diagnostics.annotations | object | `{}` | Annotations for dashboard configmaps | | monitoring.diagnostics.enabled | bool | `false` | Enable Diagnostic Dashboards to be deployed | -| monitoring.diagnostics.labels | object | `{}` | Labels for dashboard configmaps | +| monitoring.diagnostics.labels | object | `{"grafana_dashboard":"1"}` | Labels for dashboard configmaps | | monitoring.diagnostics.operator.allowCrossNamespaceImport | bool | `true` | Allow the Operator to match this resource with Grafanas outside the current namespace | | monitoring.diagnostics.operator.enabled | bool | `false` | Enable Operator Resources (GrafanaDashboard) | | monitoring.diagnostics.operator.folder | string | `""` | folder assignment for dashboard | @@ -183,12 +192,21 @@ The following Values have changed key or Value: |-----|------|---------|-------------| | webhooks.annotations | object | `{}` | Additional Annotations for all webhooks | | webhooks.exclusive | bool | `false` | When `crds.exclusive` is `true` the webhooks will be installed | +| webhooks.hooks.calculations | object | `{"enabled":false,"failurePolicy":"Fail","matchConditions":[],"matchPolicy":"Equivalent","namespaceSelector":{},"objectSelector":{},"rules":[]}` | Webhook for Custom Quota Calculations ([Read More](https://projectcapsule.dev/docs/resource-management/customquotas/#admission)) | +| webhooks.hooks.calculations.enabled | bool | `false` | Enable the Hook | +| webhooks.hooks.calculations.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | +| webhooks.hooks.calculations.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.calculations.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.calculations.namespaceSelector | object | `{}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | +| webhooks.hooks.calculations.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.calculations.rules | list | `[]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | | webhooks.hooks.config.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.config.failurePolicy | string | `"Ignore"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.config.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.config.matchPolicy | string | `"Exact"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.config.namespaceSelector | object | `{}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.config.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.config.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.config.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | | webhooks.hooks.cordoning.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.cordoning.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | @@ -196,13 +214,22 @@ The following Values have changed key or Value: | webhooks.hooks.cordoning.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.cordoning.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"},{"key":"projectcapsule.dev/cordoned","operator":"In","values":["true"]}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.cordoning.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.cordoning.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.cordoning.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE","DELETE"],"resources":["*"],"scope":"Namespaced"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | +| webhooks.hooks.customquotas.enabled | bool | `true` | Enable the Hook | +| webhooks.hooks.customquotas.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | +| webhooks.hooks.customquotas.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.customquotas.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.customquotas.namespaceSelector | object | `{}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | +| webhooks.hooks.customquotas.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.customquotas.rules | list | `[{"apiGroups":["capsule.clastix.io"],"apiVersions":["v1beta2"],"operations":["CREATE","UPDATE","DELETE"],"resources":["customquotas"],"scope":"Namespaced"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | | webhooks.hooks.customresources.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.customresources.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.customresources.matchConditions | list | `[{"expression":"!has(request.subResource) || request.subResource == \"\"","name":"ignore-subresources"},{"expression":"request.resource.resource != \"events\"","name":"ignore-events"}]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.customresources.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.customresources.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"},{"key":"projectcapsule.dev/custom-resources","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.customresources.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.customresources.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.defaults.ingress | object | `{}` | Deprecated, use webhooks.hooks.ingresses instead | | webhooks.hooks.defaults.pods | object | `{}` | Deprecated, use webhooks.hooks.pods instead | | webhooks.hooks.defaults.pvc | object | `{}` | Deprecated, use webhooks.hooks.persistentvolumeclaims instead | @@ -212,6 +239,7 @@ The following Values have changed key or Value: | webhooks.hooks.devices.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.devices.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.devices.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.devices.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.devices.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | | webhooks.hooks.gateways.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.gateways.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | @@ -219,20 +247,40 @@ The following Values have changed key or Value: | webhooks.hooks.gateways.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.gateways.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.gateways.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.gateways.opts | object | `{}` | Capsule Hook Options | +| webhooks.hooks.gateways.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | +| webhooks.hooks.globalcustomquotas.enabled | bool | `true` | Enable the Hook | +| webhooks.hooks.globalcustomquotas.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | +| webhooks.hooks.globalcustomquotas.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.globalcustomquotas.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.globalcustomquotas.namespaceSelector | object | `{}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | +| webhooks.hooks.globalcustomquotas.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.globalcustomquotas.rules | list | `[{"apiGroups":["capsule.clastix.io"],"apiVersions":["v1beta2"],"operations":["CREATE","UPDATE","DELETE"],"resources":["globalcustomquotas"],"scope":"Cluster"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | | webhooks.hooks.ingresses.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.ingresses.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.ingresses.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.ingresses.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.ingresses.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.ingresses.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.ingresses.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.ingresses.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | | webhooks.hooks.managed.enabled | bool | `true` | Enable the Hook | -| webhooks.hooks.managed.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | +| webhooks.hooks.managed.failurePolicy | string | `"Ignore"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.managed.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | -| webhooks.hooks.managed.matchPolicy | string | `"Exact"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.managed.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.managed.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | -| webhooks.hooks.managed.objectSelector | object | `{"matchExpressions":[{"key":"projectcapsule.dev/managed-by","operator":"Exists"}]}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | -| webhooks.hooks.managed.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE","DELETE"],"resources":["*"],"scope":"*"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | +| webhooks.hooks.managed.objectSelector | object | `{"matchExpressions":[{"key":"projectcapsule.dev/managed-by","operator":"In","values":["controller"]}]}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.managed.opts | object | `{}` | Capsule Hook Options | +| webhooks.hooks.managed.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE","DELETE"],"resources":["*"],"scope":"Namespaced"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | +| webhooks.hooks.metadata.enabled | bool | `true` | Enable the Hook | +| webhooks.hooks.metadata.failurePolicy | string | `"Ignore"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | +| webhooks.hooks.metadata.matchConditions | list | `[{"expression":"!has(request.subResource) || request.subResource == \"\"","name":"ignore-subresources"},{"expression":"request.resource.resource != \"events\"","name":"ignore-events"}]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.metadata.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.metadata.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | +| webhooks.hooks.metadata.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.metadata.opts | object | `{}` | Capsule Hook Options | +| webhooks.hooks.metadata.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | +| webhooks.hooks.metadata.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*"],"scope":"Namespaced"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | | webhooks.hooks.namespaceOwnerReference | object | `{}` | Deprecated, use webhooks.hooks.namespaces instead | | webhooks.hooks.namespaces.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.namespaces.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | @@ -240,6 +288,7 @@ The following Values have changed key or Value: | webhooks.hooks.namespaces.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.namespaces.namespaceSelector | object | `{}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.namespaces.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.namespaces.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.namespaces.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | | webhooks.hooks.nodes.enabled | bool | `false` | Enable the Hook | | webhooks.hooks.nodes.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | @@ -247,12 +296,14 @@ The following Values have changed key or Value: | webhooks.hooks.nodes.matchPolicy | string | `"Exact"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.nodes.namespaceSelector | object | `{}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.nodes.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.nodes.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.persistentvolumeclaims.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.persistentvolumeclaims.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.persistentvolumeclaims.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.persistentvolumeclaims.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.persistentvolumeclaims.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.persistentvolumeclaims.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.persistentvolumeclaims.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.persistentvolumeclaims.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | | webhooks.hooks.pods.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.pods.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | @@ -260,46 +311,54 @@ The following Values have changed key or Value: | webhooks.hooks.pods.matchPolicy | string | `"Exact"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.pods.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.pods.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.pods.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.pods.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | +| webhooks.hooks.replications.enabled | bool | `true` | Enable the Hook | +| webhooks.hooks.replications.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | +| webhooks.hooks.replications.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.replications.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | +| webhooks.hooks.replications.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | +| webhooks.hooks.replications.objectSelector | object | `{"matchExpressions":[{"key":"projectcapsule.dev/created-by","operator":"In","values":["replications"]}]}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.replications.opts | object | `{}` | Capsule Hook Options | +| webhooks.hooks.replications.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["UPDATE","DELETE"],"resources":["*"],"scope":"*"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | | webhooks.hooks.resourcepools.claims.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.resourcepools.claims.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.resourcepools.claims.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.resourcepools.claims.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.resourcepools.claims.namespaceSelector | object | `{}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.resourcepools.claims.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.resourcepools.claims.opts | object | `{}` | Capsule Hook Options | +| webhooks.hooks.resourcepools.claims.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | | webhooks.hooks.resourcepools.pools.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.resourcepools.pools.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.resourcepools.pools.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.resourcepools.pools.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.resourcepools.pools.namespaceSelector | object | `{}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.resourcepools.pools.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.resourcepools.pools.opts | object | `{}` | Capsule Hook Options | +| webhooks.hooks.resourcepools.pools.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | | webhooks.hooks.serviceaccounts.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.serviceaccounts.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.serviceaccounts.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.serviceaccounts.matchPolicy | string | `"Exact"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.serviceaccounts.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.serviceaccounts.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.serviceaccounts.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.services.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.services.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.services.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.services.matchPolicy | string | `"Exact"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.services.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.services.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | -| webhooks.hooks.tenantLabel.enabled | bool | `true` | Enable the Hook | -| webhooks.hooks.tenantLabel.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | -| webhooks.hooks.tenantLabel.matchConditions | list | `[{"expression":"!has(request.subResource) || request.subResource == \"\"","name":"ignore-subresources"},{"expression":"request.resource.resource != \"events\"","name":"ignore-events"}]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | -| webhooks.hooks.tenantLabel.matchPolicy | string | `"Equivalent"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | -| webhooks.hooks.tenantLabel.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | -| webhooks.hooks.tenantLabel.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | -| webhooks.hooks.tenantLabel.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | -| webhooks.hooks.tenantLabel.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*"],"scope":"Namespaced"}]` | [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) | -| webhooks.hooks.tenantResourceObjects | object | `{}` | Deprecated, use webhooks.hooks.managed instead | +| webhooks.hooks.services.opts | object | `{}` | Capsule Hook Options | +| webhooks.hooks.tenantResourceObjects | object | `{}` | Deprecated, use webhooks.hooks.replications instead | | webhooks.hooks.tenants.enabled | bool | `true` | Enable the Hook | | webhooks.hooks.tenants.failurePolicy | string | `"Fail"` | [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) | | webhooks.hooks.tenants.matchConditions | list | `[]` | [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.tenants.matchPolicy | string | `"Exact"` | [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) | | webhooks.hooks.tenants.namespaceSelector | object | `{}` | [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) | | webhooks.hooks.tenants.objectSelector | object | `{}` | [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) | +| webhooks.hooks.tenants.opts | object | `{}` | Capsule Hook Options | | webhooks.hooks.tenants.reinvocationPolicy | string | `"Never"` | [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) | | webhooks.labels | object | `{}` | Additional Labels for all webhooks | | webhooks.mutatingWebhooksTimeoutSeconds | int | `30` | Timeout in seconds for mutating webhooks | diff --git a/charts/capsule/ci/test-values.yaml b/charts/capsule/ci/test-values.yaml index 1bb39d51..72533911 100644 --- a/charts/capsule/ci/test-values.yaml +++ b/charts/capsule/ci/test-values.yaml @@ -4,6 +4,10 @@ manager: requests: cpu: 200m memory: 128Mi + options: + capsuleUserGroups: ["custom-group-1", "custom-group-2"] + userNames: ["custom-user-1", "custom-user-2"] + rbac: create: true existingClusterRoles: diff --git a/charts/capsule/crds/capsule.clastix.io_capsuleconfigurations.yaml b/charts/capsule/crds/capsule.clastix.io_capsuleconfigurations.yaml index 37f589e6..8d1f7817 100644 --- a/charts/capsule/crds/capsule.clastix.io_capsuleconfigurations.yaml +++ b/charts/capsule/crds/capsule.clastix.io_capsuleconfigurations.yaml @@ -77,7 +77,7 @@ spec: description: Annotations added to the Admission Webhook type: object client: - description: From the upstram struct + description: whats the problem properties: caBundle: description: |- @@ -157,9 +157,405 @@ spec: maxLength: 63 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string + webhooks: + description: Define Dynamic Admission Webhooks + items: + properties: + admissionReviewVersions: + description: |- + AdmissionReviewVersions is an ordered list of preferred `AdmissionReview` + versions the Webhook expects. API server will try to use first version in + the list which it supports. If none of the versions specified in this list + supported by API server, validation will fail for this object. + If a persisted webhook configuration specifies allowed versions and does not + include any versions known to the API Server, calls to the webhook will fail + and be subject to the failure policy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + failurePolicy: + description: |- + FailurePolicy defines how unrecognized errors from the admission endpoint are handled - + allowed values are Ignore or Fail. Defaults to Fail. + type: string + matchConditions: + description: |- + MatchConditions is a list of conditions that must be met for a request to be sent to this + webhook. Match conditions filter requests that have already been matched by the rules, + namespaceSelector, and objectSelector. An empty list of matchConditions matches all requests. + There are a maximum of 64 match conditions allowed. + + The exact matching logic is (in order): + 1. If ANY matchCondition evaluates to FALSE, the webhook is skipped. + 2. If ALL matchConditions evaluate to TRUE, the webhook is called. + 3. If any matchCondition evaluates to an error (but none are FALSE): + - If failurePolicy=Fail, reject the request + - If failurePolicy=Ignore, the error is ignored and the webhook is skipped + items: + description: MatchCondition represents a condition + which must by fulfilled for a request to be sent + to a webhook. + properties: + expression: + description: |- + Expression represents the expression which will be evaluated by CEL. Must evaluate to bool. + CEL expressions have access to the contents of the AdmissionRequest and Authorizer, organized into CEL variables: + + 'object' - The object from the incoming request. The value is null for DELETE requests. + 'oldObject' - The existing object. The value is null for CREATE requests. + 'request' - Attributes of the admission request(/pkg/apis/admission/types.go#AdmissionRequest). + 'authorizer' - A CEL Authorizer. May be used to perform authorization checks for the principal (user or service account) of the request. + See https://pkg.go.dev/k8s.io/apiserver/pkg/cel/library#Authz + 'authorizer.requestResource' - A CEL ResourceCheck constructed from the 'authorizer' and configured with the + request resource. + Documentation on CEL: https://kubernetes.io/docs/reference/using-api/cel/ + + Required. + type: string + name: + description: |- + Name is an identifier for this match condition, used for strategic merging of MatchConditions, + as well as providing an identifier for logging purposes. A good name should be descriptive of + the associated expression. + Name must be a qualified name consisting of alphanumeric characters, '-', '_' or '.', and + must start and end with an alphanumeric character (e.g. 'MyName', or 'my.name', or + '123-abc', regex used for validation is '([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]') with an + optional DNS subdomain prefix and '/' (e.g. 'example.com/MyName') + + Required. + type: string + required: + - expression + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + matchPolicy: + description: |- + matchPolicy defines how the "rules" list is used to match incoming requests. + Allowed values are "Exact" or "Equivalent". + + - Exact: match a request only if it exactly matches a specified rule. + For example, if deployments can be modified via apps/v1, apps/v1beta1, and extensions/v1beta1, + but "rules" only included `apiGroups:["apps"], apiVersions:["v1"], resources: ["deployments"]`, + a request to apps/v1beta1 or extensions/v1beta1 would not be sent to the webhook. + + - Equivalent: match a request if modifies a resource listed in rules, even via another API group or version. + For example, if deployments can be modified via apps/v1, apps/v1beta1, and extensions/v1beta1, + and "rules" only included `apiGroups:["apps"], apiVersions:["v1"], resources: ["deployments"]`, + a request to apps/v1beta1 or extensions/v1beta1 would be converted to apps/v1 and sent to the webhook. + + Defaults to "Equivalent" + type: string + name: + description: |- + The name of the admission webhook. + Name should be fully qualified, e.g., imagepolicy.kubernetes.io, where + "imagepolicy" is the name of the webhook, and kubernetes.io is the name + of the organization. + Required. + type: string + namespaceSelector: + description: |- + NamespaceSelector decides whether to run the webhook on an object based + on whether the namespace for that object matches the selector. If the + object itself is a namespace, the matching is performed on + object.metadata.labels. If the object is another cluster scoped resource, + it never skips the webhook. + + For example, to run the webhook on any objects whose namespace is not + associated with "runlevel" of "0" or "1"; you will set the selector as + follows: + "namespaceSelector": { + "matchExpressions": [ + { + "key": "runlevel", + "operator": "NotIn", + "values": [ + "0", + "1" + ] + } + ] + } + + If instead you want to only run the webhook on any objects whose + namespace is associated with the "environment" of "prod" or "staging"; + you will set the selector as follows: + "namespaceSelector": { + "matchExpressions": [ + { + "key": "environment", + "operator": "In", + "values": [ + "prod", + "staging" + ] + } + ] + } + + See + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + for more examples of label selectors. + + Default to the empty LabelSelector, which matches everything. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + objectSelector: + description: |- + ObjectSelector decides whether to run the webhook based on if the + object has matching labels. objectSelector is evaluated against both + the oldObject and newObject that would be sent to the webhook, and + is considered to match if either object matches the selector. A null + object (oldObject in the case of create, or newObject in the case of + delete) or an object that cannot have labels (like a + DeploymentRollback or a PodProxyOptions object) is not considered to + match. + Use the object selector only if the webhook is opt-in, because end + users may skip the admission webhook by setting the labels. + Default to the empty LabelSelector, which matches everything. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + opts: + description: Capsule Custom Admission Options + properties: + administrators: + default: false + description: |- + If enabled, the request is only sent to admission if the user is mentioned + As Part of the Capsule Administrators + type: boolean + capsuleUsers: + default: false + description: |- + If enabled, the request is only sent to admission if the user is mentioned + As Part of the Capsule Users + type: boolean + required: + - administrators + - capsuleUsers + type: object + path: + description: |- + `path` is the URL path which will be sent in any request to + this service. + type: string + reinvocationPolicy: + description: |- + reinvocationPolicy indicates whether this webhook should be called multiple times as part of a single admission evaluation. + Allowed values are "Never" and "IfNeeded". + + Never: the webhook will not be called more than once in a single admission evaluation. + + IfNeeded: the webhook will be called at least one additional time as part of the admission evaluation + if the object being admitted is modified by other admission plugins after the initial webhook call. + Webhooks that specify this option *must* be idempotent, able to process objects they previously admitted. + Note: + * the number of additional invocations is not guaranteed to be exactly one. + * if additional invocations result in further modifications to the object, webhooks are not guaranteed to be invoked again. + * webhooks that use this option may be reordered to minimize the number of additional invocations. + * to validate an object after all mutations are guaranteed complete, use a validating admission webhook instead. + + Defaults to "Never". + type: string + rules: + description: |- + Rules describes what operations on what resources/subresources the webhook cares about. + The webhook cares about an operation if it matches _any_ Rule. + However, in order to prevent ValidatingAdmissionWebhooks and MutatingAdmissionWebhooks + from putting the cluster in a state which cannot be recovered from without completely + disabling the plugin, ValidatingAdmissionWebhooks and MutatingAdmissionWebhooks are never called + on admission requests for ValidatingWebhookConfiguration and MutatingWebhookConfiguration objects. + items: + description: |- + RuleWithOperations is a tuple of Operations and Resources. It is recommended to make + sure that all the tuple expansions are valid. + properties: + apiGroups: + description: |- + APIGroups is the API groups the resources belong to. '*' is all groups. + If '*' is present, the length of the slice must be one. + Required. + items: + type: string + type: array + x-kubernetes-list-type: atomic + apiVersions: + description: |- + APIVersions is the API versions the resources belong to. '*' is all versions. + If '*' is present, the length of the slice must be one. + Required. + items: + type: string + type: array + x-kubernetes-list-type: atomic + operations: + description: |- + Operations is the operations the admission hook cares about - CREATE, UPDATE, DELETE, CONNECT or * + for all of those operations and any future admission operations that are added. + If '*' is present, the length of the slice must be one. + Required. + items: + description: OperationType specifies an operation + for a request. + type: string + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources is a list of resources this rule applies to. + + For example: + 'pods' means pods. + 'pods/log' means the log subresource of pods. + '*' means all resources, but not subresources. + 'pods/*' means all subresources of pods. + '*/scale' means all scale subresources. + '*/*' means all resources and their subresources. + + If wildcard is present, the validation rule will ensure resources do not + overlap with each other. + + Depending on the enclosing object, subresources might not be allowed. + Required. + items: + type: string + type: array + x-kubernetes-list-type: atomic + scope: + description: |- + scope specifies the scope of this rule. + Valid values are "Cluster", "Namespaced", and "*" + "Cluster" means that only cluster-scoped resources will match this rule. + Namespace API objects are cluster-scoped. + "Namespaced" means that only namespaced resources will match this rule. + "*" means that there are no scope restrictions. + Subresources match the scope of their parent resource. + Default is "*". + type: string + type: object + type: array + x-kubernetes-list-type: atomic + sideEffects: + description: |- + SideEffects states whether this webhook has side effects. + Acceptable values are: None, NoneOnDryRun (webhooks created via v1beta1 may also specify Some or Unknown). + Webhooks with side effects MUST implement a reconciliation system, since a request may be + rejected by a future step in the admission chain and the side effects therefore need to be undone. + Requests with the dryRun attribute will be auto-rejected if they match a webhook with + sideEffects == Unknown or Some. + type: string + timeoutSeconds: + description: |- + TimeoutSeconds specifies the timeout for this webhook. After the timeout passes, + the webhook call will be ignored or the API call will fail based on the + failure policy. + The timeout value must be between 1 and 30 seconds. + Default to 10 seconds. + format: int32 + type: integer + required: + - admissionReviewVersions + - name + - path + - sideEffects + type: object + type: array required: - client type: object + serviceName: + default: capsule-webhook-service + description: Service Name of the Admission Service + type: string validating: description: Configure dynamic Validating Admission for Capsule properties: @@ -169,7 +565,7 @@ spec: description: Annotations added to the Admission Webhook type: object client: - description: From the upstram struct + description: whats the problem properties: caBundle: description: |- @@ -249,6 +645,380 @@ spec: maxLength: 63 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string + webhooks: + description: Define Dynamic Admission Webhooks + items: + properties: + admissionReviewVersions: + description: |- + AdmissionReviewVersions is an ordered list of preferred `AdmissionReview` + versions the Webhook expects. API server will try to use first version in + the list which it supports. If none of the versions specified in this list + supported by API server, validation will fail for this object. + If a persisted webhook configuration specifies allowed versions and does not + include any versions known to the API Server, calls to the webhook will fail + and be subject to the failure policy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + failurePolicy: + description: |- + FailurePolicy defines how unrecognized errors from the admission endpoint are handled - + allowed values are Ignore or Fail. Defaults to Fail. + type: string + matchConditions: + description: |- + MatchConditions is a list of conditions that must be met for a request to be sent to this + webhook. Match conditions filter requests that have already been matched by the rules, + namespaceSelector, and objectSelector. An empty list of matchConditions matches all requests. + There are a maximum of 64 match conditions allowed. + + The exact matching logic is (in order): + 1. If ANY matchCondition evaluates to FALSE, the webhook is skipped. + 2. If ALL matchConditions evaluate to TRUE, the webhook is called. + 3. If any matchCondition evaluates to an error (but none are FALSE): + - If failurePolicy=Fail, reject the request + - If failurePolicy=Ignore, the error is ignored and the webhook is skipped + items: + description: MatchCondition represents a condition + which must by fulfilled for a request to be sent + to a webhook. + properties: + expression: + description: |- + Expression represents the expression which will be evaluated by CEL. Must evaluate to bool. + CEL expressions have access to the contents of the AdmissionRequest and Authorizer, organized into CEL variables: + + 'object' - The object from the incoming request. The value is null for DELETE requests. + 'oldObject' - The existing object. The value is null for CREATE requests. + 'request' - Attributes of the admission request(/pkg/apis/admission/types.go#AdmissionRequest). + 'authorizer' - A CEL Authorizer. May be used to perform authorization checks for the principal (user or service account) of the request. + See https://pkg.go.dev/k8s.io/apiserver/pkg/cel/library#Authz + 'authorizer.requestResource' - A CEL ResourceCheck constructed from the 'authorizer' and configured with the + request resource. + Documentation on CEL: https://kubernetes.io/docs/reference/using-api/cel/ + + Required. + type: string + name: + description: |- + Name is an identifier for this match condition, used for strategic merging of MatchConditions, + as well as providing an identifier for logging purposes. A good name should be descriptive of + the associated expression. + Name must be a qualified name consisting of alphanumeric characters, '-', '_' or '.', and + must start and end with an alphanumeric character (e.g. 'MyName', or 'my.name', or + '123-abc', regex used for validation is '([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]') with an + optional DNS subdomain prefix and '/' (e.g. 'example.com/MyName') + + Required. + type: string + required: + - expression + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + matchPolicy: + description: |- + matchPolicy defines how the "rules" list is used to match incoming requests. + Allowed values are "Exact" or "Equivalent". + + - Exact: match a request only if it exactly matches a specified rule. + For example, if deployments can be modified via apps/v1, apps/v1beta1, and extensions/v1beta1, + but "rules" only included `apiGroups:["apps"], apiVersions:["v1"], resources: ["deployments"]`, + a request to apps/v1beta1 or extensions/v1beta1 would not be sent to the webhook. + + - Equivalent: match a request if modifies a resource listed in rules, even via another API group or version. + For example, if deployments can be modified via apps/v1, apps/v1beta1, and extensions/v1beta1, + and "rules" only included `apiGroups:["apps"], apiVersions:["v1"], resources: ["deployments"]`, + a request to apps/v1beta1 or extensions/v1beta1 would be converted to apps/v1 and sent to the webhook. + + Defaults to "Equivalent" + type: string + name: + description: |- + The name of the admission webhook. + Name should be fully qualified, e.g., imagepolicy.kubernetes.io, where + "imagepolicy" is the name of the webhook, and kubernetes.io is the name + of the organization. + Required. + type: string + namespaceSelector: + description: |- + NamespaceSelector decides whether to run the webhook on an object based + on whether the namespace for that object matches the selector. If the + object itself is a namespace, the matching is performed on + object.metadata.labels. If the object is another cluster scoped resource, + it never skips the webhook. + + For example, to run the webhook on any objects whose namespace is not + associated with "runlevel" of "0" or "1"; you will set the selector as + follows: + "namespaceSelector": { + "matchExpressions": [ + { + "key": "runlevel", + "operator": "NotIn", + "values": [ + "0", + "1" + ] + } + ] + } + + If instead you want to only run the webhook on any objects whose + namespace is associated with the "environment" of "prod" or "staging"; + you will set the selector as follows: + "namespaceSelector": { + "matchExpressions": [ + { + "key": "environment", + "operator": "In", + "values": [ + "prod", + "staging" + ] + } + ] + } + + See + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + for more examples of label selectors. + + Default to the empty LabelSelector, which matches everything. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + objectSelector: + description: |- + ObjectSelector decides whether to run the webhook based on if the + object has matching labels. objectSelector is evaluated against both + the oldObject and newObject that would be sent to the webhook, and + is considered to match if either object matches the selector. A null + object (oldObject in the case of create, or newObject in the case of + delete) or an object that cannot have labels (like a + DeploymentRollback or a PodProxyOptions object) is not considered to + match. + Use the object selector only if the webhook is opt-in, because end + users may skip the admission webhook by setting the labels. + Default to the empty LabelSelector, which matches everything. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + opts: + description: Capsule Custom Admission Options + properties: + administrators: + default: false + description: |- + If enabled, the request is only sent to admission if the user is mentioned + As Part of the Capsule Administrators + type: boolean + capsuleUsers: + default: false + description: |- + If enabled, the request is only sent to admission if the user is mentioned + As Part of the Capsule Users + type: boolean + required: + - administrators + - capsuleUsers + type: object + path: + description: |- + `path` is the URL path which will be sent in any request to + this service. + type: string + rules: + description: |- + Rules describes what operations on what resources/subresources the webhook cares about. + The webhook cares about an operation if it matches _any_ Rule. + However, in order to prevent ValidatingAdmissionWebhooks and MutatingAdmissionWebhooks + from putting the cluster in a state which cannot be recovered from without completely + disabling the plugin, ValidatingAdmissionWebhooks and MutatingAdmissionWebhooks are never called + on admission requests for ValidatingWebhookConfiguration and MutatingWebhookConfiguration objects. + items: + description: |- + RuleWithOperations is a tuple of Operations and Resources. It is recommended to make + sure that all the tuple expansions are valid. + properties: + apiGroups: + description: |- + APIGroups is the API groups the resources belong to. '*' is all groups. + If '*' is present, the length of the slice must be one. + Required. + items: + type: string + type: array + x-kubernetes-list-type: atomic + apiVersions: + description: |- + APIVersions is the API versions the resources belong to. '*' is all versions. + If '*' is present, the length of the slice must be one. + Required. + items: + type: string + type: array + x-kubernetes-list-type: atomic + operations: + description: |- + Operations is the operations the admission hook cares about - CREATE, UPDATE, DELETE, CONNECT or * + for all of those operations and any future admission operations that are added. + If '*' is present, the length of the slice must be one. + Required. + items: + description: OperationType specifies an operation + for a request. + type: string + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources is a list of resources this rule applies to. + + For example: + 'pods' means pods. + 'pods/log' means the log subresource of pods. + '*' means all resources, but not subresources. + 'pods/*' means all subresources of pods. + '*/scale' means all scale subresources. + '*/*' means all resources and their subresources. + + If wildcard is present, the validation rule will ensure resources do not + overlap with each other. + + Depending on the enclosing object, subresources might not be allowed. + Required. + items: + type: string + type: array + x-kubernetes-list-type: atomic + scope: + description: |- + scope specifies the scope of this rule. + Valid values are "Cluster", "Namespaced", and "*" + "Cluster" means that only cluster-scoped resources will match this rule. + Namespace API objects are cluster-scoped. + "Namespaced" means that only namespaced resources will match this rule. + "*" means that there are no scope restrictions. + Subresources match the scope of their parent resource. + Default is "*". + type: string + type: object + type: array + x-kubernetes-list-type: atomic + sideEffects: + description: |- + SideEffects states whether this webhook has side effects. + Acceptable values are: None, NoneOnDryRun (webhooks created via v1beta1 may also specify Some or Unknown). + Webhooks with side effects MUST implement a reconciliation system, since a request may be + rejected by a future step in the admission chain and the side effects therefore need to be undone. + Requests with the dryRun attribute will be auto-rejected if they match a webhook with + sideEffects == Unknown or Some. + type: string + timeoutSeconds: + description: |- + TimeoutSeconds specifies the timeout for this webhook. After the timeout passes, + the webhook call will be ignored or the API call will fail based on the + failure policy. + The timeout value must be between 1 and 30 seconds. + Default to 10 seconds. + format: int32 + type: integer + required: + - admissionReviewVersions + - name + - path + - sideEffects + type: object + type: array required: - client type: object @@ -284,6 +1054,58 @@ spec: items: type: string type: array + impersonation: + description: Service Account Client configuration for impersonation + properties + properties: + caSecretKey: + default: ca.crt + description: Key in the secret that holds the CA certificate (e.g., + "ca.crt") + type: string + caSecretName: + description: Name of the secret containing the CA certificate + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + caSecretNamespace: + description: Namespace where the CA certificate secret is located + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + endpoint: + description: Kubernetes API Endpoint to use for impersonation + type: string + globalDefaultServiceAccount: + description: |- + Default ServiceAccount for global resources (GlobalTenantResource) + When defined, users are required to use this ServiceAccount anywhere in the cluster + unless they explicitly provide their own. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + globalDefaultServiceAccountNamespace: + description: |- + Default ServiceAccount for global resources (GlobalTenantResource) + When defined, users are required to use this ServiceAccount anywhere in the cluster + unless they explicitly provide their own. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + skipTlsVerify: + default: false + description: If true, TLS certificate verification is skipped + (not recommended for production) + type: boolean + tenantDefaultServiceAccount: + description: |- + Default ServiceAccount for namespaced resources (TenantResource) + When defined, users are required to use this ServiceAccount within the namespace + where they deploy the resource, unless they explicitly provide their own. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + type: object nodeMetadata: description: |- Allows to set the forbidden metadata for the worker nodes that could be patched by a Tenant. @@ -329,13 +1151,17 @@ spec: type: string mutatingWebhookConfigurationName: default: capsule-mutating-webhook-configuration - description: Name of the MutatingWebhookConfiguration which contains - the dynamic admission controller paths and resources. + description: |- + Deprecated: use dynamic admission instead + + Name of the MutatingWebhookConfiguration which contains the dynamic admission controller paths and resources. type: string validatingWebhookConfigurationName: default: capsule-validating-webhook-configuration - description: Name of the ValidatingWebhookConfiguration which - contains the dynamic admission controller paths and resources. + description: |- + Deprecated: use dynamic admission instead + + Name of the ValidatingWebhookConfiguration which contains the dynamic admission controller paths and resources. type: string required: - TLSSecretName @@ -424,10 +1250,6 @@ spec: description: CapsuleConfigurationStatus defines the Capsule configuration status. properties: - lastCacheInvalidation: - description: Last time all caches were invalided - format: date-time - type: string users: description: Users which are considered Capsule Users and are bound to the Capsule Tenant construct. diff --git a/charts/capsule/crds/capsule.clastix.io_customquotas.yaml b/charts/capsule/crds/capsule.clastix.io_customquotas.yaml new file mode 100644 index 00000000..39c49003 --- /dev/null +++ b/charts/capsule/crds/capsule.clastix.io_customquotas.yaml @@ -0,0 +1,452 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: customquotas.capsule.clastix.io +spec: + group: capsule.clastix.io + names: + kind: CustomQuota + listKind: CustomQuotaList + plural: customquotas + shortNames: + - cq + singular: customquota + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The total limit available + jsonPath: .spec.limit + name: Limit + type: string + - description: The total used amount + jsonPath: .status.usage.used + name: Used + type: string + - description: The total amount available + jsonPath: .status.usage.available + name: Available + type: string + - description: Reconcile Status + jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - description: Reconcile Message + jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1beta2 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CustomQuotaSpec. + properties: + limit: + anyOf: + - type: integer + - type: string + description: Resource Quantity as limit + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + options: + default: + emitMetricPerClaimUsage: false + description: Additional Options for the CustomQuotaSpecification + properties: + emitMetricPerClaimUsage: + default: false + description: |- + Additionally expose usage metrics for each claim contributing to the quota. + This is disabled by default to avoid high cardinality in the metrics, but can be enabled for more granular monitoring and alerting. + type: boolean + type: object + scopeSelectors: + description: Select items governed by this quota + items: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + sources: + description: Target resource + items: + properties: + apiVersion: + description: API version of the referent. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + op: + default: add + description: Operation used to evaluate usage. + enum: + - add + - sub + - count + type: string + path: + description: |- + Path on GVK where usage is evaluated. + Must be empty when op is "count". + Required and non-empty for all other operations. + type: string + selectors: + description: |- + Provide more granular selectors for these sources + The ScopeSelector and NamespaceSelector are always applied + Allowing these selectors to make further selecting on the resulting subset. + items: + properties: + fieldSelectors: + description: |- + Additional boolean JSONPath expressions. + All must evaluate to true for this selector to match. + items: + type: string + type: array + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + required: + - apiVersion + - kind + type: object + x-kubernetes-validations: + - message: path must be empty when op is 'count'; otherwise path + must be set and non-empty + rule: 'self.op == ''count'' ? !has(self.path) || size(self.path) + == 0 : has(self.path) && size(self.path) > 0' + type: array + required: + - limit + - options + - sources + type: object + status: + description: CustomQuotaStatus defines the observed state of GlobalResourceQuota. + properties: + claims: + description: Objects regarding this policy + items: + properties: + group: + type: string + kind: + type: string + name: + description: Name of the referent. + type: string + namespace: + description: Namespace of the referent, when not specified it + acts as LocalObjectReference. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + uid: + description: UID of the tracked Tenant to pin point tracking + type: string + usage: + anyOf: + - type: integer + - type: string + description: Resource Quantity for given item + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + version: + type: string + required: + - group + - kind + - name + - uid + - usage + - version + type: object + type: array + conditions: + description: Conditions + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + targets: + description: Targeting GVK + items: + properties: + group: + type: string + kind: + type: string + op: + default: add + description: Operation used to evaluate usage. + enum: + - add + - sub + - count + type: string + path: + description: |- + Path on GVK where usage is evaluated. + Must be empty when op is "count". + Required and non-empty for all other operations. + type: string + scope: + description: Path on GVK where usage is evaluated + type: string + selectors: + description: |- + Provide more granular selectors for these sources + The ScopeSelector and NamespaceSelector are always applied + Allowing these selectors to make further selecting on the resulting subset. + items: + properties: + fieldSelectors: + description: |- + Additional boolean JSONPath expressions. + All must evaluate to true for this selector to match. + items: + type: string + type: array + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + version: + type: string + required: + - group + - kind + - version + type: object + type: array + usage: + description: Usage measurements + properties: + available: + anyOf: + - type: integer + - type: string + description: Used is the current observed total available of the + resource (limit - used). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + used: + anyOf: + - type: integer + - type: string + description: Used is the current observed total usage of the resource. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - conditions + - targets + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/capsule/crds/capsule.clastix.io_globalcustomquotas.yaml b/charts/capsule/crds/capsule.clastix.io_globalcustomquotas.yaml new file mode 100644 index 00000000..cfc7d264 --- /dev/null +++ b/charts/capsule/crds/capsule.clastix.io_globalcustomquotas.yaml @@ -0,0 +1,507 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: globalcustomquotas.capsule.clastix.io +spec: + group: capsule.clastix.io + names: + kind: GlobalCustomQuota + listKind: GlobalCustomQuotaList + plural: globalcustomquotas + shortNames: + - gcq + singular: globalcustomquota + scope: Cluster + versions: + - additionalPrinterColumns: + - description: The total limit available + jsonPath: .spec.limit + name: Limit + type: string + - description: The total used amount + jsonPath: .status.usage.used + name: Used + type: string + - description: The total amount available + jsonPath: .status.usage.available + name: Available + type: string + - description: Reconcile Status + jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - description: Reconcile Message + jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + name: v1beta2 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClusterCustomQuotaSpec. + properties: + limit: + anyOf: + - type: integer + - type: string + description: Resource Quantity as limit + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + namespaceSelectors: + description: Select specifc namespaces where this Quota selects items. + items: + description: Selector for resources and their labels or selecting + origin namespaces + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + options: + default: + emitMetricPerClaimUsage: false + description: Additional Options for the CustomQuotaSpecification + properties: + emitMetricPerClaimUsage: + default: false + description: |- + Additionally expose usage metrics for each claim contributing to the quota. + This is disabled by default to avoid high cardinality in the metrics, but can be enabled for more granular monitoring and alerting. + type: boolean + type: object + scopeSelectors: + description: Select items governed by this quota + items: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + sources: + description: Target resource + items: + properties: + apiVersion: + description: API version of the referent. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + op: + default: add + description: Operation used to evaluate usage. + enum: + - add + - sub + - count + type: string + path: + description: |- + Path on GVK where usage is evaluated. + Must be empty when op is "count". + Required and non-empty for all other operations. + type: string + selectors: + description: |- + Provide more granular selectors for these sources + The ScopeSelector and NamespaceSelector are always applied + Allowing these selectors to make further selecting on the resulting subset. + items: + properties: + fieldSelectors: + description: |- + Additional boolean JSONPath expressions. + All must evaluate to true for this selector to match. + items: + type: string + type: array + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + required: + - apiVersion + - kind + type: object + x-kubernetes-validations: + - message: path must be empty when op is 'count'; otherwise path + must be set and non-empty + rule: 'self.op == ''count'' ? !has(self.path) || size(self.path) + == 0 : has(self.path) && size(self.path) > 0' + type: array + required: + - limit + - options + - sources + type: object + status: + description: CustomQuotaStatus defines the observed state of GlobalResourceQuota. + properties: + claims: + description: Objects regarding this policy + items: + properties: + group: + type: string + kind: + type: string + name: + description: Name of the referent. + type: string + namespace: + description: Namespace of the referent, when not specified it + acts as LocalObjectReference. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + uid: + description: UID of the tracked Tenant to pin point tracking + type: string + usage: + anyOf: + - type: integer + - type: string + description: Resource Quantity for given item + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + version: + type: string + required: + - group + - kind + - name + - uid + - usage + - version + type: object + type: array + conditions: + description: Conditions + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + namespaces: + description: Observed Namespaces + items: + type: string + type: array + targets: + description: Targeting GVK + items: + properties: + group: + type: string + kind: + type: string + op: + default: add + description: Operation used to evaluate usage. + enum: + - add + - sub + - count + type: string + path: + description: |- + Path on GVK where usage is evaluated. + Must be empty when op is "count". + Required and non-empty for all other operations. + type: string + scope: + description: Path on GVK where usage is evaluated + type: string + selectors: + description: |- + Provide more granular selectors for these sources + The ScopeSelector and NamespaceSelector are always applied + Allowing these selectors to make further selecting on the resulting subset. + items: + properties: + fieldSelectors: + description: |- + Additional boolean JSONPath expressions. + All must evaluate to true for this selector to match. + items: + type: string + type: array + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + version: + type: string + required: + - group + - kind + - version + type: object + type: array + usage: + description: Usage measurements + properties: + available: + anyOf: + - type: integer + - type: string + description: Used is the current observed total available of the + resource (limit - used). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + used: + anyOf: + - type: integer + - type: string + description: Used is the current observed total usage of the resource. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - conditions + - targets + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/capsule/crds/capsule.clastix.io_globaltenantresources.yaml b/charts/capsule/crds/capsule.clastix.io_globaltenantresources.yaml index 09d34e03..56ff764f 100644 --- a/charts/capsule/crds/capsule.clastix.io_globaltenantresources.yaml +++ b/charts/capsule/crds/capsule.clastix.io_globaltenantresources.yaml @@ -11,10 +11,29 @@ spec: kind: GlobalTenantResource listKind: GlobalTenantResourceList plural: globaltenantresources + shortNames: + - gtr singular: globaltenantresource scope: Cluster versions: - - name: v1beta2 + - additionalPrinterColumns: + - description: The total amount of items being replicated + jsonPath: .status.size + name: Items + type: integer + - description: Reconcile Status for the tenant + jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - description: Reconcile Message for the tenant + jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 schema: openAPIV3Schema: description: GlobalTenantResource allows to propagate resource replications @@ -40,6 +59,30 @@ spec: spec: description: GlobalTenantResourceSpec defines the desired state of GlobalTenantResource. properties: + cordoned: + default: false + description: |- + When cordoning a replication it will no longer execute any applies or deletions (paused). + This is useful for maintenances + type: boolean + dependsOn: + description: |- + DependsOn may contain a meta.NamespacedObjectReference slice + with references to TenantResource resources that must be ready before this + TenantResource can be reconciled. + items: + description: LocalObjectReference contains enough information to + locate the referenced Kubernetes resource object. + properties: + name: + description: Name of the referent. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + type: array pruningOnDelete: default: true description: |- @@ -65,6 +108,115 @@ spec: type: string type: object type: object + context: + description: |- + Provide additional template context, which can be used throughout all + the declared items for the replication + properties: + resources: + items: + properties: + apiVersion: + description: API version of the referent. + type: string + index: + description: Index to mount the resource in the template + context + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the values referent. This is useful + when you traying to get a specific resource + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the values referent. + type: string + optional: + default: true + description: Only relevant if name is set. If an item + is not optional, there will be an error thrown when + it does not exist + type: boolean + selector: + description: Selector which allows to get any amount + of these resources based on labels + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - apiVersion + - kind + type: object + type: array + type: object + generators: + description: Templates for advanced use cases + items: + properties: + missingKey: + default: zero + description: Missing Key Option for templating + enum: + - invalid + - zero + - error + type: string + template: + description: |- + Template contains any amount of yaml which is applied to Kubernetes. + This can be a single resource or multiple resources + type: string + type: object + type: array namespaceSelector: description: |- Defines the Namespace selector to select the Tenant Namespaces on which the resources must be propagated. @@ -117,6 +269,7 @@ spec: description: List of the resources already existing in other Namespaces that must be replicated. items: + description: Reference properties: apiVersion: description: API version of the referent. @@ -126,14 +279,25 @@ spec: Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string - namespace: + name: description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + Name of the values referent. This is useful + when you traying to get a specific resource + maxLength: 253 + minLength: 1 type: string + namespace: + description: Namespace of the values referent. + type: string + optional: + default: true + description: Only relevant if name is set. If an item + is not optional, there will be an error thrown when + it does not exist + type: boolean selector: - description: Label selector used to select the given resources - in the given Namespace. + description: Selector which allows to get any amount of + these resources based on labels properties: matchExpressions: description: matchExpressions is a list of label selector @@ -179,16 +343,14 @@ spec: type: object x-kubernetes-map-type: atomic required: + - apiVersion - kind - - namespace - - selector type: object type: array rawItems: description: List of raw resources that must be replicated. items: type: object - x-kubernetes-embedded-resource: true x-kubernetes-preserve-unknown-fields: true type: array type: object @@ -199,6 +361,53 @@ spec: Define the period of time upon a second reconciliation must be invoked. Keep in mind that any change to the manifests will trigger a new reconciliation. type: string + scope: + default: Namespace + description: |- + Resource Scope, Can either be + - Tenant: Create Resources for each tenant in selected Tenants + - Namespace: Create Resources for each namespace in selected Tenants + enum: + - Namespace + - Tenant + - None + type: string + serviceAccount: + description: |- + Local ServiceAccount which will perform all the actions defined in the TenantResource + You must provide permissions accordingly to that ServiceAccount + properties: + name: + description: Name of the referent. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + namespace: + description: Namespace of the referent. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + - namespace + type: object + settings: + default: {} + description: Provide additional settings + properties: + adopt: + default: false + description: Enabling this allows TenanResources to interact with + objects which were not created by a TenantResource. In this + case on prune no deletion of the entire object is made. + type: boolean + force: + default: false + description: |- + Force indicates that in case of conflicts with server-side apply, the client should acquire ownership of the conflicting field. + You may create collisions with this. + type: boolean + type: object tenantSelector: description: Defines the Tenant selector used target the tenants on which resources must be propagated. @@ -249,37 +458,131 @@ spec: required: - resources - resyncPeriod + - settings type: object status: description: GlobalTenantResourceStatus defines the observed state of GlobalTenantResource. properties: + conditions: + description: Condition of the GlobalTenantResource. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array processedItems: description: List of the replicated resources for the given TenantResource. items: + description: Advanced Status Item for pin pointing items in tenants/namespaces. properties: - apiVersion: - description: API version of the referent. + group: type: string kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string - required: - - kind - - name - - namespace + origin: + type: string + status: + properties: + created: + description: Indicates wether the resource was created or + adopted + type: boolean + lastApply: + description: |- + An opaque value that represents the internal version of this object that can + be used by clients to determine when objects have changed. May be used for optimistic + concurrency, change detection, and the watch operation on a resource or set of resources. + Clients must treat these values as opaque and passed unmodified back to the server. + They may only be valid for a particular resource or set of resources. + + Populated by the system. + Read-only. + Value must be treated as opaque by clients and . + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - status + - type + type: object + tenant: + type: string + version: + type: string type: object type: array selectedTenants: @@ -287,9 +590,28 @@ spec: items: type: string type: array + serviceAccount: + description: Serviceaccount used for impersonation + properties: + name: + description: Name of the referent. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + namespace: + description: Namespace of the referent. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + - namespace + type: object + size: + description: How many items are being replicated by the TenantResource. + type: integer required: - - processedItems - - selectedTenants + - size type: object required: - spec diff --git a/charts/capsule/crds/capsule.clastix.io_quantityledgers.yaml b/charts/capsule/crds/capsule.clastix.io_quantityledgers.yaml new file mode 100644 index 00000000..3d6d763d --- /dev/null +++ b/charts/capsule/crds/capsule.clastix.io_quantityledgers.yaml @@ -0,0 +1,295 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: quantityledgers.capsule.clastix.io +spec: + group: capsule.clastix.io + names: + kind: QuantityLedger + listKind: QuantityLedgerList + plural: quantityledgers + shortNames: + - ql + singular: quantityledger + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.targetRef.kind + name: TargetKind + type: string + - jsonPath: .spec.targetRef.namespace + name: TargetNamespace + type: string + - jsonPath: .spec.targetRef.name + name: TargetName + type: string + - jsonPath: .status.reserved + name: Reserved + type: string + - jsonPath: .status.reservations.size() + name: Reservations + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: QuotaLedgerSpec contains the immutable target reference. + properties: + targetRef: + description: TargetRef points to the quota object that this ledger + belongs to. + properties: + apiGroup: + description: APIGroup of the target quota resource, for example + "capsule.clastix.io". + type: string + kind: + description: Kind of the target quota resource, for example "CustomQuota" + or "GlobalCustomQuota". + minLength: 1 + type: string + name: + description: Name of the target quota resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the target quota resource. + Must be empty for cluster-scoped targets. + type: string + uid: + description: |- + UID of the target quota resource. + Optional, but useful for stale reference detection. + type: string + required: + - kind + - name + type: object + required: + - targetRef + type: object + status: + description: |- + QuantityLedgerStatus contains the mutable coordination state used by admission + and quota controllers. + properties: + allocated: + anyOf: + - type: integer + - type: string + description: |- + Allocated is the admission-owned total that has been accepted by the webhook. + It must be updated only through optimistic concurrency on QuantityLedger. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + conditions: + description: Conditions for the resource claim + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + pendingDeletes: + description: Pending delete hints carried over from admission delete + handling. + items: + description: |- + QuantityLedgerPendingDelete tracks objects that are expected to disappear from claims + soon, but may still temporarily appear during rebuild due to propagation delay. + properties: + createdAt: + format: date-time + type: string + objectRef: + description: |- + QuotaLedgerObjectRef identifies the object for which a reservation exists. + UID may be empty for CREATE admission before the object is persisted. + properties: + apiGroup: + description: APIGroup of the tracked object. + type: string + apiVersion: + description: APIVersion of the tracked object, for example + "v1". + minLength: 1 + type: string + kind: + description: Kind of the tracked object, for example "Pod". + minLength: 1 + type: string + name: + description: Name of the tracked object. + type: string + namespace: + description: Namespace of the tracked object. + type: string + uid: + description: UID of the tracked object. + type: string + required: + - apiVersion + - kind + type: object + required: + - createdAt + - objectRef + type: object + type: array + reservations: + description: Active inflight reservations for this quota. + items: + description: |- + QuantityLedgerReservation represents one active inflight reservation. + ID should be stable for retries of the same admission request. + In practice, admission.Request.UID is a good default. + properties: + createdAt: + description: Time the reservation was first created. + format: date-time + type: string + expiresAt: + description: Time after which the reservation may be considered + stale. + format: date-time + type: string + id: + description: Unique reservation identifier. + minLength: 1 + type: string + objectRef: + description: Object that this reservation is intended to create/update. + properties: + apiGroup: + description: APIGroup of the tracked object. + type: string + apiVersion: + description: APIVersion of the tracked object, for example + "v1". + minLength: 1 + type: string + kind: + description: Kind of the tracked object, for example "Pod". + minLength: 1 + type: string + name: + description: Name of the tracked object. + type: string + namespace: + description: Namespace of the tracked object. + type: string + uid: + description: UID of the tracked object. + type: string + required: + - apiVersion + - kind + type: object + updatedAt: + description: Time the reservation was last refreshed or updated. + format: date-time + type: string + usage: + anyOf: + - type: integer + - type: string + description: Amount reserved for this request. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - createdAt + - id + - objectRef + - updatedAt + - usage + type: object + type: array + reserved: + anyOf: + - type: integer + - type: string + description: |- + Reserved is the aggregate sum of all active reservations. + Controllers/webhooks should treat this as derived data from Reservations. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/capsule/crds/capsule.clastix.io_rulestatuses.yaml b/charts/capsule/crds/capsule.clastix.io_rulestatuses.yaml index 772d9f4e..24fb073a 100644 --- a/charts/capsule/crds/capsule.clastix.io_rulestatuses.yaml +++ b/charts/capsule/crds/capsule.clastix.io_rulestatuses.yaml @@ -40,16 +40,117 @@ spec: type: string metadata: type: object + spec: + items: + description: For future inmplementatiosn where users might manage RuleStatus + CRs tehmselves + properties: + enforce: + description: Enforcement for given rule + properties: + registries: + description: |- + Define registries which are allowed to be used within this tenant + The rules are aggregated, since you can use Regular Expressions the match registry endpoints + items: + properties: + policy: + description: Allowed PullPolicy for the given registry. + Supplying no value allows all policies. + items: + description: PullPolicy describes a policy for if/when + to pull a container image + type: string + type: array + url: + description: OCI Registry endpoint, is treated as regular + expression. + type: string + validation: + default: + - pod/images + - pod/volumes + description: Requesting Resources + items: + enum: + - pod/images + - pod/volumes + type: string + type: array + required: + - url + type: object + type: array + type: object + type: object + type: array status: description: RuleStatus contains the accumulated rules applying to namespace it's deployed in. properties: + conditions: + description: Conditions + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array rule: description: Managed Enforcement properties per Namespace (aggregated from rules) properties: enforce: - description: Enforcement Rules applied + description: Enforcement for given rule properties: registries: description: |- @@ -86,6 +187,8 @@ spec: type: array type: object type: object + required: + - conditions type: object type: object served: true diff --git a/charts/capsule/crds/capsule.clastix.io_tenantowners.yaml b/charts/capsule/crds/capsule.clastix.io_tenantowners.yaml index 1e18b892..97447ec6 100644 --- a/charts/capsule/crds/capsule.clastix.io_tenantowners.yaml +++ b/charts/capsule/crds/capsule.clastix.io_tenantowners.yaml @@ -11,6 +11,8 @@ spec: kind: TenantOwner listKind: TenantOwnerList plural: tenantowners + shortNames: + - to singular: tenantowner scope: Cluster versions: diff --git a/charts/capsule/crds/capsule.clastix.io_tenantresources.yaml b/charts/capsule/crds/capsule.clastix.io_tenantresources.yaml index 2f706d3e..188f9551 100644 --- a/charts/capsule/crds/capsule.clastix.io_tenantresources.yaml +++ b/charts/capsule/crds/capsule.clastix.io_tenantresources.yaml @@ -14,7 +14,24 @@ spec: singular: tenantresource scope: Namespaced versions: - - name: v1beta2 + - additionalPrinterColumns: + - description: The total amount of items being replicated + jsonPath: .status.size + name: Items + type: integer + - description: Reconcile Status for the tenant + jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - description: Reconcile Message for the tenant + jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 schema: openAPIV3Schema: description: |- @@ -42,6 +59,30 @@ spec: spec: description: TenantResourceSpec defines the desired state of TenantResource. properties: + cordoned: + default: false + description: |- + When cordoning a replication it will no longer execute any applies or deletions (paused). + This is useful for maintenances + type: boolean + dependsOn: + description: |- + DependsOn may contain a meta.NamespacedObjectReference slice + with references to TenantResource resources that must be ready before this + TenantResource can be reconciled. + items: + description: LocalObjectReference contains enough information to + locate the referenced Kubernetes resource object. + properties: + name: + description: Name of the referent. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + type: array pruningOnDelete: default: true description: |- @@ -67,6 +108,115 @@ spec: type: string type: object type: object + context: + description: |- + Provide additional template context, which can be used throughout all + the declared items for the replication + properties: + resources: + items: + properties: + apiVersion: + description: API version of the referent. + type: string + index: + description: Index to mount the resource in the template + context + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the values referent. This is useful + when you traying to get a specific resource + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace of the values referent. + type: string + optional: + default: true + description: Only relevant if name is set. If an item + is not optional, there will be an error thrown when + it does not exist + type: boolean + selector: + description: Selector which allows to get any amount + of these resources based on labels + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - apiVersion + - kind + type: object + type: array + type: object + generators: + description: Templates for advanced use cases + items: + properties: + missingKey: + default: zero + description: Missing Key Option for templating + enum: + - invalid + - zero + - error + type: string + template: + description: |- + Template contains any amount of yaml which is applied to Kubernetes. + This can be a single resource or multiple resources + type: string + type: object + type: array namespaceSelector: description: |- Defines the Namespace selector to select the Tenant Namespaces on which the resources must be propagated. @@ -119,6 +269,7 @@ spec: description: List of the resources already existing in other Namespaces that must be replicated. items: + description: Reference properties: apiVersion: description: API version of the referent. @@ -128,14 +279,25 @@ spec: Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string - namespace: + name: description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + Name of the values referent. This is useful + when you traying to get a specific resource + maxLength: 253 + minLength: 1 type: string + namespace: + description: Namespace of the values referent. + type: string + optional: + default: true + description: Only relevant if name is set. If an item + is not optional, there will be an error thrown when + it does not exist + type: boolean selector: - description: Label selector used to select the given resources - in the given Namespace. + description: Selector which allows to get any amount of + these resources based on labels properties: matchExpressions: description: matchExpressions is a list of label selector @@ -181,16 +343,14 @@ spec: type: object x-kubernetes-map-type: atomic required: + - apiVersion - kind - - namespace - - selector type: object type: array rawItems: description: List of raw resources that must be replicated. items: type: object - x-kubernetes-embedded-resource: true x-kubernetes-preserve-unknown-fields: true type: array type: object @@ -201,43 +361,187 @@ spec: Define the period of time upon a second reconciliation must be invoked. Keep in mind that any change to the manifests will trigger a new reconciliation. type: string + serviceAccount: + description: |- + Local ServiceAccount which will perform all the actions defined in the TenantResource + You must provide permissions accordingly to that ServiceAccount + properties: + name: + description: Name of the referent. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + settings: + default: {} + description: Provide additional settings + properties: + adopt: + default: false + description: Enabling this allows TenanResources to interact with + objects which were not created by a TenantResource. In this + case on prune no deletion of the entire object is made. + type: boolean + force: + default: false + description: |- + Force indicates that in case of conflicts with server-side apply, the client should acquire ownership of the conflicting field. + You may create collisions with this. + type: boolean + type: object required: - resources - resyncPeriod + - settings type: object status: description: TenantResourceStatus defines the observed state of TenantResource. properties: + conditions: + description: Condition of the GlobalTenantResource. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array processedItems: description: List of the replicated resources for the given TenantResource. items: + description: Advanced Status Item for pin pointing items in tenants/namespaces. properties: - apiVersion: - description: API version of the referent. + group: type: string kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string - required: - - kind - - name - - namespace + origin: + type: string + status: + properties: + created: + description: Indicates wether the resource was created or + adopted + type: boolean + lastApply: + description: |- + An opaque value that represents the internal version of this object that can + be used by clients to determine when objects have changed. May be used for optimistic + concurrency, change detection, and the watch operation on a resource or set of resources. + Clients must treat these values as opaque and passed unmodified back to the server. + They may only be valid for a particular resource or set of resources. + + Populated by the system. + Read-only. + Value must be treated as opaque by clients and . + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - status + - type + type: object + tenant: + type: string + version: + type: string type: object type: array + serviceAccount: + description: Serviceaccount used for impersonation + properties: + name: + description: Name of the referent. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + namespace: + description: Namespace of the referent. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + - namespace + type: object + size: + description: How many items are being replicated by the TenantResource. + type: integer required: - - processedItems + - size type: object required: - spec diff --git a/charts/capsule/crds/capsule.clastix.io_tenants.yaml b/charts/capsule/crds/capsule.clastix.io_tenants.yaml index b54d64cb..62fc6038 100644 --- a/charts/capsule/crds/capsule.clastix.io_tenants.yaml +++ b/charts/capsule/crds/capsule.clastix.io_tenants.yaml @@ -1214,6 +1214,11 @@ spec: description: Toggling the Tenant resources cordoning, when enable resources cannot be deleted. type: boolean + data: + description: |- + Specify additional data relating to the tenant. + Mainly useable in templating and more accessible than labels/annotations. + x-kubernetes-preserve-unknown-fields: true deviceClasses: description: Specifies options for the DeviceClass resources. properties: @@ -2249,6 +2254,11 @@ spec: permissions: description: Specify Permissions for the Tenant. properties: + allowOwnerPromotion: + default: true + description: ClusterRoles granted to the promoted ServiceAccounts + across the Tenant + type: boolean matchOwners: description: |- Matches TenantOwner objects which are promoted to owners of this tenant @@ -2488,9 +2498,10 @@ spec: Read More: https://projectcapsule.dev/docs/tenants/rules/ items: + description: Rules Distributed via Tenants properties: enforce: - description: Enforcement Rules applied + description: Enforcement for given rule properties: registries: description: |- @@ -2527,7 +2538,8 @@ spec: type: array type: object namespaceSelector: - description: Select namespaces which are going to usese + description: Select namespaces which are going to be targeted + with this rule properties: matchExpressions: description: matchExpressions is a list of label selector @@ -2572,6 +2584,74 @@ spec: type: object type: object x-kubernetes-map-type: atomic + permissions: + description: Permissions for given rule + properties: + rules: + description: |- + Define Promotion Rules which distributed additional ClusterRoles across the Tenant + for promoted ServiceAccounts. + items: + properties: + clusterRoles: + description: |- + ClusterRoles granted to the promoted ServiceAccounts across the Tenant + kubebuilder:validation:Minimum=1 + items: + type: string + type: array + selector: + description: |- + Match ServiceAccounts which are promoted which are granted these additional ClusterRoles + across the Tenant + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: array + type: object type: object type: array runtimeClasses: @@ -2898,6 +2978,41 @@ spec: - name type: object type: array + promotions: + description: Promoted ServiceAccounts across the Tenant + items: + properties: + clusterRoles: + default: + - admin + - capsule-namespace-deleter + description: Defines additional cluster-roles for the specific + Owner. + items: + type: string + type: array + kind: + description: Kind of entity. Possible values are "User", "Group", + and "ServiceAccount" + enum: + - User + - Group + - ServiceAccount + type: string + name: + description: Name of the entity. + type: string + targets: + description: Defines additional cluster-roles for the specific + Owner. + items: + type: string + type: array + required: + - kind + - name + type: object + type: array size: description: How many namespaces are assigned to the Tenant. type: integer @@ -3027,10 +3142,11 @@ spec: state: default: Active description: The operational state of the Tenant. Possible values - are "Active", "Cordoned". + are "Active", "Cordoned" or "Terminating". enum: - Cordoned - Active + - Terminating type: string required: - conditions diff --git a/charts/capsule/templates/NOTES.txt b/charts/capsule/templates/NOTES.txt index b59d4b10..8ab45033 100644 --- a/charts/capsule/templates/NOTES.txt +++ b/charts/capsule/templates/NOTES.txt @@ -1,19 +1,26 @@ -- Capsule Operator Helm Chart deployed: +✅ Capsule Operator Helm Chart deployed: # Check the capsule logs - $ kubectl logs -f deployment/{{ template "capsule.fullname" . }}-controller-manager -c manager -n {{ .Release.Namespace }} - + kubectl logs -f deployment/{{ template "capsule.fullname" . }}-controller-manager -c manager -n {{ .Release.Namespace }} # Check the capsule logs - $ kubectl logs -f deployment/{{ template "capsule.fullname" . }}-controller-manager -c manager -n {{ .Release.Namespace }} + kubectl logs -f deployment/{{ template "capsule.fullname" . }}-controller-manager -c manager -n {{ .Release.Namespace }} -- Manage this chart: +👉 Consult resources on how to enhance the installation of Project Capsule based on your needs: + + 📄 Production Setup: https://projectcapsule.dev/docs/operating/setup/installation/#production + + 📄 Admission Policies: https://projectcapsule.dev/docs/operating/admission-policies/ + + 📄 Capsule Proxy: https://projectcapsule.dev/docs/proxy/ + +⏳ Manage this chart: # Upgrade Capsule - $ helm upgrade {{ .Release.Name }} -f capsule -n {{ .Release.Namespace }} + helm upgrade {{ .Release.Name }} -f capsule -n {{ .Release.Namespace }} # Show this status again - $ helm status {{ .Release.Name }} -n {{ .Release.Namespace }} + helm status {{ .Release.Name }} -n {{ .Release.Namespace }} # Uninstall Capsule - $ helm uninstall {{ .Release.Name }} -n {{ .Release.Namespace }} + helm uninstall {{ .Release.Name }} -n {{ .Release.Namespace }} diff --git a/charts/capsule/templates/_helpers.tpl b/charts/capsule/templates/_helpers.tpl index 784e9010..2cb5f416 100644 --- a/charts/capsule/templates/_helpers.tpl +++ b/charts/capsule/templates/_helpers.tpl @@ -31,11 +31,10 @@ Create chart name and version as used by the chart label. {{- end }} {{/* -Common labels +Base labels */}} -{{- define "capsule.labels" -}} +{{- define "capsule.baselabels" -}} helm.sh/chart: {{ include "capsule.chart" . }} -{{ include "capsule.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,11 +44,23 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end }} {{- end }} +{{/* +Common labels +*/}} +{{- define "capsule.labels" -}} +{{ include "capsule.baselabels" . }} +{{ include "capsule.selectorLabels" . }} +{{- end }} + {{/* Selector labels */}} {{- define "capsule.selectorLabels" -}} app.kubernetes.io/name: {{ include "capsule.name" . }} +{{ include "capsule.selectorLabelInstance" . }} +{{- end }} + +{{- define "capsule.selectorLabelInstance" -}} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} @@ -150,7 +161,7 @@ url: {{ printf "%s/%s" (trimSuffix "/" $.ctx.Values.webhooks.service.url ) (trim service: name: {{ default (printf "%s-webhook-service" (include "capsule.fullname" $.ctx)) $.ctx.Values.webhooks.service.name }} namespace: {{ default $.ctx.Release.Namespace $.ctx.Values.webhooks.service.namespace }} - port: {{ default 443 $.ctx.Values.webhooks.service.port }} + port: {{ default 9443 $.ctx.Values.webhooks.service.port }} path: {{ required "Path is required for the function" $.path }} {{- end }} {{- end }} @@ -158,7 +169,6 @@ service: {{/* Capsule Webhook service (Without Path) - */}} {{- define "capsule.webhooks.serviceConfig" -}} {{- include "capsule.webhooks.cabundle" $ | nindent 0 }} @@ -168,11 +178,10 @@ url: {{ trimSuffix "/" $.Values.webhooks.service.url }} service: name: {{ default (printf "%s-webhook-service" (include "capsule.fullname" $)) $.Values.webhooks.service.name }} namespace: {{ default $.Release.Namespace $.Values.webhooks.service.namespace }} - port: {{ default 443 $.Values.webhooks.service.port }} + port: {{ default 9443 $.Values.webhooks.service.port }} {{- end }} {{- end }} - {{/* Capsule Webhook endpoint CA Bundle */}} diff --git a/charts/capsule/templates/_pod.tpl b/charts/capsule/templates/_pod.tpl index c7a241a4..efcd6d75 100644 --- a/charts/capsule/templates/_pod.tpl +++ b/charts/capsule/templates/_pod.tpl @@ -65,6 +65,8 @@ spec: - --zap-log-level={{ default 4 .Values.manager.options.logLevel }} - --configuration-name={{ .Values.manager.options.capsuleConfiguration }} - --workers={{ .Values.manager.options.workers }} + - --client-connection-qps={{ .Values.manager.options.clientConnectionQPS }} + - --client-connection-burst={{ .Values.manager.options.clientConnectionBurst }} {{- with .Values.manager.extraArgs }} {{- toYaml . | nindent 8 }} {{- end }} @@ -84,9 +86,9 @@ spec: {{- end }} ports: {{- if not (.Values.manager.hostNetwork) }} - - name: webhook-server - containerPort: {{ .Values.manager.webhookPort }} + - name: admission protocol: TCP + containerPort: 9443 - name: metrics containerPort: 8080 protocol: TCP diff --git a/charts/capsule/templates/configuration.yaml b/charts/capsule/templates/configuration.yaml index a38c6693..72d9a38a 100644 --- a/charts/capsule/templates/configuration.yaml +++ b/charts/capsule/templates/configuration.yaml @@ -1,4 +1,5 @@ -{{- if $.Values.manager.options.createConfiguration }} +{{- if or (and $.Values.crds.exclusive $.Values.crds.createConfig) (not $.Values.crds.exclusive) }} + {{- if $.Values.manager.options.createConfiguration }} apiVersion: capsule.clastix.io/v1beta2 kind: CapsuleConfiguration metadata: @@ -17,35 +18,22 @@ spec: cacheInvalidation: {{ .Values.manager.options.cacheInvalidation }} rbac: {{- toYaml .Values.manager.options.rbac | nindent 4 }} - admission: - validating: - name: "{{ include "capsule.fullname" . }}-dynamic" - client: - {{- include "capsule.webhooks.serviceConfig" $ | nindent 8 }} - {{- if (include "admission.labels" $) }} - labels: - {{- include "admission.labels" $ | nindent 8 }} - {{- end }} - {{- if (include "admission.annotations" $) }} - annotations: - {{- include "admission.annotations" $ | nindent 8 }} - {{- end }} - mutating: - name: "{{ include "capsule.fullname" . }}-dynamic" - client: - {{- include "capsule.webhooks.serviceConfig" $ | nindent 8 }} - {{- if (include "admission.labels" $) }} - labels: - {{- include "admission.labels" $ | nindent 8 }} - {{- end }} - {{- if (include "admission.annotations" $) }} - annotations: - {{- include "admission.annotations" $ | nindent 8 }} - {{- end }} + {{- with .Values.manager.options.impersonation }} + impersonation: + {{- toYaml . | nindent 4 }} + {{- end }} administrators: {{- toYaml .Values.manager.options.administrators | nindent 4 }} users: {{- toYaml .Values.manager.options.users | nindent 4 }} + {{- range $_, $subject := .Values.manager.options.userNames }} + - kind: "User" + name: {{ $subject | quote }} + {{- end }} + {{- range $_, $subject := .Values.manager.options.capsuleUserGroups }} + - kind: "Group" + name: {{ $subject | quote }} + {{- end }} enableTLSReconciler: {{ .Values.tls.enableController }} overrides: mutatingWebhookConfigurationName: {{ include "capsule.fullname" . }}-mutating-webhook-configuration @@ -53,10 +41,6 @@ spec: validatingWebhookConfigurationName: {{ include "capsule.fullname" . }}-validating-webhook-configuration forceTenantPrefix: {{ .Values.manager.options.forceTenantPrefix }} allowServiceAccountPromotion: {{ .Values.manager.options.allowServiceAccountPromotion }} - userGroups: - {{- toYaml .Values.manager.options.capsuleUserGroups | nindent 4 }} - userNames: - {{- toYaml .Values.manager.options.userNames | nindent 4 }} ignoreUserWithGroups: {{- toYaml .Values.manager.options.ignoreUserWithGroups | nindent 4 }} protectedNamespaceRegex: {{ .Values.manager.options.protectedNamespaceRegex | quote }} @@ -64,4 +48,1224 @@ spec: nodeMetadata: {{- toYaml . | nindent 4 }} {{- end }} + admission: + serviceName: {{ include "capsule.fullname" . }}-webhook-service + validating: + name: "{{ include "capsule.fullname" . }}-dynamic-webhook" + client: + {{- include "capsule.webhooks.serviceConfig" $ | nindent 8 }} + {{- if (include "admission.labels" $) }} + labels: + {{- include "admission.labels" $ | nindent 8 }} + {{- end }} + {{- if (include "admission.annotations" $) }} + annotations: + {{- include "admission.annotations" $ | nindent 8 }} + {{- end }} + webhooks: + {{- $any := false -}} + {{- with .Values.webhooks.hooks.namespaces }} + {{- if .enabled }} + {{- $any = true }} + - name: namespaces.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/namespaces/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - namespaces + - namespaces/status + - namespace/finalize + scope: '*' + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.customresources }} + {{- if .enabled }} + {{- $any = true }} + - name: customresources.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/generic/customresources" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - '*' + apiVersions: + - '*' + operations: + - CREATE + - UPDATE + - DELETE + resources: + - '*' + scope: Namespaced + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.cordoning }} + {{- if .enabled }} + {{- $any = true }} + - name: cordoning.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/generic/cordoning" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .rules }} + rules: + {{- toYaml . | nindent 10 }} + {{- end }} + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.devices }} + {{- if .enabled }} + {{- $any = true }} + - name: devices.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/devices/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - resource.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - resourceclaimtemplates + - resourceclaims + scope: Namespaced + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.gateways }} + {{- if .enabled }} + {{- $any = true }} + - name: gateway.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/gateways/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - gateway.networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - gateways + scope: Namespaced + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.ingresses }} + {{- if .enabled }} + {{- $any = true }} + - name: ingress.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/ingresses/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - networking.k8s.io + - extensions + apiVersions: + - v1 + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + scope: Namespaced + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.nodes }} + {{- if .enabled }} + {{- $any = true }} + - name: nodes.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/nodes/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - UPDATE + resources: + - nodes + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.pods }} + {{- if .enabled }} + {{- $any = true }} + - name: pods.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/pods/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pods + - pods/ephemeralcontainers + scope: Namespaced + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.persistentvolumeclaims }} + {{- if .enabled }} + {{- $any = true }} + - name: pvc.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/persistentvolumeclaims/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - persistentvolumeclaims + scope: Namespaced + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.services }} + {{- if .enabled }} + {{- $any = true }} + - name: services.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/services/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - services + scope: Namespaced + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with (mergeOverwrite .Values.webhooks.hooks.managed .Values.webhooks.hooks.tenantResourceObjects) }} + {{- if .enabled }} + {{- $any = true }} + - name: managed.tenant.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + path: "/generic/managed" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + {{- toYaml .rules | nindent 10 }} + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.replications }} + {{- if .enabled }} + {{- $any = true }} + - name: replications.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + path: "/generic/replications" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + {{- toYaml .rules | nindent 10 }} + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.tenants }} + {{- if .enabled }} + {{- $any = true }} + - name: tenants.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/tenants/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - capsule.clastix.io + apiVersions: + - v1beta2 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - tenants + scope: 'Cluster' + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.resourcepools.pools }} + {{- if .enabled }} + {{- $any = true }} + - name: resourcepools.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/resourcepools/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - "capsule.clastix.io" + apiVersions: + - "*" + operations: + - CREATE + - UPDATE + resources: + - resourcepools + scope: '*' + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.resourcepools.pools }} + {{- if .enabled }} + {{- $any = true }} + - name: resourcepoolclaims.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/resourcepools/claim/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - "capsule.clastix.io" + apiVersions: + - "*" + operations: + - CREATE + - UPDATE + - DELETE + resources: + - resourcepoolclaims + scope: '*' + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.serviceaccounts }} + {{- if .enabled }} + {{- $any = true }} + - name: serviceaccounts.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/serviceaccounts/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - '*' + apiVersions: + - '*' + operations: + - CREATE + - UPDATE + resources: + - 'serviceaccounts' + scope: Namespaced + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.config }} + {{- if .enabled }} + {{- $any = true }} + - name: config.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/config/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - capsule.clastix.io + apiVersions: + - v1beta2 + operations: + - UPDATE + resources: + - capsuleconfigurations + scope: 'Cluster' + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.customquotas }} + {{- if .enabled }} + {{- $any = true }} + - name: namespaced.custom-quotas.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/custom-quotas/namespaced/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + {{- toYaml .rules | nindent 10 }} + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.globalcustomquotas }} + {{- if .enabled }} + {{- $any = true }} + - name: cluster.custom-quotas.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/custom-quotas/cluster/validating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + {{- toYaml .rules | nindent 10 }} + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.calculations }} + {{- if .enabled }} + {{- $any = true }} + - name: calculation.custom-quotas.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/custom-quotas/calculations" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + {{- toYaml .rules | nindent 10 }} + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + + {{- if not $any }} + [] + {{- end }} + mutating: + name: "{{ include "capsule.fullname" . }}-dynamic-webhook" + client: + {{- include "capsule.webhooks.serviceConfig" $ | nindent 8 }} + {{- if (include "admission.labels" $) }} + labels: + {{- include "admission.labels" $ | nindent 8 }} + {{- end }} + {{- if (include "admission.annotations" $) }} + annotations: + {{- include "admission.annotations" $ | nindent 8 }} + {{- end }} + webhooks: + {{- $any := false -}} + {{- with (mergeOverwrite .Values.webhooks.hooks.namespaces .Values.webhooks.hooks.namespaceOwnerReference) }} + {{- if .enabled }} + {{- $any = true }} + - name: namespaces.tenants.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/namespaces/mutating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + reinvocationPolicy: {{ .reinvocationPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - namespaces + scope: '*' + sideEffects: NoneOnDryRun + timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + + {{- with .Values.webhooks.hooks.persistentvolumeclaims }} + {{- if .enabled }} + {{- $any = true }} + - name: pvc.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/persistentvolumeclaims/mutating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + reinvocationPolicy: {{ .reinvocationPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - persistentvolumeclaims + scope: Namespaced + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + + {{- with (mergeOverwrite .Values.webhooks.hooks.pods .Values.webhooks.hooks.defaults.pods) }} + {{- if .enabled }} + {{- $any = true }} + - name: pod.defaults.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + path: "/defaults" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + reinvocationPolicy: {{ .reinvocationPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + scope: "Namespaced" + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with (mergeOverwrite .Values.webhooks.hooks.persistentvolumeclaims .Values.webhooks.hooks.defaults.pvc) }} + {{- if .enabled }} + {{- $any = true }} + - name: storage.defaults.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + path: "/defaults" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + reinvocationPolicy: {{ .reinvocationPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - persistentvolumeclaims + scope: "Namespaced" + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with (mergeOverwrite .Values.webhooks.hooks.ingresses .Values.webhooks.hooks.defaults.ingress) }} + {{- if .enabled }} + {{- $any = true }} + - name: ingress.defaults.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + path: "/defaults" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + reinvocationPolicy: {{ .reinvocationPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1beta1 + - v1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + scope: "Namespaced" + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.gateways }} + {{- if .enabled }} + {{- $any = true }} + - name: gateway.defaults.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + path: "/defaults" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + reinvocationPolicy: {{ .reinvocationPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - gateway.networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - gateways + scope: "Namespaced" + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.resourcepools.pools }} + {{- if .enabled }} + {{- $any = true }} + - name: resourcepools.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/resourcepools/mutating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + reinvocationPolicy: {{ .reinvocationPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - "capsule.clastix.io" + apiVersions: + - "*" + operations: + - CREATE + - UPDATE + resources: + - resourcepools + scope: '*' + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.resourcepools.claims }} + {{- if .enabled }} + {{- $any = true }} + - name: resourcepoolclaims.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/resourcepools/claim/mutating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + reinvocationPolicy: {{ .reinvocationPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + - apiGroups: + - "capsule.clastix.io" + apiVersions: + - "*" + operations: + - CREATE + - UPDATE + resources: + - resourcepoolclaims + scope: '*' + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.tenants }} + {{- if .enabled }} + {{- $any = true }} + - name: tenants.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/tenants/mutating" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + reinvocationPolicy: {{ .reinvocationPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + - apiGroups: + - capsule.clastix.io + apiVersions: + - "*" + operations: + - CREATE + - UPDATE + - DELETE + resources: + - tenants + scope: 'Cluster' + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.metadata }} + {{- if .enabled }} + {{- $any = true }} + - name: metadata.misc.projectcapsule.dev + {{- with .opts }} + opts: + {{- toYaml . | nindent 10 }} + {{- end }} + admissionReviewVersions: + - v1 + - v1beta1 + path: "/generic/metadata" + failurePolicy: {{ .failurePolicy }} + matchPolicy: {{ .matchPolicy }} + reinvocationPolicy: {{ .reinvocationPolicy }} + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .objectSelector }} + objectSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .matchConditions }} + matchConditions: + {{- toYaml . | nindent 10 }} + {{- end }} + rules: + {{- toYaml .rules | nindent 10 }} + sideEffects: None + timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} + {{- end }} + {{- end }} + {{- if not $any }} + [] + {{- end }} +{{- end }} {{- end }} diff --git a/charts/capsule/templates/crd-lifecycle/crds.tpl b/charts/capsule/templates/crd-lifecycle/crds.tpl index d994c72b..86485988 100644 --- a/charts/capsule/templates/crd-lifecycle/crds.tpl +++ b/charts/capsule/templates/crd-lifecycle/crds.tpl @@ -48,7 +48,9 @@ metadata: {{- include "capsule.crds.annotations" . | nindent 4 }} labels: app.kubernetes.io/component: {{ include "capsule.crds.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.crds.component" . | quote }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} + {{- include "capsule.baselabels" . | nindent 4 }} data: content: | {{- printf "---\n%s" (toYaml $p) | nindent 4 }} diff --git a/charts/capsule/templates/crd-lifecycle/job.yaml b/charts/capsule/templates/crd-lifecycle/job.yaml index 50867673..29a24fe3 100644 --- a/charts/capsule/templates/crd-lifecycle/job.yaml +++ b/charts/capsule/templates/crd-lifecycle/job.yaml @@ -16,7 +16,9 @@ metadata: {{- end }} labels: app.kubernetes.io/component: {{ include "capsule.crds.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.crds.component" . | quote }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} + {{- include "capsule.baselabels" . | nindent 4 }} {{- with $Values.labels }} {{- . | toYaml | nindent 4 }} {{- end }} @@ -35,7 +37,8 @@ spec: {{- end }} labels: app.kubernetes.io/component: {{ include "capsule.crds.component" . | quote }} - {{- include "capsule.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ include "capsule.crds.component" . | quote }} + {{- include "capsule.selectorLabelInstance" . | nindent 8 }} {{- with $Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/capsule/templates/crd-lifecycle/rbac.yaml b/charts/capsule/templates/crd-lifecycle/rbac.yaml index a5f8f380..524f964c 100644 --- a/charts/capsule/templates/crd-lifecycle/rbac.yaml +++ b/charts/capsule/templates/crd-lifecycle/rbac.yaml @@ -3,7 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "capsule.crds.name" . }} + name: capsule:{{ .Release.Name }}:crds namespace: {{ .Release.Namespace | quote }} annotations: # create hook dependencies in the right order @@ -11,7 +11,9 @@ metadata: {{- include "capsule.crds.annotations" . | nindent 4 }} labels: app.kubernetes.io/component: {{ include "capsule.crds.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.crds.component" . | quote }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} + {{- include "capsule.baselabels" . | nindent 4 }} rules: - apiGroups: - "" @@ -33,6 +35,9 @@ rules: - tenants.capsule.clastix.io - tenantowners.capsule.clastix.io - rulestatuses.capsule.clastix.io + - customquotas.capsule.clastix.io + - globalcustomquotas.capsule.clastix.io + - quantityledgers.capsule.clastix.io verbs: - create - delete @@ -43,7 +48,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: {{ include "capsule.crds.name" . }} + name: capsule:{{ .Release.Name }}:crds namespace: {{ .Release.Namespace | quote }} annotations: # create hook dependencies in the right order @@ -51,11 +56,13 @@ metadata: {{- include "capsule.crds.annotations" . | nindent 4 }} labels: app.kubernetes.io/component: {{ include "capsule.crds.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.crds.component" . | quote }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} + {{- include "capsule.baselabels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ include "capsule.crds.name" . }} + name: capsule:{{ .Release.Name }}:crds subjects: - kind: ServiceAccount name: {{ include "capsule.crds.name" . }} diff --git a/charts/capsule/templates/crd-lifecycle/serviceaccount.yaml b/charts/capsule/templates/crd-lifecycle/serviceaccount.yaml index 9ab32821..c4ec4105 100644 --- a/charts/capsule/templates/crd-lifecycle/serviceaccount.yaml +++ b/charts/capsule/templates/crd-lifecycle/serviceaccount.yaml @@ -11,5 +11,7 @@ metadata: {{- include "capsule.crds.annotations" . | nindent 4 }} labels: app.kubernetes.io/component: {{ include "capsule.crds.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.crds.component" . | quote }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} + {{- include "capsule.baselabels" . | nindent 4 }} {{- end }} diff --git a/charts/capsule/templates/dashboards/diagnostics.yaml b/charts/capsule/templates/dashboards/diagnostics.yaml index 299d60f3..c21e3976 100644 --- a/charts/capsule/templates/dashboards/diagnostics.yaml +++ b/charts/capsule/templates/dashboards/diagnostics.yaml @@ -1,6 +1,7 @@ +{{- if or (and $.Values.crds.exclusive $.Values.crds.createDiagnostics) (not $.Values.crds.exclusive) }} {{- if $.Values.monitoring.diagnostics.enabled }} - {{ range $path, $_ := .Files.Glob "dashboards/**-dashboard.json" }} + {{ range $path, $_ := .Files.Glob "diagnostics/**-dashboard.json" }} {{- with $ }} {{- $content := (.Files.Get $path) }} --- @@ -49,3 +50,4 @@ spec: {{- end }} {{- end }} {{- end }} +{{- end }} diff --git a/charts/capsule/templates/mutatingwebhookconfiguration.yaml b/charts/capsule/templates/mutatingwebhookconfiguration.yaml deleted file mode 100644 index 49c5bc20..00000000 --- a/charts/capsule/templates/mutatingwebhookconfiguration.yaml +++ /dev/null @@ -1,342 +0,0 @@ -{{- if or (not $.Values.crds.exclusive) ($.Values.webhooks.exclusive) }} -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: {{ include "capsule.fullname" . }}-mutating-webhook-configuration - namespace: {{ $.Release.Namespace }} - labels: - {{- include "capsule.labels" $ | nindent 4 }} - {{- include "admission.labels" . | nindent 4 }} - annotations: - {{- include "admission.annotations" . | nindent 4 }} -webhooks: -{{- with (mergeOverwrite .Values.webhooks.hooks.pods .Values.webhooks.hooks.defaults.pods) }} - {{- if .enabled }} -- name: pod.defaults.projectcapsule.dev - admissionReviewVersions: - - v1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/defaults" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - reinvocationPolicy: {{ .reinvocationPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - scope: "Namespaced" - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with (mergeOverwrite .Values.webhooks.hooks.persistentvolumeclaims .Values.webhooks.hooks.defaults.pvc) }} - {{- if .enabled }} -- name: storage.defaults.projectcapsule.dev - admissionReviewVersions: - - v1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/defaults" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - reinvocationPolicy: {{ .reinvocationPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - persistentvolumeclaims - scope: "Namespaced" - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with (mergeOverwrite .Values.webhooks.hooks.ingresses .Values.webhooks.hooks.defaults.ingress) }} - {{- if .enabled }} -- name: ingress.defaults.projectcapsule.dev - admissionReviewVersions: - - v1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/defaults" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - reinvocationPolicy: {{ .reinvocationPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - networking.k8s.io - apiVersions: - - v1beta1 - - v1 - operations: - - CREATE - - UPDATE - resources: - - ingresses - scope: "Namespaced" - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.gateways }} - {{- if .enabled }} -- name: gateway.defaults.projectcapsule.dev - admissionReviewVersions: - - v1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/defaults" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - reinvocationPolicy: {{ .reinvocationPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - gateway.networking.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - gateways - scope: "Namespaced" - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with (mergeOverwrite .Values.webhooks.hooks.namespaces .Values.webhooks.hooks.namespaceOwnerReference) }} - {{- if .enabled }} -- name: namespaces.tenants.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/namespaces/mutating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - reinvocationPolicy: {{ .reinvocationPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - namespaces - scope: '*' - sideEffects: NoneOnDryRun - timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.resourcepools.pools }} - {{- if .enabled }} -- name: resourcepools.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/resourcepools/mutating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - reinvocationPolicy: {{ .reinvocationPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - "capsule.clastix.io" - apiVersions: - - "*" - operations: - - CREATE - - UPDATE - resources: - - resourcepools - scope: '*' - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.resourcepools.claims }} - {{- if .enabled }} -- name: resourcepoolclaims.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/resourcepools/claim/mutating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - reinvocationPolicy: {{ .reinvocationPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - "capsule.clastix.io" - apiVersions: - - "*" - operations: - - CREATE - - UPDATE - resources: - - resourcepoolclaims - scope: '*' - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.tenants }} - {{- if .enabled }} -- name: tenants.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/tenants/mutating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - reinvocationPolicy: {{ .reinvocationPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - capsule.clastix.io - apiVersions: - - "*" - operations: - - CREATE - - UPDATE - - DELETE - resources: - - tenants - scope: 'Cluster' - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.tenantLabel }} - {{- if .enabled }} -- name: assign.misc.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/misc/tenant-label" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - reinvocationPolicy: {{ .reinvocationPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - {{- toYaml .rules | nindent 4 }} - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- end }} diff --git a/charts/capsule/templates/post-install/job.yaml b/charts/capsule/templates/post-install/job.yaml index 8f40da95..e5b55a46 100644 --- a/charts/capsule/templates/post-install/job.yaml +++ b/charts/capsule/templates/post-install/job.yaml @@ -1,7 +1,7 @@ {{- $Values := mergeOverwrite $.Values.global.jobs.kubectl $.Values.jobs -}} - -{{- if .Values.tls.create }} - {{- if and (not $.Values.crds.exclusive) $.Values.global.jobs.postInstall.enabled }} +{{- if and (not $.Values.crds.exclusive) $.Values.global.jobs.postInstall.enabled }} + {{- if .Values.tls.create }} +--- apiVersion: batch/v1 kind: Job metadata: @@ -9,7 +9,9 @@ metadata: namespace: {{ $.Release.Namespace }} labels: app.kubernetes.io/component: {{ include "capsule.post-install.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.post-install.component" . | quote }} + {{- include "capsule.baselabels" . | nindent 4 }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} {{- with $Values.labels }} {{- . | toYaml | nindent 4 }} {{- end }} @@ -33,7 +35,8 @@ spec: {{- end }} labels: app.kubernetes.io/component: {{ include "capsule.post-install.component" . | quote }} - {{- include "capsule.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ include "capsule.post-install.component" . | quote }} + {{- include "capsule.selectorLabelInstance" . | nindent 8 }} {{- with $Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} @@ -70,7 +73,7 @@ spec: {{- end }} serviceAccountName: {{ include "capsule.post-install.name" . }} containers: - - name: post-install + - name: tls image: {{ include "capsule.jobsFullyQualifiedDockerImage" . }} imagePullPolicy: {{ $Values.image.pullPolicy }} command: diff --git a/charts/capsule/templates/post-install/rbac.yaml b/charts/capsule/templates/post-install/rbac.yaml index d4d2a03b..f7901eba 100644 --- a/charts/capsule/templates/post-install/rbac.yaml +++ b/charts/capsule/templates/post-install/rbac.yaml @@ -11,7 +11,9 @@ metadata: {{- include "capsule.post-install.annotations" . | nindent 4 }} labels: app.kubernetes.io/component: {{ include "capsule.post-install.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.post-install.component" . | quote }} + {{- include "capsule.baselabels" . | nindent 4 }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} rules: - apiGroups: - "" @@ -31,7 +33,9 @@ metadata: {{- include "capsule.post-install.annotations" . | nindent 4 }} labels: app.kubernetes.io/component: {{ include "capsule.post-install.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.post-install.component" . | quote }} + {{- include "capsule.baselabels" . | nindent 4 }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role diff --git a/charts/capsule/templates/post-install/serviceaccount.yaml b/charts/capsule/templates/post-install/serviceaccount.yaml index 28e89a95..375f3bdc 100644 --- a/charts/capsule/templates/post-install/serviceaccount.yaml +++ b/charts/capsule/templates/post-install/serviceaccount.yaml @@ -10,6 +10,8 @@ metadata: {{- include "capsule.post-install.annotations" . | nindent 4 }} labels: app.kubernetes.io/component: {{ include "capsule.post-install.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.post-install.component" . | quote }} + {{- include "capsule.baselabels" . | nindent 4 }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} {{- end }} {{- end }} diff --git a/charts/capsule/templates/pre-delete/job.yaml b/charts/capsule/templates/pre-delete/job.yaml index ad5350a3..ae2de6a6 100644 --- a/charts/capsule/templates/pre-delete/job.yaml +++ b/charts/capsule/templates/pre-delete/job.yaml @@ -9,7 +9,9 @@ metadata: namespace: {{ $.Release.Namespace }} labels: app.kubernetes.io/component: {{ include "capsule.pre-delete.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.pre-delete.component" . | quote }} + {{- include "capsule.baselabels" . | nindent 4 }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} {{- with $Values.labels }} {{- . | toYaml | nindent 4 }} {{- end }} @@ -33,7 +35,8 @@ spec: {{- end }} labels: app.kubernetes.io/component: {{ include "capsule.pre-delete.component" . | quote }} - {{- include "capsule.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ include "capsule.pre-delete.component" . | quote }} + {{- include "capsule.selectorLabelInstance" . | nindent 8 }} {{- with $Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/capsule/templates/pre-delete/rbac.yaml b/charts/capsule/templates/pre-delete/rbac.yaml index 11ad98c4..0585152d 100644 --- a/charts/capsule/templates/pre-delete/rbac.yaml +++ b/charts/capsule/templates/pre-delete/rbac.yaml @@ -11,7 +11,9 @@ metadata: {{- include "capsule.pre-delete.annotations" . | nindent 4 }} labels: app.kubernetes.io/component: {{ include "capsule.pre-delete.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.pre-delete.component" . | quote }} + {{- include "capsule.baselabels" . | nindent 4 }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} rules: - apiGroups: - rbac.authorization.k8s.io @@ -35,7 +37,9 @@ metadata: {{- include "capsule.pre-delete.annotations" . | nindent 4 }} labels: app.kubernetes.io/component: {{ include "capsule.pre-delete.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.pre-delete.component" . | quote }} + {{- include "capsule.baselabels" . | nindent 4 }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} rules: - apiGroups: - "" @@ -57,7 +61,9 @@ metadata: {{- include "capsule.pre-delete.annotations" . | nindent 4 }} labels: app.kubernetes.io/component: {{ include "capsule.pre-delete.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.pre-delete.component" . | quote }} + {{- include "capsule.baselabels" . | nindent 4 }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -78,7 +84,9 @@ metadata: {{- include "capsule.pre-delete.annotations" . | nindent 4 }} labels: app.kubernetes.io/component: {{ include "capsule.pre-delete.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.pre-delete.component" . | quote }} + {{- include "capsule.baselabels" . | nindent 4 }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role diff --git a/charts/capsule/templates/pre-delete/serviceaccount.yaml b/charts/capsule/templates/pre-delete/serviceaccount.yaml index 60a35545..5777459d 100644 --- a/charts/capsule/templates/pre-delete/serviceaccount.yaml +++ b/charts/capsule/templates/pre-delete/serviceaccount.yaml @@ -10,5 +10,7 @@ metadata: {{- include "capsule.pre-delete.annotations" . | nindent 4 }} labels: app.kubernetes.io/component: {{ include "capsule.pre-delete.component" . | quote }} - {{- include "capsule.labels" . | nindent 4 }} + app.kubernetes.io/name: {{ include "capsule.pre-delete.component" . | quote }} + {{- include "capsule.baselabels" . | nindent 4 }} + {{- include "capsule.selectorLabelInstance" . | nindent 4 }} {{- end }} diff --git a/charts/capsule/templates/rbac-tenants.yaml b/charts/capsule/templates/rbac-tenants.yaml index d0cb93f7..c04fb6cf 100644 --- a/charts/capsule/templates/rbac-tenants.yaml +++ b/charts/capsule/templates/rbac-tenants.yaml @@ -1,8 +1,9 @@ {{- if $.Values.rbac.resourcepoolclaims.create }} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "capsule.fullname" $ }}-resourcepoolclaims + name: capsule:{{ include "capsule.fullname" $ }}:resourcepoolclaims labels: {{- toYaml $.Values.rbac.resourcepoolclaims.labels | nindent 4 }} rules: @@ -11,10 +12,11 @@ rules: verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] {{- end }} {{- if $.Values.rbac.resources.create }} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "capsule.fullname" $ }}-resources + name: capsule:{{ include "capsule.fullname" $ }}:tenantresources labels: {{- toYaml $.Values.rbac.resources.labels | nindent 4 }} rules: diff --git a/charts/capsule/templates/rbac.yaml b/charts/capsule/templates/rbac.yaml index 62d168b3..79462dfa 100644 --- a/charts/capsule/templates/rbac.yaml +++ b/charts/capsule/templates/rbac.yaml @@ -1,6 +1,416 @@ -{{- if not $.Values.crds.exclusive }} +{{- if or (and $.Values.crds.exclusive $.Values.crds.createRBAC) (not $.Values.crds.exclusive) }} {{- if $.Values.manager.rbac.create }} --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: capsule:{{ .Release.Name }}:aggregate + labels: + {{- include "capsule.labels" . | nindent 4 }} + {{- with .Values.customAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +aggregationRule: + clusterRoleSelectors: + - matchLabels: + projectcapsule.dev/aggregate-to-controller: "true" + - matchLabels: + projectcapsule.dev/aggregate-to-controller-instance: {{ .Release.Name }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: capsule:{{ include "capsule.fullname" . }}:aggregate + labels: + {{- include "capsule.labels" . | nindent 4 }} + {{- with .Values.customAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: capsule:{{ .Release.Name }}:aggregate +subjects: +- kind: ServiceAccount + name: {{ include "capsule.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + {{- if $.Values.manager.rbac.minimal }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: capsule:{{ include "capsule.fullname" . }}:controller + labels: + {{- include "capsule.labels" . | nindent 4 }} + {{- with .Values.customAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: capsule:{{ .Release.Name }}:controller +subjects: +- kind: ServiceAccount + name: {{ include "capsule.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: capsule:{{ .Release.Name }}:controller + labels: + {{- include "capsule.labels" . | nindent 4 }} + {{- with .Values.customAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: +- apiGroups: + - "*" + resources: + - "*" + verbs: + - delete + - deletecollection +- apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["impersonate"] +- apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch"] +- apiGroups: + - "capsule.clastix.io" + resources: + - capsuleconfigurations + - capsuleconfigurations/status + - resourcepoolclaims + - resourcepoolclaims/status + - resourcepools + - resourcepools/status + - tenantresources + - tenantresources/status + - globaltenantresources + - globaltenantresources/status + - tenants + - tenants/status + - tenantowners + - tenantowners/status + - rulestatuses + - rulestatuses/status + - customquotas + - customquotas/status + - globalcustomquotas + - globalcustomquotas/status + - quantityledgers + - quantityledgers/status + verbs: + - create + - delete + - get + - patch + - update + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + resourceNames: + - capsuleconfigurations.capsule.clastix.io + - resourcepoolclaims.capsule.clastix.io + - resourcepools.capsule.clastix.io + - tenantresources.capsule.clastix.io + - globaltenantresources.capsule.clastix.io + - tenants.capsule.clastix.io + - tenantowners.capsule.clastix.io + - rulestatuses.capsule.clastix.io + verbs: + - get + - patch + - update +- apiGroups: + - "*" + resources: + - "*" + verbs: + - "get" + - "list" + - "watch" +- apiGroups: + - "" + resources: + - "services" + - "pods" + - "persistentvolumes" + verbs: + - "get" + - "list" + - "watch" + - "patch" + - "update" +- apiGroups: + - "" + resources: + - "namespaces" + verbs: + - "get" + - "list" + - "watch" + - "patch" + - "update" + - "delete" + - "create" +- apiGroups: + - "" + resources: + - "limitranges" + - "resourcequotas" + verbs: + - "get" + - "list" + - "watch" + - "create" + - "patch" + - "update" + - "delete" + - "deletecollection" +- apiGroups: + - "" + resources: + - "secrets" + verbs: + - "list" + - "watch" +- apiGroups: + - "networking.k8s.io" + resources: + - "networkpolicies" + verbs: + - "get" + - "list" + - "watch" + - "patch" + - "update" + - "delete" + - "deletecollection" +- apiGroups: + - "discovery.k8s.io" + resources: + - "endpointslices" + verbs: + - "get" + - "list" + - "watch" + - "patch" + - "update" + - "delete" + - "deletecollection" +- apiGroups: + - "rbac.authorization.k8s.io" + resources: + - "rolebindings" + verbs: + - "get" + - "list" + - "watch" + - "create" + - "patch" + - "update" + - "delete" + - "deletecollection" +- apiGroups: + - "gateway.networking.k8s.io" + resources: + - "gatewayclasses" + verbs: + - "get" + - "list" + - "watch" +- apiGroups: + - "networking.k8s.io" + resources: + - "ingressclasses" + - "ingresses" + verbs: + - "get" + - "list" + - "watch" +- apiGroups: + - "node.k8s.io" + resources: + - "runtimeclasses" + verbs: + - "get" + - "list" + - "watch" +- apiGroups: + - "" + resources: + - "nodes" + - "resourcequotas" + verbs: + - "get" + - "list" + - "watch" + - "patch" + - "update" +- apiGroups: + - "resource.k8s.io" + resources: + - "deviceclasses" + verbs: + - "get" + - "list" + - "watch" +- apiGroups: + - "scheduling.k8s.io" + resources: + - "priorityclasses" + verbs: + - "get" + - "list" + - "watch" +- apiGroups: + - "storage.k8s.io" + resources: + - "storageclasses" + verbs: + - "get" + - "list" + - "watch" +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - watch + - list +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - create +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + resourceNames: + - "{{ include "capsule.fullname" . }}-mutating-webhook-configuration" + - "{{ include "capsule.fullname" . }}-dynamic-webhook" + verbs: + - get + - patch + - update + - delete +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - watch + - list +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - create +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + resourceNames: + - {{ include "capsule.fullname" . }}-validating-webhook-configuration + - "{{ include "capsule.fullname" . }}-dynamic-webhook" + verbs: + - get + - patch + - update + - delete +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles"] + verbs: + - "list" + - "watch" + - get + - create + - update + - patch + - delete +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles"] + resourceNames: + - {{ $.Values.manager.options.rbac.provisioner}} + - {{ $.Values.manager.options.rbac.deleter}} + verbs: + - get + - create + - update + - patch + - delete +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterrolebindings"] + verbs: + - "list" + - "watch" + - get + - create + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: capsule:{{ .Release.Name }}:controller + namespace: {{ $.Release.Namespace }} + labels: + {{- include "capsule.labels" . | nindent 4 }} + {{- with .Values.customAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: capsule:{{ .Release.Name }}:controller + namespace: {{ $.Release.Namespace }} + labels: + {{- include "capsule.labels" $ | nindent 4 }} + {{- with $.Values.customAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: capsule:{{ .Release.Name }}:controller +subjects: +- kind: ServiceAccount + name: {{ include "capsule.serviceAccountName" $ }} + namespace: {{ $.Release.Namespace }} + {{- else }} +--- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -19,13 +429,14 @@ subjects: - kind: ServiceAccount name: {{ include "capsule.serviceAccountName" . }} namespace: {{ .Release.Namespace }} + {{- end }} {{- end }} {{- range $_, $cr := $.Values.manager.rbac.existingClusterRoles }} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: {{ include "capsule.fullname" $ }}-{{ $cr }} + name: capsule:{{ include "capsule.fullname" $ }}:{{ $cr }} labels: {{- include "capsule.labels" $ | nindent 4 }} {{- with $.Values.customAnnotations }} @@ -46,7 +457,7 @@ subjects: kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: {{ include "capsule.fullname" $ }}-{{ $nr }} + name: capsule:{{ include "capsule.fullname" $ }}:{{ $nr }} namespace: {{ $.Release.Namespace }} labels: {{- include "capsule.labels" $ | nindent 4 }} diff --git a/charts/capsule/templates/serviceaccount.yaml b/charts/capsule/templates/serviceaccount.yaml index d8729cbe..6f7c92aa 100644 --- a/charts/capsule/templates/serviceaccount.yaml +++ b/charts/capsule/templates/serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if not $.Values.crds.exclusive }} +{{- if or (and $.Values.crds.exclusive $.Values.crds.createRBAC) (not $.Values.crds.exclusive) }} {{- if .Values.serviceAccount.create -}} apiVersion: v1 kind: ServiceAccount diff --git a/charts/capsule/templates/validatingwebhookconfiguration.yaml b/charts/capsule/templates/validatingwebhookconfiguration.yaml deleted file mode 100644 index f4e6a33b..00000000 --- a/charts/capsule/templates/validatingwebhookconfiguration.yaml +++ /dev/null @@ -1,594 +0,0 @@ -{{- if or (not $.Values.crds.exclusive) ($.Values.webhooks.exclusive) }} -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: {{ include "capsule.fullname" . }}-validating-webhook-configuration - namespace: {{ $.Release.Namespace }} - labels: - {{- include "capsule.labels" $ | nindent 4 }} - {{- include "admission.labels" . | nindent 4 }} - annotations: - {{- include "admission.annotations" . | nindent 4 }} -webhooks: -{{- with .Values.webhooks.hooks.customresources }} - {{- if .enabled }} -- name: customresources.misc.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/misc/customresources" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - '*' - apiVersions: - - '*' - operations: - - CREATE - - UPDATE - - DELETE - resources: - - '*' - scope: Namespaced - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.cordoning }} - {{- if .enabled }} -- name: cordoning.misc.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/misc/cordoning" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .rules }} - rules: - {{- toYaml . | nindent 4 }} - {{- end }} - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.devices }} - {{- if .enabled }} -- name: devices.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/devices/validating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - resource.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - resourceclaimtemplates - - resourceclaims - scope: Namespaced - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.gateways }} - {{- if .enabled }} -- name: gateway.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/gateways/validating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - gateway.networking.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - gateways - scope: Namespaced - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.ingresses }} - {{- if .enabled }} -- name: ingress.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/ingresses/validating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - networking.k8s.io - - extensions - apiVersions: - - v1 - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - ingresses - scope: Namespaced - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.namespaces }} - {{- if .enabled }} -- name: namespaces.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/namespaces/validating " "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - - DELETE - resources: - - namespaces - - namespaces/status - - namespace/finalize - scope: '*' - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.nodes }} - {{- if .enabled }} -- name: nodes.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/nodes/validating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - UPDATE - resources: - - nodes - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.pods }} - {{- if .enabled }} -- name: pods.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/pods/validating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - pods - - pods/ephemeralcontainers - scope: Namespaced - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.persistentvolumeclaims }} - {{- if .enabled }} -- name: pvc.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/persistentvolumeclaims/validating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - persistentvolumeclaims - scope: Namespaced - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.services }} - {{- if .enabled }} -- name: services.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/services/validating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - services - scope: Namespaced - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with (mergeOverwrite .Values.webhooks.hooks.managed .Values.webhooks.hooks.tenantResourceObjects) }} - {{- if .enabled }} -- name: resource-objects.tenant.projectcapsule.dev - admissionReviewVersions: - - v1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/misc/managed" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - {{- toYaml .rules | nindent 4 }} - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.tenants }} - {{- if .enabled }} -- name: tenants.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/tenants/validating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - capsule.clastix.io - apiVersions: - - v1beta2 - operations: - - CREATE - - UPDATE - - DELETE - resources: - - tenants - scope: 'Cluster' - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.resourcepools.pools }} - {{- if .enabled }} -- name: resourcepools.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/resourcepools/validating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - "capsule.clastix.io" - apiVersions: - - "*" - operations: - - CREATE - - UPDATE - resources: - - resourcepools - scope: '*' - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.resourcepools.pools }} - {{- if .enabled }} -- name: resourcepoolclaims.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/resourcepools/claim/validating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - "capsule.clastix.io" - apiVersions: - - "*" - operations: - - CREATE - - UPDATE - - DELETE - resources: - - resourcepoolclaims - scope: '*' - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.serviceaccounts }} - {{- if .enabled }} -- name: serviceaccounts.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/serviceaccounts/validating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - '*' - apiVersions: - - '*' - operations: - - CREATE - - UPDATE - resources: - - 'serviceaccounts' - scope: Namespaced - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- with .Values.webhooks.hooks.config }} - {{- if .enabled }} -- name: config.projectcapsule.dev - admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - {{- include "capsule.webhooks.service" (dict "path" "/config/validating" "ctx" $) | nindent 4 }} - failurePolicy: {{ .failurePolicy }} - matchPolicy: {{ .matchPolicy }} - {{- with .namespaceSelector }} - namespaceSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .objectSelector }} - objectSelector: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .matchConditions }} - matchConditions: - {{- toYaml . | nindent 4 }} - {{- end }} - rules: - - apiGroups: - - capsule.clastix.io - apiVersions: - - v1beta2 - operations: - - UPDATE - resources: - - capsuleconfigurations - scope: 'Cluster' - sideEffects: None - timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} - {{- end }} -{{- end }} -{{- end }} diff --git a/charts/capsule/templates/webhook-service.yaml b/charts/capsule/templates/webhook-service.yaml index f13028ef..983e4d2b 100644 --- a/charts/capsule/templates/webhook-service.yaml +++ b/charts/capsule/templates/webhook-service.yaml @@ -12,10 +12,9 @@ metadata: {{- end }} spec: ports: - - port: 443 - name: https + - name: admission protocol: TCP - targetPort: {{ .Values.manager.webhookPort }} + port: 9443 selector: {{- include "capsule.selectorLabels" . | nindent 4 }} sessionAffinity: None diff --git a/charts/capsule/values.schema.json b/charts/capsule/values.schema.json index 9cb33974..c094c7a1 100644 --- a/charts/capsule/values.schema.json +++ b/charts/capsule/values.schema.json @@ -19,6 +19,36 @@ } } }, + "conversions": { + "type": "object", + "properties": { + "service": { + "type": "object", + "properties": { + "caBundle": { + "description": "CABundle for the webhook service", + "type": "string" + }, + "name": { + "description": "Custom service name for the webhook service", + "type": "string" + }, + "namespace": { + "description": "Custom service namespace for the webhook service", + "type": "string" + }, + "port": { + "description": "Custom service port for the webhook service", + "type": "null" + }, + "url": { + "description": "The URL where the capsule webhook services are running (Overwrites cluster scoped service definition)", + "type": "string" + } + } + } + } + }, "crds": { "type": "object", "properties": { @@ -30,6 +60,14 @@ "description": "Create additionally CapsuleConfiguration even if CRDs are exclusive", "type": "boolean" }, + "createDiagnostics": { + "description": "Create Diagnostic Dashboards even if CRDs are exclusive", + "type": "boolean" + }, + "createRBAC": { + "description": "Create RBAC and Serviceaccount even if CRDs are exclusive", + "type": "boolean" + }, "exclusive": { "description": "Only install the CRDs, no other primitives", "type": "boolean" @@ -343,6 +381,14 @@ "description": "DEPRECATED: use users properties. Names of the users considered as Capsule users.", "type": "array" }, + "clientConnectionBurst": { + "description": "Burst to use for interacting with kubernetes apiserver", + "type": "integer" + }, + "clientConnectionQPS": { + "description": "QPS to use for interacting with kubernetes apiserver", + "type": "number" + }, "createConfiguration": { "description": "Create Configuration", "type": "boolean" @@ -359,6 +405,10 @@ "description": "Define groups which when found in the request of a user will be ignored by the Capsule this might be useful if you have one group where all the users are in, but you want to separate administrators from normal users with additional groups.", "type": "array" }, + "impersonation": { + "description": "Impersonation", + "type": "object" + }, "labels": { "description": "Additional labels to add to the CapsuleConfiguration resource", "type": "object" @@ -466,6 +516,10 @@ "existingRoles": { "description": "Specifies further cluster roles to be added to the Capsule manager service account.", "type": "array" + }, + "strict": { + "description": "Strongly restrict the RBAC assigned to Capsule Controller. When set to true you must aggregate further permissions by yourself.", + "type": "boolean" } } }, @@ -524,7 +578,12 @@ }, "labels": { "description": "Labels for dashboard configmaps", - "type": "object" + "type": "object", + "properties": { + "grafana_dashboard": { + "type": "string" + } + } }, "namespace": { "description": "Custom namespace for dashboard configmaps", @@ -570,7 +629,12 @@ }, "labels": { "description": "Labels for dashboard configmaps", - "type": "object" + "type": "object", + "properties": { + "grafana_dashboard": { + "type": "string" + } + } }, "operator": { "type": "object", @@ -710,6 +774,7 @@ "type": "object", "properties": { "resourcepoolclaims": { + "description": "Allow the creation of ResourcePoolClaims", "type": "object", "properties": { "create": { @@ -726,6 +791,7 @@ } }, "resources": { + "description": "Allow the creation of TenantResources", "type": "object", "properties": { "create": { @@ -832,6 +898,40 @@ "hooks": { "type": "object", "properties": { + "calculations": { + "description": "Webhook for Custom Quota Calculations ([Read More](https://projectcapsule.dev/docs/resource-management/customquotas/#admission))", + "type": "object", + "properties": { + "enabled": { + "description": "Enable the Hook", + "type": "boolean" + }, + "failurePolicy": { + "description": "[FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy)", + "type": "string" + }, + "matchConditions": { + "description": "[MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", + "type": "array" + }, + "matchPolicy": { + "description": "[MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", + "type": "string" + }, + "namespaceSelector": { + "description": "[NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector)", + "type": "object" + }, + "objectSelector": { + "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", + "type": "object" + }, + "rules": { + "description": "[Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules)", + "type": "array" + } + } + }, "config": { "type": "object", "properties": { @@ -859,6 +959,10 @@ "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, "reinvocationPolicy": { "description": "[ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy)", "type": "string" @@ -925,6 +1029,75 @@ "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, + "rules": { + "description": "[Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules)", + "type": "array", + "items": { + "type": "object", + "properties": { + "apiGroups": { + "type": "array", + "items": { + "type": "string" + } + }, + "apiVersions": { + "type": "array", + "items": { + "type": "string" + } + }, + "operations": { + "type": "array", + "items": { + "type": "string" + } + }, + "resources": { + "type": "array", + "items": { + "type": "string" + } + }, + "scope": { + "type": "string" + } + } + } + } + } + }, + "customquotas": { + "type": "object", + "properties": { + "enabled": { + "description": "Enable the Hook", + "type": "boolean" + }, + "failurePolicy": { + "description": "[FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy)", + "type": "string" + }, + "matchConditions": { + "description": "[MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", + "type": "array" + }, + "matchPolicy": { + "description": "[MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", + "type": "string" + }, + "namespaceSelector": { + "description": "[NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector)", + "type": "object" + }, + "objectSelector": { + "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", + "type": "object" + }, "rules": { "description": "[Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules)", "type": "array", @@ -1016,6 +1189,10 @@ "objectSelector": { "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" + }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" } } }, @@ -1079,6 +1256,10 @@ "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, "reinvocationPolicy": { "description": "[ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy)", "type": "string" @@ -1127,6 +1308,79 @@ "objectSelector": { "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" + }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, + "reinvocationPolicy": { + "description": "[ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy)", + "type": "string" + } + } + }, + "globalcustomquotas": { + "type": "object", + "properties": { + "enabled": { + "description": "Enable the Hook", + "type": "boolean" + }, + "failurePolicy": { + "description": "[FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy)", + "type": "string" + }, + "matchConditions": { + "description": "[MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", + "type": "array" + }, + "matchPolicy": { + "description": "[MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", + "type": "string" + }, + "namespaceSelector": { + "description": "[NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector)", + "type": "object" + }, + "objectSelector": { + "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", + "type": "object" + }, + "rules": { + "description": "[Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules)", + "type": "array", + "items": { + "type": "object", + "properties": { + "apiGroups": { + "type": "array", + "items": { + "type": "string" + } + }, + "apiVersions": { + "type": "array", + "items": { + "type": "string" + } + }, + "operations": { + "type": "array", + "items": { + "type": "string" + } + }, + "resources": { + "type": "array", + "items": { + "type": "string" + } + }, + "scope": { + "type": "string" + } + } + } } } }, @@ -1173,6 +1427,10 @@ "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, "reinvocationPolicy": { "description": "[ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy)", "type": "string" @@ -1232,12 +1490,122 @@ }, "operator": { "type": "string" + }, + "values": { + "type": "array", + "items": { + "type": "string" + } } } } } } }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, + "rules": { + "description": "[Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules)", + "type": "array", + "items": { + "type": "object", + "properties": { + "apiGroups": { + "type": "array", + "items": { + "type": "string" + } + }, + "apiVersions": { + "type": "array", + "items": { + "type": "string" + } + }, + "operations": { + "type": "array", + "items": { + "type": "string" + } + }, + "resources": { + "type": "array", + "items": { + "type": "string" + } + }, + "scope": { + "type": "string" + } + } + } + } + } + }, + "metadata": { + "type": "object", + "properties": { + "enabled": { + "description": "Enable the Hook", + "type": "boolean" + }, + "failurePolicy": { + "description": "[FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy)", + "type": "string" + }, + "matchConditions": { + "description": "[MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", + "type": "array", + "items": { + "type": "object", + "properties": { + "expression": { + "type": "string" + }, + "name": { + "type": "string" + } + } + } + }, + "matchPolicy": { + "description": "[MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", + "type": "string" + }, + "namespaceSelector": { + "description": "[NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector)", + "type": "object", + "properties": { + "matchExpressions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "operator": { + "type": "string" + } + } + } + } + } + }, + "objectSelector": { + "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", + "type": "object" + }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, + "reinvocationPolicy": { + "description": "[ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy)", + "type": "string" + }, "rules": { "description": "[Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules)", "type": "array", @@ -1307,6 +1675,10 @@ "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, "reinvocationPolicy": { "description": "[ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy)", "type": "string" @@ -1339,6 +1711,10 @@ "objectSelector": { "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" + }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" } } }, @@ -1385,6 +1761,10 @@ "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, "reinvocationPolicy": { "description": "[ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy)", "type": "string" @@ -1434,12 +1814,123 @@ "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, "reinvocationPolicy": { "description": "[ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy)", "type": "string" } } }, + "replications": { + "type": "object", + "properties": { + "enabled": { + "description": "Enable the Hook", + "type": "boolean" + }, + "failurePolicy": { + "description": "[FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy)", + "type": "string" + }, + "matchConditions": { + "description": "[MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", + "type": "array" + }, + "matchPolicy": { + "description": "[MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", + "type": "string" + }, + "namespaceSelector": { + "description": "[NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector)", + "type": "object", + "properties": { + "matchExpressions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "operator": { + "type": "string" + } + } + } + } + } + }, + "objectSelector": { + "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", + "type": "object", + "properties": { + "matchExpressions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "values": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, + "rules": { + "description": "[Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules)", + "type": "array", + "items": { + "type": "object", + "properties": { + "apiGroups": { + "type": "array", + "items": { + "type": "string" + } + }, + "apiVersions": { + "type": "array", + "items": { + "type": "string" + } + }, + "operations": { + "type": "array", + "items": { + "type": "string" + } + }, + "resources": { + "type": "array", + "items": { + "type": "string" + } + }, + "scope": { + "type": "string" + } + } + } + } + } + }, "resourcepools": { "type": "object", "properties": { @@ -1469,6 +1960,14 @@ "objectSelector": { "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" + }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, + "reinvocationPolicy": { + "description": "[ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy)", + "type": "string" } } }, @@ -1498,6 +1997,14 @@ "objectSelector": { "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" + }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, + "reinvocationPolicy": { + "description": "[ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy)", + "type": "string" } } } @@ -1545,6 +2052,10 @@ "objectSelector": { "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" + }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" } } }, @@ -1590,107 +2101,15 @@ "objectSelector": { "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" - } - } - }, - "tenantLabel": { - "type": "object", - "properties": { - "enabled": { - "description": "Enable the Hook", - "type": "boolean" }, - "failurePolicy": { - "description": "[FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy)", - "type": "string" - }, - "matchConditions": { - "description": "[MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", - "type": "array", - "items": { - "type": "object", - "properties": { - "expression": { - "type": "string" - }, - "name": { - "type": "string" - } - } - } - }, - "matchPolicy": { - "description": "[MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy)", - "type": "string" - }, - "namespaceSelector": { - "description": "[NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector)", - "type": "object", - "properties": { - "matchExpressions": { - "type": "array", - "items": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "operator": { - "type": "string" - } - } - } - } - } - }, - "objectSelector": { - "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", + "opts": { + "description": "Capsule Hook Options", "type": "object" - }, - "reinvocationPolicy": { - "description": "[ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy)", - "type": "string" - }, - "rules": { - "description": "[Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules)", - "type": "array", - "items": { - "type": "object", - "properties": { - "apiGroups": { - "type": "array", - "items": { - "type": "string" - } - }, - "apiVersions": { - "type": "array", - "items": { - "type": "string" - } - }, - "operations": { - "type": "array", - "items": { - "type": "string" - } - }, - "resources": { - "type": "array", - "items": { - "type": "string" - } - }, - "scope": { - "type": "string" - } - } - } } } }, "tenantResourceObjects": { - "description": "Deprecated, use webhooks.hooks.managed instead", + "description": "Deprecated, use webhooks.hooks.replications instead", "type": "object" }, "tenants": { @@ -1720,6 +2139,10 @@ "description": "[ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector)", "type": "object" }, + "opts": { + "description": "Capsule Hook Options", + "type": "object" + }, "reinvocationPolicy": { "description": "[ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy)", "type": "string" diff --git a/charts/capsule/values.yaml b/charts/capsule/values.yaml index 9f136118..2f417960 100644 --- a/charts/capsule/values.yaml +++ b/charts/capsule/values.yaml @@ -69,16 +69,20 @@ global: crds: # -- Install the CustomResourceDefinitions (This also manages the lifecycle of the CRDs for update operations) install: true - # -- Only install the CRDs, no other primitives - exclusive: false - # -- Create additionally CapsuleConfiguration even if CRDs are exclusive - createConfig: false # -- Extra Labels for CRDs labels: {} # -- Extra Annotations for CRDs annnotations: {} # -- Render CRDS inline (in this case use --skip-crds when installing the chart and create the capsuleconfiguration independently) inline: false + # -- Only install the CRDs, no other primitives + exclusive: false + # -- Create additionally CapsuleConfiguration even if CRDs are exclusive + createConfig: false + # -- Create RBAC and Serviceaccount even if CRDs are exclusive + createRBAC: false + # -- Create Diagnostic Dashboards even if CRDs are exclusive + createDiagnostics: false # Secret Options tls: @@ -96,10 +100,12 @@ proxy: # These are ClusterRoles which grant permissions for Capsule CRDs to Tenant Owners rbac: + # -- Allow the creation of TenantResources resources: create: false labels: rbac.authorization.k8s.io/aggregate-to-admin: "true" + # -- Allow the creation of ResourcePoolClaims resourcepoolclaims: create: false labels: @@ -112,9 +118,15 @@ manager: rbac: # -- Specifies whether RBAC resources should be created. create: true + + # -- Strongly restrict the RBAC assigned to Capsule Controller. + # When set to true you must aggregate further permissions by yourself. + strict: false + # -- Specifies further cluster roles to be added to the Capsule manager service account. existingClusterRoles: [] # - cluster-admin + # -- Specifies further cluster roles to be added to the Capsule manager service account. existingRoles: [] # - namespace-admin @@ -178,6 +190,10 @@ manager: workers: 1 # -- Set the log verbosity of the capsule with a value from 1 to 5 logLevel: "info" + # -- QPS to use for interacting with kubernetes apiserver + clientConnectionQPS: 20.0 + # -- Burst to use for interacting with kubernetes apiserver + clientConnectionBurst: 30 # -- Define entities which are considered part of the Capsule construct. # Users not mentioned here will be ignored by Capsule users: @@ -188,6 +204,8 @@ manager: # for interacting with namespaces. Because if that label is not defined, it's assumed that namespace interaction was not targeted towards a tenant and will therefor # be ignored by capsule. May also be handy in GitOps scenarios where certain service accounts need to be able to manage namespaces for all tenants. administrators: [] + # - kind: "User" + # name: "kubernetes-admin" # - kind: User # name: alice # -- Define groups which when found in the request of a user will be ignored by the Capsule @@ -213,7 +231,7 @@ manager: deniedRegex: "" # -- Duration after which the in-memory cache is invalidated (based on usaage) and re-fetched from the API server - cacheInvalidation: 24h0m0s + cacheInvalidation: 0h30m0s # -- Managed RBAC configuration for the controller rbac: @@ -229,6 +247,8 @@ manager: # -- Name for the ClusterRole required to grant Namespace Provision permissions. provisioner: capsule-namespace-provisioner + # -- Impersonation + impersonation: {} # -- DEPRECATED: use users properties. # Names of the users considered as Capsule users. @@ -368,8 +388,8 @@ monitoring: # -- Annotations for dashboard configmaps annotations: {} # -- Labels for dashboard configmaps - labels: {} - # grafana_dashboard: "1" + labels: + grafana_dashboard: "1" # Grafana Operator operator: @@ -390,8 +410,8 @@ monitoring: # -- Annotations for dashboard configmaps annotations: {} # -- Labels for dashboard configmaps - labels: {} - # grafana_dashboard: "1" + labels: + grafana_dashboard: "1" # -- Custom namespace for dashboard configmaps namespace: "" @@ -433,6 +453,22 @@ monitoring: relabelings: [] +# Conversions Webhook configurations +conversions: + # Configure custom webhook service + service: + # -- The URL where the capsule webhook services are running (Overwrites cluster scoped service definition) + url: "" + # -- CABundle for the webhook service + caBundle: "" + # -- Custom service name for the webhook service + name: "" + # -- Custom service namespace for the webhook service + namespace: "" + # -- Custom service port for the webhook service + port: + + # Webhooks configurations webhooks: # -- When `crds.exclusive` is `true` the webhooks will be installed @@ -463,7 +499,8 @@ webhooks: # Admission Webhook Configuration hooks: - tenantLabel: + + customquotas: # -- Enable the Hook enabled: true # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) @@ -473,6 +510,81 @@ webhooks: # -- [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) objectSelector: {} # -- [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) + namespaceSelector: {} + # -- [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) + matchConditions: [] + # -- [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) + rules: + - apiGroups: + - capsule.clastix.io + apiVersions: + - v1beta2 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - customquotas + scope: 'Namespaced' + globalcustomquotas: + # -- Enable the Hook + enabled: true + # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) + failurePolicy: Fail + # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) + matchPolicy: Equivalent + # -- [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) + objectSelector: {} + # -- [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) + namespaceSelector: {} + # -- [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) + matchConditions: [] + # -- [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) + rules: + - apiGroups: + - capsule.clastix.io + apiVersions: + - v1beta2 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - globalcustomquotas + scope: 'Cluster' + + # -- Webhook for Custom Quota Calculations ([Read More](https://projectcapsule.dev/docs/resource-management/customquotas/#admission)) + calculations: + # -- Enable the Hook + enabled: false + # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) + failurePolicy: Fail + # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) + matchPolicy: Equivalent + # -- [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) + objectSelector: {} + # -- [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) + namespaceSelector: {} + # matchExpressions: + # - key: capsule.clastix.io/tenant + # operator: Exists + # -- [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) + matchConditions: [] + # -- [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) + rules: [] + + metadata: + # -- Enable the Hook + enabled: true + # -- Capsule Hook Options + opts: {} + # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) + failurePolicy: Ignore + # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) + matchPolicy: Equivalent + # -- [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) + objectSelector: {} + # -- [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) namespaceSelector: matchExpressions: - key: capsule.clastix.io/tenant @@ -503,6 +615,8 @@ webhooks: pools: # -- Enable the Hook enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Fail # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -513,10 +627,14 @@ webhooks: namespaceSelector: {} # -- [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) matchConditions: [] + # -- [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) + reinvocationPolicy: Never claims: # -- Enable the Hook enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Fail # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -527,10 +645,15 @@ webhooks: namespaceSelector: {} # -- [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) matchConditions: [] + # -- [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) + reinvocationPolicy: Never + customresources: # -- Enable the Hook enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Fail # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -552,10 +675,11 @@ webhooks: - name: ignore-events expression: 'request.resource.resource != "events"' - namespaces: # -- Enable the Hook enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Fail # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -572,6 +696,8 @@ webhooks: cordoning: # -- Enable the Hook enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Fail # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -610,23 +736,8 @@ webhooks: gateways: # -- Enable the Hook enabled: true - # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) - failurePolicy: Fail - # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) - matchPolicy: Equivalent - # -- [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) - objectSelector: {} - # -- [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) - namespaceSelector: - matchExpressions: - - key: capsule.clastix.io/tenant - operator: Exists - # -- [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) - matchConditions: [] - - ingresses: - # -- Enable the Hook - enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Fail # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -642,9 +753,35 @@ webhooks: matchConditions: [] # -- [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) reinvocationPolicy: Never + + + ingresses: + # -- Enable the Hook + enabled: true + # -- Capsule Hook Options + opts: {} + # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) + failurePolicy: Fail + # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) + matchPolicy: Equivalent + # -- [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) + objectSelector: {} + # -- [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) + namespaceSelector: + matchExpressions: + - key: capsule.clastix.io/tenant + operator: Exists + # -- [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) + matchConditions: [] + # -- [ReinvocationPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#reinvocation-policy) + reinvocationPolicy: Never + + devices: # -- Enable the Hook enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Fail # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -664,6 +801,8 @@ webhooks: pods: # -- Enable the Hook enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Fail # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -683,6 +822,8 @@ webhooks: persistentvolumeclaims: # -- Enable the Hook enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Fail # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -702,6 +843,8 @@ webhooks: tenants: # -- Enable the Hook enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Fail # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -718,6 +861,8 @@ webhooks: config: # -- Enable the Hook enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Ignore # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -734,19 +879,23 @@ webhooks: managed: # -- Enable the Hook enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) - failurePolicy: Fail + failurePolicy: Ignore # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) - matchPolicy: Exact + matchPolicy: Equivalent # -- [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) objectSelector: matchExpressions: - - key: "projectcapsule.dev/managed-by" - operator: Exists + - key: projectcapsule.dev/managed-by + operator: In + values: + - "controller" # -- [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) namespaceSelector: matchExpressions: - - key: capsule.clastix.io/tenant + - key: "capsule.clastix.io/tenant" operator: Exists # -- [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) matchConditions: [] @@ -762,11 +911,57 @@ webhooks: - DELETE resources: - '*' - scope: '*' + scope: "Namespaced" + + replications: + # -- Enable the Hook + enabled: true + # -- Capsule Hook Options + opts: {} + # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) + failurePolicy: Fail + # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) + matchPolicy: Equivalent + # -- [ObjectSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector) + objectSelector: + matchExpressions: + - key: projectcapsule.dev/created-by + operator: In + values: + - "replications" + # -- [NamespaceSelector](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector) + namespaceSelector: + matchExpressions: + - key: "capsule.clastix.io/tenant" + operator: Exists + # -- [MatchConditions](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) + matchConditions: [] + # - name: "exclude-privileged-users" + # expression: > + # !( + # request.userInfo.username in [ + # "kubernetes-admin" + # ] + # ) + + # -- [Rules](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-rules) + rules: + - apiGroups: + - '*' + apiVersions: + - '*' + operations: + - UPDATE + - DELETE + resources: + - '*' + scope: "*" services: # -- Enable the Hook enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Fail # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -784,6 +979,8 @@ webhooks: nodes: # -- Enable the Hook enabled: false + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Fail # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -798,6 +995,8 @@ webhooks: serviceaccounts: # -- Enable the Hook enabled: true + # -- Capsule Hook Options + opts: {} # -- [FailurePolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) failurePolicy: Fail # -- [MatchPolicy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy) @@ -823,5 +1022,5 @@ webhooks: # -- Deprecated, use webhooks.hooks.pods instead pods: {} - # -- Deprecated, use webhooks.hooks.managed instead + # -- Deprecated, use webhooks.hooks.replications instead tenantResourceObjects: {} diff --git a/cmd/main.go b/cmd/main.go index 1147b369..4f81e49f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -4,27 +4,36 @@ package main import ( + "crypto/tls" goflag "flag" "fmt" "os" + "path/filepath" goRuntime "runtime" flag "github.com/spf13/pflag" _ "go.uber.org/automaxprocs" "go.uber.org/zap/zapcore" + admissionv1 "k8s.io/api/admissionregistration/v1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilVersion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ctrlwebhook "sigs.k8s.io/controller-runtime/pkg/webhook" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -33,12 +42,15 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/cache" "github.com/projectcapsule/capsule/internal/controllers/admission" - configcontroller "github.com/projectcapsule/capsule/internal/controllers/cfg" + cacheinvalidator "github.com/projectcapsule/capsule/internal/controllers/cfg/invalidator" + configcontroller "github.com/projectcapsule/capsule/internal/controllers/cfg/status" + customquotacontroller "github.com/projectcapsule/capsule/internal/controllers/customquotas" podlabelscontroller "github.com/projectcapsule/capsule/internal/controllers/pod" "github.com/projectcapsule/capsule/internal/controllers/pv" rbaccontroller "github.com/projectcapsule/capsule/internal/controllers/rbac" "github.com/projectcapsule/capsule/internal/controllers/resourcepools" "github.com/projectcapsule/capsule/internal/controllers/resources" + rulestatuscontroller "github.com/projectcapsule/capsule/internal/controllers/rulestatus" servicelabelscontroller "github.com/projectcapsule/capsule/internal/controllers/servicelabels" tenantcontroller "github.com/projectcapsule/capsule/internal/controllers/tenant" tlscontroller "github.com/projectcapsule/capsule/internal/controllers/tls" @@ -46,11 +58,12 @@ import ( "github.com/projectcapsule/capsule/internal/metrics" "github.com/projectcapsule/capsule/internal/webhook" cfgvalidation "github.com/projectcapsule/capsule/internal/webhook/cfg" + customquotavalidation "github.com/projectcapsule/capsule/internal/webhook/customquota" "github.com/projectcapsule/capsule/internal/webhook/defaults" "github.com/projectcapsule/capsule/internal/webhook/dra" "github.com/projectcapsule/capsule/internal/webhook/gateway" + "github.com/projectcapsule/capsule/internal/webhook/generic" "github.com/projectcapsule/capsule/internal/webhook/ingress" - "github.com/projectcapsule/capsule/internal/webhook/misc" namespacemutation "github.com/projectcapsule/capsule/internal/webhook/namespace/mutation" namespacevalidation "github.com/projectcapsule/capsule/internal/webhook/namespace/validation" "github.com/projectcapsule/capsule/internal/webhook/node" @@ -62,11 +75,10 @@ import ( "github.com/projectcapsule/capsule/internal/webhook/serviceaccounts" tenantmutation "github.com/projectcapsule/capsule/internal/webhook/tenant/mutation" tenantvalidation "github.com/projectcapsule/capsule/internal/webhook/tenant/validation" - tntresource "github.com/projectcapsule/capsule/internal/webhook/tenantresource" - "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/handlers" "github.com/projectcapsule/capsule/pkg/runtime/indexers" + "github.com/projectcapsule/capsule/pkg/utils" ) var ( @@ -81,6 +93,7 @@ func init() { utilruntime.Must(capsulev1beta2.AddToScheme(scheme)) utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) utilruntime.Must(gatewayv1.Install(scheme)) + utilruntime.Must(admissionv1.AddToScheme(scheme)) } func printVersion() { @@ -91,27 +104,134 @@ func printVersion() { setupLog.Info(fmt.Sprintf("Go OS/Arch: %s/%s", goRuntime.GOOS, goRuntime.GOARCH)) } -//nolint:maintidx,cyclop +//nolint:maintidx,gocyclo,cyclop,gocognit func main() { controllerConfig := utilscontroller.ControllerOptions{} - var enableLeaderElection, enablePprof, version bool + var ( + metricsAddr, metricsCertPath, metricsCertName, metricsCertKey string + webhookCertPath, webhookCertName, webhookCertKey string - var metricsAddr, ns string + enableLeaderElection bool + enablePprof bool + version bool + secureMetrics bool + enableHTTP2 bool - var webhookPort int + clientConnectionQPS float32 + clientConnectionBurst int32 + + webhookPort int + ) var goFlagSet goflag.FlagSet - flag.IntVar(&controllerConfig.MaxConcurrentReconciles, "workers", 1, "MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run.") - flag.IntVar(&webhookPort, "webhook-port", 9443, "The port the webhook server binds to.") - flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, + var tlsOpts []func(*tls.Config) + + flag.StringVar( + &controllerConfig.ConfigurationName, + "configuration-name", + "default", + "The CapsuleConfiguration resource name to use", + ) + + flag.BoolVar( + &enableLeaderElection, + "enable-leader-election", + false, "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&version, "version", false, "Print the Capsule version and exit") - flag.StringVar(&controllerConfig.ConfigurationName, "configuration-name", "default", "The CapsuleConfiguration resource name to use") - flag.BoolVar(&enablePprof, "enable-pprof", false, "Enables Pprof endpoint for profiling (not recommend in production)") + "Enabling this will ensure there is only one active controller manager.", + ) + flag.IntVar( + &controllerConfig.MaxConcurrentReconciles, + "workers", + 1, + "MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run.", + ) + flag.StringVar( + &metricsAddr, + "metrics-addr", + ":8080", + "The address the metric endpoint binds to.", + ) + flag.BoolVar( + &secureMetrics, + "metrics-secure", + false, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.", + ) + flag.StringVar( + &metricsCertPath, + "metrics-cert-path", + "", + "The directory that contains the metrics server certificate.", + ) + flag.StringVar( + &metricsCertName, + "metrics-cert-name", + "tls.crt", + "The name of the metrics server certificate file.", + ) + flag.StringVar( + &metricsCertKey, + "metrics-cert-key", + "tls.key", + "The name of the metrics server key file.", + ) + flag.IntVar( + &webhookPort, + "webhook-port", + 9443, + "The port the webhook server binds to.", + ) + flag.StringVar( + &webhookCertPath, + "webhook-cert-path", + "/tmp/k8s-webhook-server/serving-certs", + "The directory that contains the webhook certificate.", + ) + flag.StringVar( + &webhookCertName, + "webhook-cert-name", + "tls.crt", + "The name of the webhook certificate file.", + ) + flag.StringVar( + &webhookCertKey, + "webhook-cert-key", + "tls.key", + "The name of the webhook key file.", + ) + flag.BoolVar( + &enableHTTP2, + "enable-http2", false, + "If set, HTTP/2 will be enabled for the metrics and webhook servers", + ) + flag.Float32Var( + &clientConnectionQPS, + "client-connection-qps", + 20.0, + "QPS to use for interacting with kubernetes apiserver.", + ) + flag.Int32Var( + &clientConnectionBurst, + "client-connection-burst", + 30, + "Burst to use for interacting with kubernetes apiserver.", + ) + + flag.BoolVar( + &enablePprof, + "enable-pprof", + false, + "Enables Pprof endpoint for profiling (not recommend in production)", + ) + flag.BoolVar( + &version, + "version", + false, + "Print the Capsule version and exit", + ) opts := zap.Options{ EncoderConfigOptions: append([]zap.EncoderConfigOption{}, func(config *zapcore.EncoderConfig) { @@ -131,10 +251,19 @@ func main() { os.Exit(0) } + ctx := ctrl.SetupSignalHandler() + setupLog.V(5).Info("Controller", "Options", controllerConfig) - if ns = os.Getenv("NAMESPACE"); len(ns) == 0 { - setupLog.Error(fmt.Errorf("unable to determinate the Namespace Capsule is running on"), "unable to start manager") + var ns string + + if ns = os.Getenv(configuration.EnvironmentControllerNamespace); len(ns) == 0 { + setupLog.Error(fmt.Errorf("unable to determinate the Namespace Capsule is running on. Please export %s", configuration.EnvironmentControllerNamespace), "unable to start manager") + os.Exit(1) + } + + if serviceAccountName := os.Getenv(configuration.EnvironmentServiceaccountName); len(serviceAccountName) == 0 { + setupLog.Error(fmt.Errorf("unable to determinate the ServiceAccount Capsule is running with. Please export %s", configuration.EnvironmentServiceaccountName), "unable to start manager") os.Exit(1) } @@ -143,13 +272,167 @@ func main() { os.Exit(1) } - ctrlOpts := ctrl.Options{ + restConfig, err := ctrl.GetConfig() + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + // Suppress Warnings + restConfig.WarningHandler = rest.NoWarnings{} + restConfig.QPS = clientConnectionQPS + restConfig.Burst = int(clientConnectionBurst) + + directClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{ Scheme: scheme, - Metrics: metricsserver.Options{ - BindAddress: metricsAddr, - }, + }) + if err != nil { + setupLog.Error(err, "unable to create the direct client") + os.Exit(1) + } + + directCfg := configuration.NewCapsuleConfiguration(ctx, directClient, restConfig, controllerConfig.ConfigurationName) + + tlsReconciler := &tlscontroller.Reconciler{} + + if directCfg.EnableTLSConfiguration() { + tlsReconciler = &tlscontroller.Reconciler{ + Client: directClient, + Log: ctrl.Log.WithName("capsule.ctrl").WithName("tls"), + Namespace: ns, + Configuration: directCfg, + } + + tlsCert := &corev1.Secret{} + + if err = directClient.Get(ctx, types.NamespacedName{ + Namespace: ns, + Name: directCfg.TLSSecretName(), + }, tlsCert); err != nil { + if !apierrors.IsNotFound(err) { + setupLog.Error(err, "unable to get Capsule TLS secret") + os.Exit(1) + } + + tlsCert = &corev1.Secret{} + tlsCert.Name = directCfg.TLSSecretName() + tlsCert.Namespace = ns + tlsCert.Data = map[string][]byte{} + } + + // Reconcile TLS certificates before starting controllers and webhooks + if err = tlsReconciler.ReconcileCertificates(ctx, tlsCert); err != nil { + setupLog.Error(err, "unable to reconcile Capsule TLS secret") + os.Exit(1) + } + } + + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + + c.NextProtos = []string{"http/1.1"} + } + + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + // Create watchers for metrics and webhooks certificates + var metricsCertWatcher, webhookCertWatcher *certwatcher.CertWatcher + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.20.4/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + TLSOpts: tlsOpts, + } + + if secureMetrics { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.20.4/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + if len(metricsCertPath) > 0 { + setupLog.Info( + "Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", + metricsCertPath, + "metrics-cert-name", + metricsCertName, + "metrics-cert-key", + metricsCertKey, + ) + + var err error + + metricsCertWatcher, err = certwatcher.New( + filepath.Join(metricsCertPath, metricsCertName), + filepath.Join(metricsCertPath, metricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append( + metricsServerOptions.TLSOpts, + func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }, + ) + } + + // Initial webhook TLS options + webhookTLSOpts := tlsOpts + + if len(webhookCertPath) > 0 { + setupLog.Info( + "Initializing webhook certificate watcher using provided certificates", + "webhook-cert-path", + webhookCertPath, + "webhook-cert-name", + webhookCertName, + "webhook-cert-key", + webhookCertKey, + ) + + var err error + + webhookCertWatcher, err = certwatcher.New( + filepath.Join(webhookCertPath, webhookCertName), + filepath.Join(webhookCertPath, webhookCertKey), + ) + if err != nil { + setupLog.Error(err, "Failed to initialize webhook certificate watcher") + os.Exit(1) + } + + webhookTLSOpts = append(webhookTLSOpts, func(config *tls.Config) { + config.GetCertificate = webhookCertWatcher.GetCertificate + }) + } + + ctrlOpts := ctrl.Options{ + Scheme: scheme, + Metrics: metricsServerOptions, WebhookServer: ctrlwebhook.NewServer(ctrlwebhook.Options{ - Port: webhookPort, + Port: webhookPort, + TLSOpts: webhookTLSOpts, }), LeaderElection: enableLeaderElection, LeaderElectionID: "42c733ea.clastix.capsule.io", @@ -165,65 +448,81 @@ func main() { ctrlOpts.PprofBindAddress = ":8082" } - manager, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrlOpts) + setupLog.Info("initializing manager") + + // Suppress Warnings + restConfig.WarningHandler = rest.NoWarnings{} + restConfig.QPS = clientConnectionQPS + restConfig.Burst = int(clientConnectionBurst) + + manager, err := ctrl.NewManager(restConfig, ctrlOpts) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) } + if metricsCertWatcher != nil { + if err := manager.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher") + os.Exit(1) + } + } + + if webhookCertWatcher != nil { + if err := manager.Add(webhookCertWatcher); err != nil { + setupLog.Error(err, "unable to add webhook certificate watcher") + os.Exit(1) + } + } + _ = manager.AddReadyzCheck("ping", healthz.Ping) _ = manager.AddHealthzCheck("ping", healthz.Ping) - ctx := ctrl.SetupSignalHandler() - - cfg := configuration.NewCapsuleConfiguration(ctx, manager.GetClient(), controllerConfig.ConfigurationName) - - directClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{ - Scheme: manager.GetScheme(), - Mapper: manager.GetRESTMapper(), - }) + dc, err := discovery.NewDiscoveryClientForConfig(manager.GetConfig()) if err != nil { - setupLog.Error(err, "unable to create the direct client") + setupLog.Error(err, "unable to create discovery client") os.Exit(1) } - directCfg := configuration.NewCapsuleConfiguration(ctx, directClient, controllerConfig.ConfigurationName) + dynamicClient, err := dynamic.NewForConfig(manager.GetConfig()) + if err != nil { + setupLog.Error(err, "unable to create dynamic client") + os.Exit(1) + } + + setupLog.Info("initializing capsule configuration") + + cfg := configuration.NewCapsuleConfiguration(ctx, manager.GetClient(), manager.GetConfig(), controllerConfig.ConfigurationName) + + setupLog.Info("initializing caches") + + // Initialize Notifiers (Channels) + customQuotaCh := make(chan event.TypedGenericEvent[*capsulev1beta2.CustomQuota], 1024) + globalCustomQuotaCh := make(chan event.TypedGenericEvent[*capsulev1beta2.GlobalCustomQuota], 1024) + + // Initialize Caches + impersonationCache := cache.NewImpersonationCache() + registryCache := cache.NewRegistryRuleSetCache() + customQuotaQuantityCache := cache.NewQuantityCache[string]() + jsonPathCache := cache.NewJSONPathCache() + targetsCache := cache.NewCompiledTargetsCache[string]() if directCfg.EnableTLSConfiguration() { - tlsReconciler := &tlscontroller.Reconciler{ - Client: directClient, - Log: ctrl.Log.WithName("capsule.ctrl").WithName("tls"), - Namespace: ns, - Configuration: directCfg, - } - if err = tlsReconciler.SetupWithManager(manager); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Namespace") os.Exit(1) } - - tlsCert := &corev1.Secret{} - - if err = directClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: directCfg.TLSSecretName()}, tlsCert); err != nil { - setupLog.Error(err, "unable to get Capsule TLS secret") - os.Exit(1) - } - // Reconcile TLS certificates before starting controllers and webhooks - if err = tlsReconciler.ReconcileCertificates(ctx, tlsCert); err != nil { - setupLog.Error(err, "unable to reconcile Capsule TLS secret") - os.Exit(1) - } } - registryCache := cache.NewRegistryRuleSetCache() - if err = (&tenantcontroller.Manager{ - RESTConfig: manager.GetConfig(), - Client: manager.GetClient(), - Metrics: metrics.MustMakeTenantRecorder(), - Log: ctrl.Log.WithName("capsule.ctrl").WithName("tenant"), - Recorder: manager.GetEventRecorder("tenant-controller"), - Configuration: cfg, + RESTConfig: manager.GetConfig(), + Client: manager.GetClient(), + DynamicClient: dynamicClient, + DiscoveryClient: dc, + Metrics: metrics.MustMakeTenantRecorder(), + Log: ctrl.Log.WithName("capsule.ctrl").WithName("tenant"), + Recorder: manager.GetEventRecorder("tenant-controller"), + Configuration: cfg, }).SetupWithManager(manager, controllerConfig); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Tenant") os.Exit(1) @@ -234,6 +533,8 @@ func main() { os.Exit(1) } + setupLog.Info("registering indexers") + if err = indexers.AddToManager(ctx, setupLog, manager); err != nil { setupLog.Error(err, "unable to setup indexers") os.Exit(1) @@ -241,14 +542,18 @@ func main() { var kubeVersion *utilVersion.Version - if kubeVersion, err = utils.GetK8sVersion(); err != nil { + if kubeVersion, err = utils.GetK8sVersionFromConfig(dc); err != nil { setupLog.Error(err, "unable to get kubernetes version") os.Exit(1) } + setupLog.Info("registering webhooks") + // webhooks: the order matters, don't change it and just append webhooksList := append( make([]handlers.Webhook, 0), + route.GenericReplicasHandler(), + route.GenericManagedHandler(cfg), route.Pod( pod.Handler( pod.ImagePullPolicy(), @@ -259,10 +564,15 @@ func main() { ), ), route.Ingress(ingress.Class(cfg, kubeVersion), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()), - route.PVC( + route.PVCValidating( pvc.Handler( - pvc.Validating(), - pvc.PersistentVolumeReuse(), + pvc.PersistentVolumeValidatingVolume(), + pvc.PersistentVolumeValidatingClass(), + ), + ), + route.PVCMutating( + pvc.Handler( + pvc.PersistentVolumeMutatingVolume(), ), ), route.Service( @@ -270,15 +580,16 @@ func main() { service.Validating(), ), ), - route.TenantResourceObjects(handlers.InCapsuleGroups(cfg, tntresource.WriteOpsHandler())), - route.Cordoning(handlers.InCapsuleGroups(cfg, misc.CordoningHandler(cfg))), route.Node(handlers.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))), + route.Cordoning(handlers.InCapsuleGroups(cfg, generic.CordoningHandler(cfg))), route.ServiceAccounts( serviceaccounts.Handler( - serviceaccounts.Validating(cfg), + cfg, + serviceaccounts.Promotion(cfg), + serviceaccounts.OwnerPromotion(cfg), ), ), - route.MiscCustomResources(misc.ResourceCounterHandler(manager.GetClient())), + route.GenericCustomResources(generic.ResourceCounterHandler(manager.GetClient())), route.Gateway(gateway.Class(cfg)), route.DeviceClass(dra.DeviceClass()), route.Defaults(defaults.Handler(cfg, kubeVersion)), @@ -288,6 +599,7 @@ func main() { route.TenantValidation( tenantvalidation.Handler(cfg, tenantvalidation.NameHandler(), + tenantvalidation.NamespaceMetadataHandler(), tenantvalidation.RoleBindingRegexHandler(), tenantvalidation.IngressClassRegexHandler(), tenantvalidation.StorageClassRegexHandler(), @@ -305,8 +617,7 @@ func main() { route.NamespaceValidation( namespacevalidation.NamespaceHandler( cfg, - namespacevalidation.PatchHandler(cfg), - namespacevalidation.FreezeHandler(cfg), + namespacevalidation.CordoningHandler(cfg), namespacevalidation.QuotaHandler(), namespacevalidation.PrefixHandler(cfg), namespacevalidation.UserMetadataHandler(), @@ -318,21 +629,35 @@ func main() { cfg, namespacemutation.OwnerReferenceHandler(cfg), namespacemutation.MetadataHandler(cfg), - namespacemutation.CordoningLabelHandler(cfg), + namespacemutation.NamespacePatchGuardHandler(cfg), ), ), route.ResourcePoolMutation((resourcepool.PoolMutationHandler(ctrl.Log.WithName("webhooks").WithName("resourcepool")))), route.ResourcePoolValidation((resourcepool.PoolValidationHandler(ctrl.Log.WithName("webhooks").WithName("resourcepool")))), route.ResourcePoolClaimMutation((resourcepool.ClaimMutationHandler(ctrl.Log.WithName("webhooks").WithName("resourcepoolclaims")))), route.ResourcePoolClaimValidation((resourcepool.ClaimValidationHandler(ctrl.Log.WithName("webhooks").WithName("resourcepoolclaims")))), - route.MiscTenantAssignment( - misc.TenantAssignmentHandler(), + route.CustomQuotaValidation((customquotavalidation.CustomQuotaValidationHandler( + targetsCache, + jsonPathCache, + ))), + route.GlobalCustomQuotaValidation((customquotavalidation.GlobalCustomQuotaValidationHandler( + targetsCache, + jsonPathCache, + ))), + route.CalculationCustomQuotas( + customquotavalidation.ObjectCalculationHandler( + targetsCache, + jsonPathCache, + ), ), - route.MiscManagedValidation( - handlers.InCapsuleGroups(cfg, misc.ManagedValidatingHandler()), + route.GenericTenantAssignment( + generic.TenantAssignmentHandler(), ), route.ConfigValidation( - cfgvalidation.WarningHandler(), + cfgvalidation.Handler(cfg, + cfgvalidation.WarningHandler(), + cfgvalidation.ServiceAccountHandler(), + ), ), ) @@ -389,21 +714,47 @@ func main() { } if err = (&configcontroller.Manager{ - Client: manager.GetClient(), - RegistryCache: registryCache, - Log: ctrl.Log.WithName("capsule.ctrl").WithName("configuration"), + Rest: manager.GetConfig(), + Client: manager.GetClient(), + Log: ctrl.Log.WithName("capsule.ctrl").WithName("configuration"), }).SetupWithManager(manager, controllerConfig); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CapsuleConfiguration") os.Exit(1) } - if err = (&resources.Global{}).SetupWithManager(manager, controllerConfig); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "resources.Global") + if err = (&rulestatuscontroller.Manager{ + Client: manager.GetClient(), + Log: ctrl.Log.WithName("capsule.ctrl").WithName("ruleset"), + }).SetupWithManager(manager, controllerConfig); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "RuleSet") os.Exit(1) } - if err = (&resources.Namespaced{}).SetupWithManager(manager, controllerConfig); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "resources.Namespaced") + setupLog.Info("initializing controllers") + + localInvalidator := &cacheinvalidator.CacheInvalidator{ + Log: ctrl.Log.WithName("capsule.ctrl").WithName("invalidator"), + Client: manager.GetClient(), + Configuration: directCfg, + ImpersonationCache: impersonationCache, + RegistryCache: registryCache, + JSONPathCache: jsonPathCache, + TargetsCache: targetsCache, + } + + if err := localInvalidator.SetupWithManager(manager, controllerConfig); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "invalidator") + os.Exit(1) + } + + if err := resources.Add( + ctrl.Log.WithName("controllers").WithName("TenantResources"), + manager, + cfg, + controllerConfig, + impersonationCache, + ); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "tenantresources") os.Exit(1) } @@ -412,7 +763,7 @@ func main() { manager, manager.GetEventRecorder("admission-ctrl"), controllerConfig, - cfg, + directCfg, ); err != nil { setupLog.Error(err, "unable to create controller", "controller", "admission") os.Exit(1) @@ -428,6 +779,20 @@ func main() { os.Exit(1) } + if err = customquotacontroller.Add(ctrl.Log.WithName("controllers").WithName("CustomQuotas"), + manager, + manager.GetEventRecorder("customquotas-ctrl"), + controllerConfig, + customQuotaQuantityCache, + jsonPathCache, + targetsCache, + customQuotaCh, + globalCustomQuotaCh, + ); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "customquotas") + os.Exit(1) + } + setupLog.Info("starting manager") if err = manager.Start(ctx); err != nil { diff --git a/e2e/additional_role_bindings_test.go b/e2e/additional_role_bindings_test.go index 21c2f1ae..008cec84 100644 --- a/e2e/additional_role_bindings_test.go +++ b/e2e/additional_role_bindings_test.go @@ -13,28 +13,32 @@ import ( "k8s.io/apimachinery/pkg/types" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespace with an additional Role Binding", Label("tenant"), func() { +var _ = Describe("creating a Namespace with an additional Role Binding", Ordered, Label("tenant", "permissions", "rolebindings"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "additional-role-binding", + Name: "e2e-additional-role-binding", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "dale", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-additional-role-binding", Kind: "User", }, }, }, }, - AdditionalRoleBindings: []api.AdditionalRoleBindingsSpec{ + AdditionalRoleBindings: []rbac.AdditionalRoleBindingsSpec{ { - ClusterRoleName: "crds-rolebinding", + ClusterRoleName: "view", Subjects: []rbacv1.Subject{ { Kind: "Group", @@ -52,16 +56,31 @@ var _ = Describe("creating a Namespace with an additional Role Binding", Label(" tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) + JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should be assigned to each Namespace", func() { + ns1 := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + NamespaceCreation(ns1, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + + ns2 := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + NamespaceCreation(ns2, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) t := &capsulev1beta2.Tenant{} Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns1).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns2).Should(Succeed()) + VerifyTenantRoleBindings(t) }) }) diff --git a/e2e/administrators_test.go b/e2e/config_administrators_test.go similarity index 55% rename from e2e/administrators_test.go rename to e2e/config_administrators_test.go index 840ed4a0..f3b01eb2 100644 --- a/e2e/administrators_test.go +++ b/e2e/config_administrators_test.go @@ -13,23 +13,26 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("Administrators", Label("namespace", "permissions"), func() { +var _ = Describe("Administrators", Ordered, Label("namespace", "permissions", "administrators", "config"), func() { originConfig := &capsulev1beta2.CapsuleConfiguration{} tnt1 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tnt-admins-1", + Name: "e2e-tnt-admins-1", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "paul", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tnt-admins-1", Kind: "User", }, }, @@ -40,14 +43,17 @@ var _ = Describe("Administrators", Label("namespace", "permissions"), func() { tnt2 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tnt-admins-2", + Name: "e2e-tnt-admins-2", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "george", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tnt-admins-2", Kind: "User", }, }, @@ -56,7 +62,7 @@ var _ = Describe("Administrators", Label("namespace", "permissions"), func() { }, } - admin := api.UserSpec{ + admin := rbac.UserSpec{ Name: "admin", Kind: "User", } @@ -71,17 +77,18 @@ var _ = Describe("Administrators", Label("namespace", "permissions"), func() { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) } ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { - configuration.Spec.Administrators = []api.UserSpec{admin} + configuration.Spec.Administrators = []rbac.UserSpec{admin} }) }) JustAfterEach(func() { for _, tnt := range []*capsulev1beta2.Tenant{tnt1, tnt2} { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) } Eventually(func() error { @@ -121,8 +128,7 @@ var _ = Describe("Administrators", Label("namespace", "permissions"), func() { Expect(len(ns.OwnerReferences)).To(Equal(0)) PatchTenantLabelForNamespace(tnt1, ns, ownerClient(admin), defaultTimeoutInterval).Should(Succeed()) - - NamespaceIsPartOfTenant(tnt1, ns) + NamespaceIsPartOfTenant(tnt1, ns).Should(Succeed()) }) }) @@ -136,22 +142,27 @@ var _ = Describe("Administrators", Label("namespace", "permissions"), func() { }) By("verifing tenant state", func() { - TenantNamespaceList(tnt1, defaultTimeoutInterval).Should(ContainElements(ns1.GetName())) + NamespaceIsPartOfTenant(tnt1, ns1).Should(Succeed()) - t := &capsulev1beta2.Tenant{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt1.GetName()}, t)).Should(Succeed()) - Expect(t.Status.Size).To(Equal(uint(1))) + Eventually(func(g Gomega) { + t := &capsulev1beta2.Tenant{} + g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt1.GetName()}, t)).To(Succeed()) + g.Expect(t.Status.Size).To(Equal(uint(1))) - instance := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{Name: ns1.GetName(), UID: ns1.GetUID()}) - Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") + instance := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ + Name: ns1.GetName(), + UID: ns1.GetUID(), + }) + g.Expect(instance).ToNot(BeNil(), "Namespace instance should not be nil") - condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).ToNot(BeNil(), "Condition instance should not be nil") - Expect(instance.Name).To(Equal(ns1.GetName())) - Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected namespace condition status to be True") - Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected namespace condition reason to be Succeeded") + g.Expect(instance.Name).To(Equal(ns1.GetName())) + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected namespace condition status to be True") + g.Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") + g.Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected namespace condition reason to be Succeeded") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) ns2 := NewNamespace("", map[string]string{ @@ -163,23 +174,25 @@ var _ = Describe("Administrators", Label("namespace", "permissions"), func() { }) By("verifing tenant state", func() { - TenantNamespaceList(tnt2, defaultTimeoutInterval).Should(ContainElements(ns2.GetName())) + NamespaceIsPartOfTenant(tnt2, ns2).Should(Succeed()) - t := &capsulev1beta2.Tenant{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt2.GetName()}, t)).Should(Succeed()) + Eventually(func(g Gomega) { + t := &capsulev1beta2.Tenant{} + g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt2.GetName()}, t)).Should(Succeed()) - Expect(t.Status.Size).To(Equal(uint(1))) + g.Expect(t.Status.Size).To(Equal(uint(1))) - instance := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{Name: ns2.GetName(), UID: ns2.GetUID()}) - Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") + instance := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{Name: ns2.GetName(), UID: ns2.GetUID()}) + g.Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") - condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") - Expect(instance.Name).To(Equal(ns2.GetName())) - Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected namespace condition status to be True") - Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected namespace condition reason to be Succeeded") + g.Expect(instance.Name).To(Equal(ns2.GetName())) + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected namespace condition status to be True") + g.Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") + g.Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected namespace condition reason to be Succeeded") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("deleting namespace", func() { diff --git a/e2e/config_client_test.go b/e2e/config_client_test.go new file mode 100644 index 00000000..bd14d7ec --- /dev/null +++ b/e2e/config_client_test.go @@ -0,0 +1,90 @@ +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" +) + +var _ = Describe("CapsuleConfiguration - ServiceAccountClient", Ordered, Label("config", "impersonation"), func() { + originalConfig := &capsulev1beta2.CapsuleConfiguration{} + + BeforeEach(func() { + Expect(k8sClient.Get(context.Background(), client.ObjectKey{Name: defaultConfigurationName}, originalConfig)).To(Succeed()) + }) + + AfterEach(func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec = originalConfig.Spec + }) + }) + + It("sets skip TLS verify", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.Impersonation = capsulev1beta2.ServiceAccountClient{ + SkipTLSVerify: true, + } + }) + + capsuleCfg := configuration.NewCapsuleConfiguration(context.TODO(), k8sClient, cfg, defaultConfigurationName) + clientCfg, err := capsuleCfg.ServiceAccountClient(context.TODO()) + Expect(err).NotTo(HaveOccurred()) + Expect(clientCfg.TLSClientConfig.Insecure).To(BeTrue()) + }) + + It("loads CA from secret", func() { + caData := []byte("dummy-ca-data") + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-capsule-ca", + Namespace: "default", + }, + Data: map[string][]byte{ + "ca.crt": caData, + }, + } + Expect(k8sClient.Create(context.TODO(), secret)).To(Succeed()) + + DeferCleanup(func() { + s := &corev1.Secret{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}, s) + if err != nil { + if apierrors.IsNotFound(err) { + return + } + + Expect(err).ToNot(HaveOccurred()) + return + } + + err = k8sClient.Delete(context.TODO(), s) + if err != nil && !apierrors.IsNotFound(err) { + Expect(err).ToNot(HaveOccurred()) + } + }) + + // Create configuration pointing to the secret + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.Impersonation = capsulev1beta2.ServiceAccountClient{ + CASecretName: meta.RFC1123Name(secret.Name), + CASecretNamespace: meta.RFC1123SubdomainName(secret.Namespace), + CASecretKey: "ca.crt", + } + }) + + cfg := configuration.NewCapsuleConfiguration(context.TODO(), k8sClient, cfg, defaultConfigurationName) + clientCfg, err := cfg.ServiceAccountClient(context.TODO()) + Expect(err).NotTo(HaveOccurred()) + Expect(clientCfg.TLSClientConfig.CAData).To(Equal(caData)) + }) +}) diff --git a/e2e/custom_capsule_group_test.go b/e2e/config_custom_capsule_group_test.go similarity index 65% rename from e2e/custom_capsule_group_test.go rename to e2e/config_custom_capsule_group_test.go index 0d672271..8e7735c1 100644 --- a/e2e/custom_capsule_group_test.go +++ b/e2e/config_custom_capsule_group_test.go @@ -12,30 +12,34 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespace as Tenant owner with custom --capsule-group", Label("config"), func() { +var _ = Describe("creating a Namespace as Tenant owner with custom --capsule-group", Ordered, Label("config"), func() { originConfig := &capsulev1beta2.CapsuleConfiguration{} tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-assigned-custom-group", + Name: "e2e-assigned-custom-group", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "alice", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-assigned-custom-group-1", Kind: "User", }, }, }, { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "bob", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-assigned-custom-group-2", Kind: "User", }, }, @@ -51,9 +55,10 @@ var _ = Describe("creating a Namespace as Tenant owner with custom --capsule-gro tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) // Restore Configuration Eventually(func() error { @@ -71,10 +76,12 @@ var _ = Describe("creating a Namespace as Tenant owner with custom --capsule-gro ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.UserNames = []string{} configuration.Spec.UserGroups = []string{} - configuration.Spec.Users = []api.UserSpec{{Kind: api.GroupOwner, Name: "test"}} + configuration.Spec.Users = []rbac.UserSpec{{Kind: rbac.GroupOwner, Name: "test"}} }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) }) @@ -82,37 +89,43 @@ var _ = Describe("creating a Namespace as Tenant owner with custom --capsule-gro ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.UserNames = []string{} configuration.Spec.UserGroups = []string{} - configuration.Spec.Users = []api.UserSpec{{Kind: api.UserOwner, Name: "alice"}, {Kind: api.GroupOwner, Name: "test"}} + configuration.Spec.Users = []rbac.UserSpec{{Kind: rbac.UserOwner, Name: "e2e-assigned-custom-group-1"}, {Kind: rbac.GroupOwner, Name: "test"}} }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) }) It("should succeed and be available in Tenant namespaces list with default single group", func() { ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.UserNames = []string{} configuration.Spec.UserGroups = []string{} - configuration.Spec.Users = []api.UserSpec{{Kind: api.GroupOwner, Name: "projectcapsule.dev"}} + configuration.Spec.Users = []rbac.UserSpec{{Kind: rbac.GroupOwner, Name: "projectcapsule.dev"}} }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) }) It("should fail when group is ignored", func() { ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.UserNames = []string{} configuration.Spec.UserGroups = []string{} - configuration.Spec.Users = []api.UserSpec{{Kind: api.GroupOwner, Name: "projectcapsule.dev"}} + configuration.Spec.Users = []rbac.UserSpec{{Kind: rbac.GroupOwner, Name: "projectcapsule.dev"}} configuration.Spec.IgnoreUserWithGroups = []string{"projectcapsule.dev"} }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) }) @@ -121,11 +134,13 @@ var _ = Describe("creating a Namespace as Tenant owner with custom --capsule-gro ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.UserNames = []string{} configuration.Spec.UserGroups = []string{} - configuration.Spec.Users = []api.UserSpec{{Kind: api.UserOwner, Name: tnt.Spec.Owners[0].Name}} + configuration.Spec.Users = []rbac.UserSpec{{Kind: rbac.UserOwner, Name: tnt.Spec.Owners[0].Name}} configuration.Spec.IgnoreUserWithGroups = []string{} }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) }) @@ -135,25 +150,31 @@ var _ = Describe("creating a Namespace as Tenant owner with custom --capsule-gro configuration.Spec.UserNames = []string{} configuration.Spec.UserGroups = []string{} configuration.Spec.IgnoreUserWithGroups = []string{} - configuration.Spec.Users = []api.UserSpec{{Kind: api.UserOwner, Name: tnt.Spec.Owners[0].Name}} + configuration.Spec.Users = []rbac.UserSpec{{Kind: rbac.UserOwner, Name: tnt.Spec.Owners[0].Name}} }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[1].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + NamespaceIsNotPartOfTenant(tnt, ns).Should(Succeed()) }) It("should fail when group is ignored", func() { ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.UserNames = []string{} configuration.Spec.UserGroups = []string{} - configuration.Spec.Users = []api.UserSpec{{Kind: api.UserOwner, Name: tnt.Spec.Owners[0].Name}} + configuration.Spec.Users = []rbac.UserSpec{{Kind: rbac.UserOwner, Name: tnt.Spec.Owners[0].Name}} configuration.Spec.IgnoreUserWithGroups = []string{"projectcapsule.dev"} }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) - NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + NamespaceCreation(ns, tnt.Spec.Owners[1].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + NamespaceIsNotPartOfTenant(tnt, ns).Should(Succeed()) }) }) diff --git a/e2e/force_tenant_prefix_test.go b/e2e/config_force_tenant_prefix_test.go similarity index 52% rename from e2e/force_tenant_prefix_test.go rename to e2e/config_force_tenant_prefix_test.go index 5b980d4a..226720cd 100644 --- a/e2e/force_tenant_prefix_test.go +++ b/e2e/config_force_tenant_prefix_test.go @@ -11,20 +11,23 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespace with Tenant name prefix enforcement", Label("tenant"), func() { +var _ = Describe("creating a Namespace with Tenant name prefix enforcement", Ordered, Label("tenant", "config", "prefix"), func() { t1 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "awesome", + Name: "e2e-prefix", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "john", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-prefix", Kind: "User", }, }, @@ -34,14 +37,17 @@ var _ = Describe("creating a Namespace with Tenant name prefix enforcement", Lab } t2 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "awesome-tenant", + Name: "e2e-prefix-tenant", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "john", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-prefix", Kind: "User", }, }, @@ -55,42 +61,63 @@ var _ = Describe("creating a Namespace with Tenant name prefix enforcement", Lab t1.ResourceVersion = "" return k8sClient.Create(context.TODO(), t1) }).Should(Succeed()) + TenantReady(t1, metav1.ConditionTrue, defaultTimeoutInterval) + EventuallyCreation(func() error { t2.ResourceVersion = "" return k8sClient.Create(context.TODO(), t2) }).Should(Succeed()) + TenantReady(t2, metav1.ConditionTrue, defaultTimeoutInterval) ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.ForceTenantPrefix = true }) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), t1)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), t2)).Should(Succeed()) + EventuallyDeletion(t1) + EventuallyDeletion(t2) ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.ForceTenantPrefix = false }) }) + It("should fail when non using prefix (Single Tenant)", func() { + EventuallyDeletion(t2) + ns := NewNamespace("custom") + NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + }) + + It("should fail when non using prefix (Single Tenant)", func() { + EventuallyDeletion(t2) + ns := NewNamespace("e2e-prefix") + NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + }) + + It("should succeed using prefix (Single Tenant)", func() { + EventuallyDeletion(t2) + + ns := NewNamespace("e2e-prefix-namespace") + NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + }) + It("should fail when non using prefix", func() { - ns := NewNamespace("awesome") + ns := NewNamespace("e2e-prefix") NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) }) It("should succeed using prefix", func() { - ns := NewNamespace("awesome-namespace") + ns := NewNamespace("e2e-prefix-namespace") NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) }) It("should succeed and assigned according to closest match", func() { - ns1 := NewNamespace("awesome-tenant") - ns2 := NewNamespace("awesome-tenant-namespace") + ns1 := NewNamespace("e2e-prefix-tenant") + ns2 := NewNamespace("e2e-prefix-tenant-namespace") NamespaceCreation(ns1, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns1).Should(Succeed()) NamespaceCreation(ns2, t2.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - - TenantNamespaceList(t1, defaultTimeoutInterval).Should(ContainElement(ns1.GetName())) - TenantNamespaceList(t2, defaultTimeoutInterval).Should(ContainElement(ns2.GetName())) + NamespaceIsPartOfTenant(t2, ns2).Should(Succeed()) }) }) diff --git a/e2e/protected_namespace_regex_test.go b/e2e/config_protected_regex_test.go similarity index 71% rename from e2e/protected_namespace_regex_test.go rename to e2e/config_protected_regex_test.go index ee6482fe..7f0fed9c 100644 --- a/e2e/protected_namespace_regex_test.go +++ b/e2e/config_protected_regex_test.go @@ -12,22 +12,26 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespace with a protected Namespace regex enabled", Label("namespace"), func() { +var _ = Describe("creating a Namespace with a protected Namespace regex enabled", Ordered, Label("config", "namespace"), func() { originConfig := &capsulev1beta2.CapsuleConfiguration{} tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-protected-namespace", + Name: "e2e-protected-namespace", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "alice", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-protected-namespace", Kind: "User", }, }, @@ -43,9 +47,10 @@ var _ = Describe("creating a Namespace with a protected Namespace regex enabled" tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) // Restore Configuration Eventually(func() error { @@ -64,15 +69,20 @@ var _ = Describe("creating a Namespace with a protected Namespace regex enabled" configuration.Spec.ProtectedNamespaceRegexpString = `^.*[-.]system$` }) - ns := NewNamespace("test-ok") + ns := NewNamespace("e2e-protected-namespace-ok", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) }) It("should fail using a value non matching the regex", func() { - ns := NewNamespace("test-system") + ns := NewNamespace("e2e-protected-namespace-system", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.ProtectedNamespaceRegexpString = "" diff --git a/e2e/custom_resource_quota_test.go b/e2e/custom_resource_quota_test.go index 245681f0..d9300a98 100644 --- a/e2e/custom_resource_quota_test.go +++ b/e2e/custom_resource_quota_test.go @@ -19,23 +19,27 @@ import ( "k8s.io/client-go/kubernetes/scheme" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("when Tenant limits custom Resource Quota", Label("resourcequota"), func() { +var _ = Describe("when Tenant limits custom Resource Quota", Ordered, Label("resourcequota"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "limiting-resources", + Name: "e2e-limiting-resources", + Labels: map[string]string{ + "env": "e2e", + }, Annotations: map[string]string{ "quota.resources.capsule.clastix.io/foos.test.clastix.io_v1": "3", }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "resource", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-limiting-resources", Kind: "User", }, }, @@ -93,22 +97,25 @@ var _ = Describe("when Tenant limits custom Resource Quota", Label("resourcequot EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { Expect(k8sClient.Delete(context.TODO(), crd)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should block resources in overflow", func() { dynamicClient := dynamic.NewForConfigOrDie(cfg) for _, i := range []int{1, 2, 3} { - ns := NewNamespace(fmt.Sprintf("limiting-resources-ns-%d", i)) + ns := NewNamespace(fmt.Sprintf("e2e-limiting-resources-ns-%d", i), map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) obj := &unstructured.Unstructured{ Object: map[string]interface{}{ @@ -127,7 +134,9 @@ var _ = Describe("when Tenant limits custom Resource Quota", Label("resourcequot } for _, i := range []int{1, 2, 3} { - ns := NewNamespace(fmt.Sprintf("limiting-resources-ns-%d", i)) + ns := NewNamespace(fmt.Sprintf("limiting-resources-ns-%d", i), map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) obj := &unstructured.Unstructured{ Object: map[string]interface{}{ diff --git a/e2e/customquota_global_test.go b/e2e/customquota_global_test.go new file mode 100644 index 00000000..c4845219 --- /dev/null +++ b/e2e/customquota_global_test.go @@ -0,0 +1,3368 @@ +package e2e + +import ( + "context" + "fmt" + "sort" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes/scheme" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + capmeta "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/gvk" + "github.com/projectcapsule/capsule/pkg/runtime/quota" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +func expectLedgerSettled(ctx context.Context, namespace, name string) { + Eventually(func(g Gomega) { + obj := &capsulev1beta2.QuantityLedger{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: namespace, + }, obj)).To(Succeed(), + "failed to get QuantityLedger %s/%s", + namespace, + name, + ) + + g.Expect(obj.Status.Reserved.IsZero()).To(BeTrue(), + "ledger %s/%s still has reserved=%q reservations=%+v pendingDeletes=%+v", + namespace, + name, + obj.Status.Reserved.String(), + obj.Status.Reservations, + obj.Status.PendingDeletes, + ) + + g.Expect(obj.Status.PendingDeletes).To(BeEmpty(), + "ledger %s/%s still has pendingDeletes=%+v reserved=%q reservations=%+v", + namespace, + name, + obj.Status.PendingDeletes, + obj.Status.Reserved.String(), + obj.Status.Reservations, + ) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func expectGlobalQuotaUsedAndClaims(ctx context.Context, name string, used string, claims int) { + expectedUsed := resource.MustParse(used) + + Eventually(func(g Gomega) { + obj := &capsulev1beta2.GlobalCustomQuota{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, obj)).To(Succeed(), + "failed to get GlobalCustomQuota %s", name) + + g.Expect( + obj.Status.Usage.Used.Cmp(expectedUsed), + ).To(Equal(0), + "unexpected used value for GlobalCustomQuota %s: used=%q expectedUsed=%q available=%q claims=%d expectedClaims=%d", + name, + obj.Status.Usage.Used.String(), + used, + obj.Status.Usage.Available.String(), + len(obj.Status.Claims), + claims, + ) + + g.Expect(obj.Status.Usage.Used.Sign()).To(BeNumerically(">=", 0), + "usage went negative for GlobalCustomQuota %s: used=%q", name, obj.Status.Usage.Used.String()) + + g.Expect(obj.Status.Usage.Available.Sign()).To(BeNumerically(">=", 0), + "available went negative for GlobalCustomQuota %s: available=%q", name, obj.Status.Usage.Available.String()) + + g.Expect(len(obj.Status.Claims)).To(Equal(claims), + "unexpected claims for GlobalCustomQuota %s: used=%q expectedUsed=%q available=%q claims=%d expectedClaims=%d", + name, + obj.Status.Usage.Used.String(), + used, + obj.Status.Usage.Available.String(), + len(obj.Status.Claims), + claims, + ) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func expectCustomQuotaUsedAndClaims(ctx context.Context, namespace, name string, used string, claims int) { + expectedUsed := resource.MustParse(used) + + Eventually(func(g Gomega) { + obj := &capsulev1beta2.CustomQuota{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: namespace, + }, obj)).To(Succeed(), + "failed to get CustomQuota %s/%s", namespace, name) + + g.Expect( + obj.Status.Usage.Used.Cmp(expectedUsed), + ).To(Equal(0), + "unexpected used value for CustomQuota %s/%s: used=%q expectedUsed=%q available=%q claims=%d expectedClaims=%d", + namespace, + name, + obj.Status.Usage.Used.String(), + used, + obj.Status.Usage.Available.String(), + len(obj.Status.Claims), + claims, + ) + + g.Expect(obj.Status.Usage.Used.Sign()).To(BeNumerically(">=", 0), + "usage went negative for CustomQuota %s/%s: used=%q", namespace, name, obj.Status.Usage.Used.String()) + + g.Expect(obj.Status.Usage.Available.Sign()).To(BeNumerically(">=", 0), + "available went negative for CustomQuota %s/%s: available=%q", namespace, name, obj.Status.Usage.Available.String()) + + g.Expect(len(obj.Status.Claims)).To(Equal(claims), + "unexpected claims for CustomQuota %s/%s: used=%q expectedUsed=%q available=%q claims=%d expectedClaims=%d", + namespace, + name, + obj.Status.Usage.Used.String(), + used, + obj.Status.Usage.Available.String(), + len(obj.Status.Claims), + claims, + ) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func awaitGlobalQuotaReady(ctx context.Context, name string) { + Eventually(func(g Gomega) { + gq := &capsulev1beta2.GlobalCustomQuota{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, gq)).To(Succeed()) + + g.Expect(gq.Status.Targets).NotTo(Equal(0)) + + // status should be initialized by the controller + g.Expect(gq.Status.Usage.Used.String()).NotTo(BeEmpty()) + g.Expect(gq.Status.Usage.Available.String()).NotTo(BeEmpty()) + + ledger := &capsulev1beta2.QuantityLedger{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: "capsule-system", + }, ledger)).To(Succeed()) + + g.Expect(capmeta.IsStatusConditionTrue(gq.Status.Conditions, capmeta.ReadyCondition)).To(BeTrue()) + + g.Expect(ledger.Spec.TargetRef.Kind).To(Equal("GlobalCustomQuota")) + g.Expect(ledger.Spec.TargetRef.Name).To(Equal(name)) + + // initial ledger should be settled before test objects are created + g.Expect(ledger.Status.Reserved.IsZero()).To(BeTrue()) + g.Expect(ledger.Status.PendingDeletes).To(BeEmpty()) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func awaitCustomQuotaReady(ctx context.Context, namespace, name string) { + Eventually(func(g Gomega) { + cq := &capsulev1beta2.CustomQuota{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: namespace, + }, cq)).To(Succeed()) + + g.Expect(cq.Status.Usage.Used.String()).NotTo(BeEmpty()) + g.Expect(cq.Status.Usage.Available.String()).NotTo(BeEmpty()) + + ledger := &capsulev1beta2.QuantityLedger{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: namespace, + }, ledger)).To(Succeed()) + + g.Expect(ledger.Spec.TargetRef.Kind).To(Equal("CustomQuota")) + g.Expect(ledger.Spec.TargetRef.Name).To(Equal(name)) + g.Expect(ledger.Spec.TargetRef.Namespace).To(Equal(namespace)) + + g.Expect(ledger.Status.Reserved.IsZero()).To(BeTrue()) + g.Expect(ledger.Status.PendingDeletes).To(BeEmpty()) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func getGlobalQuota(ctx context.Context, name string) *capsulev1beta2.GlobalCustomQuota { + obj := &capsulev1beta2.GlobalCustomQuota{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, obj)).To(Succeed()) + return obj +} + +func getLedger(ctx context.Context, namespace, name string) *capsulev1beta2.QuantityLedger { + obj := &capsulev1beta2.QuantityLedger{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: namespace, + }, obj)).To(Succeed()) + return obj +} + +var _ = Describe("when GlobalCustomQuota uses ledger-backed reconciliation", Ordered, Label("global", "globalcustomquota", "customquota", "ledger"), Ordered, func() { + const ( + testNamespace = "global-custom-quota-e2e-test" + tenantLabel = "e2e.capsule.dev/test-suite" + tenantValue = "global-custom-quota-e2e" + ) + + var ( + ctx context.Context + ns *corev1.Namespace + ) + + awaitAllGlobalQuotasReady := func(names ...string) { + for _, name := range names { + awaitGlobalQuotaReady(ctx, name) + } + } + + expectPodCreationDeniedContaining := func(build func(name string) *corev1.Pod, expected string) { + Eventually(func() string { + name := fmt.Sprintf("denied-%d", time.Now().UnixNano()) + obj := build(name) + + err := k8sClient.Create(ctx, obj) + if err == nil { + _ = k8sClient.Delete(ctx, obj) + return "" + } + + return err.Error() + }, defaultTimeoutInterval, defaultPollInterval).Should(ContainSubstring(expected)) + } + + expectGlobalQuotaWildcardNamespaces := func(name string) { + Eventually(func(g Gomega) { + obj := &capsulev1beta2.GlobalCustomQuota{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, obj)).To(Succeed()) + + g.Expect(obj.Status.Namespaces).To(ContainElement("*"), + "expected wildcard namespace status for %s, got=%v", + name, obj.Status.Namespaces, + ) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + expectGlobalQuotaNamespaces := func(name string, expected ...string) { + Eventually(func(g Gomega) { + obj := &capsulev1beta2.GlobalCustomQuota{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, obj)).To(Succeed()) + + actual := append([]string(nil), obj.Status.Namespaces...) + sort.Strings(actual) + + want := append([]string(nil), expected...) + sort.Strings(want) + + g.Expect(actual).To(Equal(want), + "unexpected status.namespaces for %s: got=%v want=%v", + name, actual, want, + ) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + BeforeAll(func() { + ctx = context.Background() + utilruntime.Must(capsulev1beta2.AddToScheme(scheme.Scheme)) + }) + + BeforeEach(func() { + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + Labels: map[string]string{ + tenantLabel: tenantValue, + "env": "e2e", + }, + }, + } + + EventuallyCreation(func() error { + ns.ResourceVersion = "" + return k8sClient.Create(ctx, ns) + }).Should(Succeed()) + }) + + AfterEach(func() { + ForceDeleteNamespace(ctx, testNamespace) + + req, err := labels.NewRequirement("e2e.capsule.dev/test-suite", selection.Equals, []string{"globalcustomquota-ledger"}) + Expect(err).NotTo(HaveOccurred()) + + var list capsulev1beta2.GlobalCustomQuotaList + Expect(k8sClient.List( + context.TODO(), + &list, + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*req), + }, + )).Should(Succeed()) + + for i := range list.Items { + EventuallyDeletion(&list.Items[i]) + } + }) + + It("aggregates a custom pod quantity path and settles the corresponding ledger", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-pod-cpu-requests", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("500m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, q.GetName()) + + dep := MakeDeployment(testNamespace, "cpu-requests", 2, map[string]string{ + "track": "yes", + }, "100m") + EventuallyCreation(func() error { + dep.ResourceVersion = "" + return k8sClient.Create(ctx, dep) + }).Should(Succeed()) + ExpectPodsForDeployment(ctx, testNamespace, "cpu-requests", 2) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "200m", 2) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + + ScaleDeployment(ctx, testNamespace, "cpu-requests", 4) + ExpectPodsForDeployment(ctx, testNamespace, "cpu-requests", 4) + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "400m", 4) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + + ledger := getLedger(ctx, ControllerNamespace, q.GetName()) + Expect(ledger.Spec.TargetRef.Kind).To(Equal("GlobalCustomQuota")) + Expect(ledger.Spec.TargetRef.Name).To(Equal(q.GetName())) + + gq := getGlobalQuota(ctx, q.GetName()) + Expect(gq.Status.Usage.Used.Cmp(resource.MustParse("400m"))).To(Equal(0)) + }) + + It("marks the quota not ready when an existing matching object has no value at the configured quantity path", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-missing-path-not-ready", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + pod := MakePod(testNamespace, "missing-emptydir-size", nil, nil, "nginx:1.27.0", "", "") + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + + Eventually(func(g Gomega) { + obj := &capsulev1beta2.GlobalCustomQuota{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: q.GetName()}, obj)).To(Succeed()) + + cond := obj.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Message).To(ContainSubstring("did not resolve to any value")) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + It("denies creating a matching object when the configured quantity path resolves to no value", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-missing-path-deny-create", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, q.GetName()) + + Eventually(func() error { + pod := MakePod(testNamespace, "missing-path-denied", nil, nil, "nginx:1.27.0", "", "") + return k8sClient.Create(ctx, pod) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring("did not resolve to any value")), + ) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("remains consistent under concurrent pod creations for a GlobalCustomQuota", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-concurrent-pod-count", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, q.GetName()) + const ( + totalAttempts = 100 + expectedSuccess = 10 + ) + + type result struct { + name string + err error + } + + results := make(chan result, totalAttempts) + + var ( + mu sync.Mutex + succeeded int + successSeen = map[string]struct{}{} + ) + + for i := 0; i < totalAttempts; i++ { + i := i + + go func() { + name := fmt.Sprintf("gq-concurrent-pod-%02d", i) + pod := MakePod( + testNamespace, + name, + map[string]string{"track": "yes"}, + nil, + "nginx:1.27.0", + "", + "", + ) + + Eventually(func() error { + mu.Lock() + done := succeeded >= expectedSuccess + mu.Unlock() + + if done { + return nil + } + + err := k8sClient.Create(ctx, pod) + if err != nil { + return err + } + + mu.Lock() + defer mu.Unlock() + + if _, exists := successSeen[name]; !exists { + successSeen[name] = struct{}{} + succeeded++ + } + + return nil + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + mu.Lock() + _, ok := successSeen[name] + mu.Unlock() + + if ok { + results <- result{name: name, err: nil} + + return + } + + results <- result{name: name, err: fmt.Errorf("not created because expected success count was already reached")} + }() + } + + var failed int + var failedMsgs []string + + for i := 0; i < totalAttempts; i++ { + res := <-results + if res.err != nil { + failed++ + failedMsgs = append(failedMsgs, fmt.Sprintf("%s: %v", res.name, res.err)) + } + } + + mu.Lock() + successNames := make([]string, 0, len(successSeen)) + for name := range successSeen { + successNames = append(successNames, name) + } + succeeded = len(successNames) + mu.Unlock() + + sort.Strings(successNames) + + gq := getGlobalQuota(ctx, q.GetName()) + ledger := getLedger(ctx, ControllerNamespace, q.GetName()) + + Expect(succeeded).To(Equal(expectedSuccess), + "unexpected number of successful concurrent pod creations\nsuccesses=%d\nfailures=%d\nsuccessNames=%v\nfailureDetails=%v\nquotaUsed=%q\nquotaAvailable=%q\nclaims=%d\nledgerReserved=%q\nledgerReservations=%+v\nledgerPendingDeletes=%+v", + succeeded, + failed, + successNames, + failedMsgs, + gq.Status.Usage.Used.String(), + gq.Status.Usage.Available.String(), + len(gq.Status.Claims), + ledger.Status.Reserved.String(), + ledger.Status.Reservations, + ledger.Status.PendingDeletes, + ) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "10", 10) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("uses the smallest matching quota as authoritative while accounting successful pod count in both global and namespaced quotas", func() { + gq := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-mixed-pod-count", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("5"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + cq := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-mixed-pod-count", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gq) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, cq) }).Should(Succeed()) + + awaitGlobalQuotaReady(ctx, gq.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, cq.GetName()) + + pod1 := MakePod(testNamespace, "mixed-count-1", nil, nil, "nginx:1.27.0", "", "") + pod2 := MakePod(testNamespace, "mixed-count-2", nil, nil, "nginx:1.27.0", "", "") + + EventuallyCreation(func() error { + pod1.ResourceVersion = "" + return k8sClient.Create(ctx, pod1) + }).Should(Succeed()) + EventuallyCreation(func() error { + pod2.ResourceVersion = "" + return k8sClient.Create(ctx, pod2) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "2", 2) + expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "2", 2) + expectLedgerSettled(ctx, ControllerNamespace, gq.GetName()) + expectLedgerSettled(ctx, cq.GetNamespace(), cq.GetName()) + + Eventually(func() error { + pod3 := MakePod(testNamespace, "mixed-count-3", nil, nil, "nginx:1.27.0", "", "") + return k8sClient.Create(ctx, pod3) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring(`CustomQuota "cq-mixed-pod-count"`)), + ) + + expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "2", 2) + expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "2", 2) + }) + + It("uses the smallest matching quota as authoritative while accounting successful cpu usage in both global and namespaced quotas", func() { + gq := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-mixed-pod-cpu", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("500m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + cq := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-mixed-pod-cpu", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("200m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gq) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, cq) }).Should(Succeed()) + + awaitGlobalQuotaReady(ctx, gq.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, cq.GetName()) + + pod1 := MakePod(testNamespace, "mixed-cpu-1", nil, nil, "nginx:1.27.0", "100m", "") + pod2 := MakePod(testNamespace, "mixed-cpu-2", nil, nil, "nginx:1.27.0", "100m", "") + + EventuallyCreation(func() error { + pod1.ResourceVersion = "" + return k8sClient.Create(ctx, pod1) + }).Should(Succeed()) + EventuallyCreation(func() error { + pod2.ResourceVersion = "" + return k8sClient.Create(ctx, pod2) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "200m", 2) + expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "200m", 2) + expectLedgerSettled(ctx, ControllerNamespace, gq.GetName()) + expectLedgerSettled(ctx, cq.GetNamespace(), cq.GetName()) + + Eventually(func() error { + pod3 := MakePod(testNamespace, "mixed-cpu-3", nil, nil, "nginx:1.27.0", "100m", "") + return k8sClient.Create(ctx, pod3) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring(`CustomQuota "cq-mixed-pod-cpu"`)), + ) + }) + + It("accounts only the matching subset for overlapping global and namespaced quota selectors", func() { + gq := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-mixed-overlap", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + cq := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-mixed-overlap", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + "tier": "frontend", + }, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gq) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, cq) }).Should(Succeed()) + + awaitGlobalQuotaReady(ctx, gq.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, cq.GetName()) + + p1 := MakePod(testNamespace, "mixed-overlap-1", map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") + p2 := MakePod(testNamespace, "mixed-overlap-2", map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") + p3 := MakePod(testNamespace, "mixed-overlap-3", map[string]string{"track": "yes", "tier": "backend"}, nil, "nginx:1.27.0", "", "") + + EventuallyCreation(func() error { p1.ResourceVersion = ""; return k8sClient.Create(ctx, p1) }).Should(Succeed()) + EventuallyCreation(func() error { p2.ResourceVersion = ""; return k8sClient.Create(ctx, p2) }).Should(Succeed()) + EventuallyCreation(func() error { p3.ResourceVersion = ""; return k8sClient.Create(ctx, p3) }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "3", 3) + expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "2", 2) + expectLedgerSettled(ctx, ControllerNamespace, gq.GetName()) + expectLedgerSettled(ctx, cq.GetNamespace(), cq.GetName()) + + Eventually(func() error { + p4 := MakePod(testNamespace, "mixed-overlap-4", map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") + return k8sClient.Create(ctx, p4) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring(`CustomQuota "cq-mixed-overlap"`)), + ) + }) + + It("tracks different paths independently when global and namespaced quotas match the same pod gvk", func() { + gq := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-mixed-path-cpu", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("500m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + cq := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-mixed-path-emptydir", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gq) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, cq) }).Should(Succeed()) + + awaitGlobalQuotaReady(ctx, gq.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, cq.GetName()) + + p1 := MakePod(testNamespace, "mixed-path-1", nil, nil, "nginx:1.27.0", "100m", "1Gi") + p2 := MakePod(testNamespace, "mixed-path-2", nil, nil, "nginx:1.27.0", "100m", "1Gi") + + EventuallyCreation(func() error { p1.ResourceVersion = ""; return k8sClient.Create(ctx, p1) }).Should(Succeed()) + EventuallyCreation(func() error { p2.ResourceVersion = ""; return k8sClient.Create(ctx, p2) }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "200m", 2) + expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "2Gi", 2) + expectLedgerSettled(ctx, ControllerNamespace, gq.GetName()) + expectLedgerSettled(ctx, cq.GetNamespace(), cq.GetName()) + + Eventually(func() error { + p3 := MakePod(testNamespace, "mixed-path-3", nil, nil, "nginx:1.27.0", "100m", "1Gi") + return k8sClient.Create(ctx, p3) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring(`CustomQuota "cq-mixed-path-emptydir"`)), + ) + }) + + It("accounts deployment scaling in both global and namespaced quotas and denies when the smaller namespaced quota is exceeded", func() { + gq := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-mixed-scale", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + cq := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-mixed-scale", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("3"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gq) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, cq) }).Should(Succeed()) + + awaitGlobalQuotaReady(ctx, gq.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, cq.GetName()) + + dep := MakeDeployment(testNamespace, "mixed-scale", 1, nil, "") + EventuallyCreation(func() error { + dep.ResourceVersion = "" + return k8sClient.Create(ctx, dep) + }).Should(Succeed()) + ExpectPodsForDeployment(ctx, testNamespace, "mixed-scale", 1) + + expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "1", 1) + expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "1", 1) + + ScaleDeployment(ctx, testNamespace, "mixed-scale", 3) + ExpectPodsForDeployment(ctx, testNamespace, "mixed-scale", 3) + expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "3", 3) + expectCustomQuotaUsedAndClaims(ctx, cq.GetNamespace(), cq.GetName(), "3", 3) + + ScaleDeployment(ctx, testNamespace, "mixed-scale", 4) + ExpectPodsForDeployment(ctx, testNamespace, "mixed-scale", 3) + + Eventually(func(g Gomega) { + obj := &capsulev1beta2.CustomQuota{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: cq.GetName(), Namespace: testNamespace}, obj)).To(Succeed()) + g.Expect(obj.Status.Usage.Used.Cmp(resource.MustParse("3"))).To(Equal(0)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + It("clamps usage to zero for a pure subtraction source", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-sub-only-clamps-zero", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.resources.requests.storage", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, q.GetName()) + + pvc := MakePVC(testNamespace, "sub-only-pvc", "2Gi") + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 1) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("subtracts matching pvc storage from added pod emptyDir storage", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-add-sub-storage", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.resources.requests.storage", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, q.GetName()) + + pod := MakePod(testNamespace, "add-sub-pod", nil, nil, "nginx:1.27.0", "", "3Gi") + pvc := MakePVC(testNamespace, "add-sub-pvc", "1Gi") + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "2Gi", 2) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("clamps mixed add and subtraction result to zero when subtraction exceeds additions", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-add-sub-clamp-zero", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.resources.requests.storage", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, q.GetName()) + + pod := MakePod(testNamespace, "add-sub-clamp-pod", nil, nil, "nginx:1.27.0", "", "1Gi") + pvc := MakePVC(testNamespace, "add-sub-clamp-pvc", "2Gi") + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 2) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("supports subtraction with label selectors and removes the subtraction when the object no longer matches", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-sub-label-selector", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.resources.requests.storage", + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "discount": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, q.GetName()) + + pod := MakePod(testNamespace, "sub-label-pod", nil, nil, "nginx:1.27.0", "", "3Gi") + pvc := MakePVC(testNamespace, "sub-label-pvc", "1Gi") + pvc.Labels = map[string]string{"discount": "yes"} + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "2Gi", 2) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + + Eventually(func() error { + obj := &corev1.PersistentVolumeClaim{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace}, obj); err != nil { + return err + } + obj.Labels = map[string]string{"discount": "no"} + return k8sClient.Update(ctx, obj) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3Gi", 1) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("reconciles subtraction correctly when the subtracting resource is deleted", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-sub-delete-reconcile", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.resources.requests.storage", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, q.GetName()) + + pod := MakePod(testNamespace, "sub-delete-pod", nil, nil, "nginx:1.27.0", "", "3Gi") + pvc := MakePVC(testNamespace, "sub-delete-pvc", "1Gi") + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "2Gi", 2) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + + EventuallyDeletion(pvc) + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3Gi", 1) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("subtracts cpu requests from counted pod usage across multiple matching pods and clamps at zero", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-sub-cpu-clamp", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, q.GetName()) + + pod1 := MakePod(testNamespace, "sub-cpu-pod-1", nil, nil, "nginx:1.27.0", "500m", "") + pod2 := MakePod(testNamespace, "sub-cpu-pod-2", nil, nil, "nginx:1.27.0", "500m", "") + + EventuallyCreation(func() error { + pod1.ResourceVersion = "" + return k8sClient.Create(ctx, pod1) + }).Should(Succeed()) + EventuallyCreation(func() error { + pod2.ResourceVersion = "" + return k8sClient.Create(ctx, pod2) + }).Should(Succeed()) + + // 2 - 1.0 = 1 + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 2) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("applies subtraction while scaling a deployment", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-sub-deployment-scale", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, q.GetName()) + + dep := MakeDeployment(testNamespace, "sub-scale", 2, nil, "250m") + EventuallyCreation(func() error { + dep.ResourceVersion = "" + return k8sClient.Create(ctx, dep) + }).Should(Succeed()) + ExpectPodsForDeployment(ctx, testNamespace, "sub-scale", 2) + + // 2 - 0.5 = 1.5 + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1500m", 2) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + + ScaleDeployment(ctx, testNamespace, "sub-scale", 4) + ExpectPodsForDeployment(ctx, testNamespace, "sub-scale", 4) + + // 4 - 1.0 = 3 + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3", 4) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("uses the smallest matching quota as authoritative even when both quotas use subtraction", func() { + small := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-sub-small", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + large := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-sub-large", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("5"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, small) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, large) }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, small.GetName()) + awaitGlobalQuotaReady(ctx, large.GetName()) + + pod1 := MakePod(testNamespace, "sub-auth-1", nil, nil, "nginx:1.27.0", "500m", "") + pod2 := MakePod(testNamespace, "sub-auth-2", nil, nil, "nginx:1.27.0", "500m", "") + pod3 := MakePod(testNamespace, "sub-auth-3", nil, nil, "nginx:1.27.0", "500m", "") + + EventuallyCreation(func() error { + pod1.ResourceVersion = "" + return k8sClient.Create(ctx, pod1) + }).Should(Succeed()) + EventuallyCreation(func() error { + pod2.ResourceVersion = "" + return k8sClient.Create(ctx, pod2) + }).Should(Succeed()) + + // 2 - 1.0 = 1 + expectGlobalQuotaUsedAndClaims(ctx, small.GetName(), "1", 2) + expectGlobalQuotaUsedAndClaims(ctx, large.GetName(), "1", 2) + expectLedgerSettled(ctx, ControllerNamespace, small.GetName()) + expectLedgerSettled(ctx, ControllerNamespace, large.GetName()) + + Eventually(func() error { + pod3.ResourceVersion = "" + return k8sClient.Create(ctx, pod3) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + // 3 - 1.5 = 1.5 still fits into both limits, so one more to push smaller first + expectGlobalQuotaUsedAndClaims(ctx, small.GetName(), "1500m", 3) + expectGlobalQuotaUsedAndClaims(ctx, large.GetName(), "1500m", 3) + + Eventually(func() error { + p := MakePod(testNamespace, "sub-auth-4", nil, nil, "nginx:1.27.0", "500m", "") + return k8sClient.Create(ctx, p) + }, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring(`GlobalCustomQuota "gq-sub-small"`))) + }) + + It("posts wildcard namespace status when no namespaceSelectors are configured and keeps it stable as namespaces change", func() { + quota := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-nsstatus-all", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + }, + } + + extraA := NewNamespace("gq-nsstatus-all-a", map[string]string{"purpose": "e2e"}) + extraB := NewNamespace("gq-nsstatus-all-b", map[string]string{"purpose": "e2e"}) + + EventuallyCreation(func() error { return k8sClient.Create(ctx, quota) }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, quota.GetName()) + + expectGlobalQuotaWildcardNamespaces(quota.GetName()) + + NamespaceDeletionAdmin(extraA, defaultTimeoutInterval) + expectGlobalQuotaWildcardNamespaces(quota.GetName()) + + NamespaceDeletionAdmin(extraB, defaultTimeoutInterval) + expectGlobalQuotaWildcardNamespaces(quota.GetName()) + + NamespaceDeletionAdmin(extraA, defaultTimeoutInterval) + expectGlobalQuotaWildcardNamespaces(quota.GetName()) + + NamespaceDeletionAdmin(extraB, defaultTimeoutInterval) + expectGlobalQuotaWildcardNamespaces(quota.GetName()) + }) + + It("posts all namespaces matched by multiple namespaceSelectors and updates status on namespace create and delete", func() { + nsA1 := NewNamespace("gq-nsstatus-a1", map[string]string{"team": "a"}) + nsA2 := NewNamespace("gq-nsstatus-a2", map[string]string{"team": "a"}) + nsB1 := NewNamespace("gq-nsstatus-b1", map[string]string{"team": "b"}) + nsOther := NewNamespace("gq-nsstatus-other", map[string]string{"team": "other"}) + + NamespaceCreationAdmin(nsA1, defaultTimeoutInterval).Should(Succeed()) + NamespaceCreationAdmin(nsB1, defaultTimeoutInterval).Should(Succeed()) + NamespaceCreationAdmin(nsOther, defaultTimeoutInterval).Should(Succeed()) + + quota := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-nsstatus-multi-selectors", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "team": "a", + }, + }, + }, + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "team": "b", + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, quota) }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, quota.GetName()) + + expectGlobalQuotaNamespaces(quota.GetName(), + "gq-nsstatus-a1", + "gq-nsstatus-b1", + ) + + NamespaceCreationAdmin(nsA2, defaultTimeoutInterval).Should(Succeed()) + expectGlobalQuotaNamespaces(quota.GetName(), + "gq-nsstatus-a1", + "gq-nsstatus-a2", + "gq-nsstatus-b1", + ) + + NamespaceDeletionAdmin(nsB1, defaultTimeoutInterval).Should(Succeed()) + expectGlobalQuotaNamespaces(quota.GetName(), + "gq-nsstatus-a1", + "gq-nsstatus-a2", + ) + }) + + It("posts an empty namespace status when namespaceSelectors match no namespaces and updates when matches appear or disappear", func() { + quota := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-nsstatus-empty", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "team": "does-not-exist", + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, quota) }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, quota.GetName()) + + expectGlobalQuotaNamespaces(quota.GetName()) + + ns := NewNamespace("gq-nsstatus-empty-match", map[string]string{"team": "does-not-exist"}) + NamespaceCreationAdmin(ns, defaultTimeoutInterval).Should(Succeed()) + + expectGlobalQuotaNamespaces(quota.GetName(), "gq-nsstatus-empty-match") + + NamespaceDeletionAdmin(ns, defaultTimeoutInterval).Should(Succeed()) + expectGlobalQuotaNamespaces(quota.GetName()) + }) + + It("does not produce negative usage when a matching pod is relabeled to no longer match", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-no-negative-on-relabel", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, q.GetName()) + + pod := MakePod(testNamespace, "no-negative-on-relabel", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "") + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) + + UpdatePodLabels(ctx, testNamespace, "no-negative-on-relabel", map[string]string{"track": "no"}) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + + Eventually(func(g Gomega) { + obj := &capsulev1beta2.GlobalCustomQuota{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: q.GetName()}, obj)).To(Succeed()) + g.Expect(obj.Status.Usage.Used.Sign()).To(BeNumerically(">=", 0)) + g.Expect(obj.Status.Usage.Used.Cmp(resource.MustParse("0"))).To(Equal(0)) + g.Expect(len(obj.Status.Claims)).To(Equal(0)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + It("counts pods correctly while scaling a deployment", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-pod-count", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("5"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitGlobalQuotaReady(ctx, q.GetName()) + + dep := MakeDeployment(testNamespace, "counted", 1, nil, "") + EventuallyCreation(func() error { + dep.ResourceVersion = "" + return k8sClient.Create(ctx, dep) + }).Should(Succeed()) + ExpectPodsForDeployment(ctx, testNamespace, "counted", 1) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) + + ScaleDeployment(ctx, testNamespace, "counted", 3) + ExpectPodsForDeployment(ctx, testNamespace, "counted", 3) + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3", 3) + + ScaleDeployment(ctx, testNamespace, "counted", 2) + ExpectPodsForDeployment(ctx, testNamespace, "counted", 2) + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "2", 2) + + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("tracks count with a single MatchLabels selector and updates when the pod no longer matches", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-count-single-matchlabel", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + + pod := MakePod(testNamespace, "single-matchlabel", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "") + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) + + UpdatePodLabels(ctx, testNamespace, "single-matchlabel", map[string]string{"track": "no"}) + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("tracks count with multiple MatchLabels and updates when the pod no longer matches", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-count-multi-matchlabel", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + "tier": "frontend", + }, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + + pod := MakePod(testNamespace, "multi-matchlabel", map[string]string{ + "track": "yes", + "tier": "frontend", + }, nil, "nginx:1.27.0", "", "") + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) + + UpdatePodLabels(ctx, testNamespace, "multi-matchlabel", map[string]string{ + "track": "yes", + "tier": "backend", + }) + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("tracks count with a single field selector and updates when the pod no longer matches", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-count-single-fieldselector", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{ + `.spec.containers[?(@.image=="nginx:1.27.0")]`, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + + pod := MakePod(testNamespace, "single-fieldselector", nil, nil, "nginx:1.27.0", "", "") + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) + + UpdatePodImage(ctx, testNamespace, "single-fieldselector", "nginx:1.26.0") + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("tracks count with multiple field selectors and updates when the pod no longer matches", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-count-multi-fieldselector", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{ + `.spec.containers[?(@.image=="nginx:1.27.0")]`, + `.spec.containers[?(@.name=="main")]`, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + + pod := MakePod(testNamespace, "multi-fieldselector", nil, nil, "nginx:1.27.0", "", "") + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) + + UpdatePodImage(ctx, testNamespace, "multi-fieldselector", "nginx:1.26.0") + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("aggregates multiple sources across pod emptyDir size and pvc storage size", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-multi-source-storage", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("3Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Path: ".spec.volumes[*].emptyDir.sizeLimit", + Operation: quota.OpAdd, + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Path: ".spec.resources.requests.storage", + Operation: quota.OpAdd, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + + pod := MakePod(testNamespace, "multi-source-pod", nil, nil, "nginx:1.27.0", "", "1Gi") + pvc := MakePVC(testNamespace, "multi-source-pvc", "2Gi") + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3Gi", 2) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("tracks count with multiple field selectors and updates when the pod no longer matches", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-count-multi-fieldselector", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{ + `.spec.containers[?(@.image=="nginx:1.27.0")]`, + `.spec.containers[?(@.name=="main")]`, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + + pod := MakePod(testNamespace, "multi-fieldselector", nil, nil, "nginx:1.27.0", "", "") + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "1", 1) + + UpdatePodImage(ctx, testNamespace, "multi-fieldselector", "nginx:1.26.0") + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("aggregates multiple sources with selectors across pod emptyDir and pvc storage", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-multi-source-selectors-storage", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Path: ".spec.volumes[*].emptyDir.sizeLimit", + Operation: quota.OpAdd, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Path: ".spec.resources.requests.storage", + Operation: quota.OpAdd, + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{ + `.spec.accessModes[?(@=="ReadWriteOnce")]`, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + + matchingPod := MakePod(testNamespace, "matching-emptydir", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "1Gi") + nonMatchingPod := MakePod(testNamespace, "ignored-emptydir", map[string]string{"track": "no"}, nil, "nginx:1.27.0", "", "5Gi") + + matchingPVC := MakePVC(testNamespace, "matching-pvc", "2Gi") + nonMatchingPVC := MakePVC(testNamespace, "ignored-pvc", "4Gi") + nonMatchingPVC.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + + EventuallyCreation(func() error { + matchingPod.ResourceVersion = "" + return k8sClient.Create(ctx, matchingPod) + }).Should(Succeed()) + + EventuallyCreation(func() error { + nonMatchingPod.ResourceVersion = "" + return k8sClient.Create(ctx, nonMatchingPod) + }).Should(Succeed()) + + EventuallyCreation(func() error { + matchingPVC.ResourceVersion = "" + return k8sClient.Create(ctx, matchingPVC) + }).Should(Succeed()) + + EventuallyCreation(func() error { + nonMatchingPVC.ResourceVersion = "" + return k8sClient.Create(ctx, nonMatchingPVC) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3Gi", 2) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("reconciles multiple sources when objects stop matching or are deleted", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-multi-source-reconcile", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Path: ".spec.volumes[*].emptyDir.sizeLimit", + Operation: quota.OpAdd, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Path: ".spec.resources.requests.storage", + Operation: quota.OpAdd, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + + pod := MakePod(testNamespace, "reconcile-emptydir", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "1Gi") + pvc := MakePVC(testNamespace, "reconcile-pvc", "2Gi") + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "3Gi", 2) + + UpdatePodLabels(ctx, testNamespace, "reconcile-emptydir", map[string]string{"track": "no"}) + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "2Gi", 1) + + EventuallyDeletion(pvc) + expectGlobalQuotaUsedAndClaims(ctx, q.GetName(), "0", 0) + expectLedgerSettled(ctx, ControllerNamespace, q.GetName()) + }) + + It("rejects admission when a field selector uses an invalid jsonpath filter on a scalar", func() { + q := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-invalid-fieldselector", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{ + `.spec.restartPolicy[?(@=="Always")]`, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + + pod := MakePod(testNamespace, "invalid-selector-pod", nil, nil, "nginx:1.27.0", "", "") + + Eventually(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring("is not array or slice and cannot be filtered")), + ) + }) + + It("uses the smallest matching global quota as authoritative while accounting usage in all matching quotas for pod count", func() { + small := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-pod-count-small", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + large := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-pod-count-large", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("5"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, small) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, large) }).Should(Succeed()) + + awaitAllGlobalQuotasReady(small.GetName(), large.GetName()) + + pod1 := MakePod(testNamespace, "multi-gq-count-1", nil, nil, "nginx:1.27.0", "", "") + pod2 := MakePod(testNamespace, "multi-gq-count-2", nil, nil, "nginx:1.27.0", "", "") + + EventuallyCreation(func() error { + pod1.ResourceVersion = "" + return k8sClient.Create(ctx, pod1) + }).Should(Succeed()) + EventuallyCreation(func() error { + pod2.ResourceVersion = "" + return k8sClient.Create(ctx, pod2) + }).Should(Succeed()) + + expectLedgerSettled(ctx, ControllerNamespace, small.GetName()) + expectLedgerSettled(ctx, ControllerNamespace, large.GetName()) + expectGlobalQuotaUsedAndClaims(ctx, small.GetName(), "2", 2) + expectGlobalQuotaUsedAndClaims(ctx, large.GetName(), "2", 2) + + expectPodCreationDeniedContaining(func(name string) *corev1.Pod { + return MakePod(testNamespace, name, nil, nil, "nginx:1.27.0", "", "") + }, `GlobalCustomQuota "gq-pod-count-small"`) + + expectGlobalQuotaUsedAndClaims(ctx, small.GetName(), "2", 2) + expectGlobalQuotaUsedAndClaims(ctx, large.GetName(), "2", 2) + }) + + It("uses the smallest matching global quota as authoritative while accounting usage in all matching quotas for pod cpu requests", func() { + small := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-pod-cpu-small", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("200m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + large := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-pod-cpu-large", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("500m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, small) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, large) }).Should(Succeed()) + + awaitAllGlobalQuotasReady(small.GetName(), large.GetName()) + + pod1 := MakePod(testNamespace, "multi-gq-cpu-1", nil, nil, "nginx:1.27.0", "100m", "") + pod2 := MakePod(testNamespace, "multi-gq-cpu-2", nil, nil, "nginx:1.27.0", "100m", "") + + EventuallyCreation(func() error { + pod1.ResourceVersion = "" + return k8sClient.Create(ctx, pod1) + }).Should(Succeed()) + EventuallyCreation(func() error { + pod2.ResourceVersion = "" + return k8sClient.Create(ctx, pod2) + }).Should(Succeed()) + + expectLedgerSettled(ctx, ControllerNamespace, small.GetName()) + expectLedgerSettled(ctx, ControllerNamespace, large.GetName()) + expectGlobalQuotaUsedAndClaims(ctx, small.GetName(), "200m", 2) + expectGlobalQuotaUsedAndClaims(ctx, large.GetName(), "200m", 2) + + expectPodCreationDeniedContaining(func(name string) *corev1.Pod { + return MakePod(testNamespace, name, nil, nil, "nginx:1.27.0", "100m", "") + }, `GlobalCustomQuota "gq-pod-cpu-small"`) + + expectGlobalQuotaUsedAndClaims(ctx, small.GetName(), "200m", 2) + expectGlobalQuotaUsedAndClaims(ctx, large.GetName(), "200m", 2) + }) + + It("accounts only the matching subset for overlapping selectors on the same pod gvk", func() { + + broad := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-pod-track-broad", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("5"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + narrow := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-pod-track-frontend", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + "tier": "frontend", + }, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, broad) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, narrow) }).Should(Succeed()) + + awaitAllGlobalQuotasReady(narrow.GetName(), broad.GetName()) + + pod1 := MakePod(testNamespace, "track-frontend-1", map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") + pod2 := MakePod(testNamespace, "track-frontend-2", map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") + pod3 := MakePod(testNamespace, "track-backend-1", map[string]string{"track": "yes", "tier": "backend"}, nil, "nginx:1.27.0", "", "") + pod4 := MakePod(testNamespace, "track-backend-2", map[string]string{"track": "yes", "tier": "backend"}, nil, "nginx:1.27.0", "", "") + + EventuallyCreation(func() error { pod1.ResourceVersion = ""; return k8sClient.Create(ctx, pod1) }).Should(Succeed()) + EventuallyCreation(func() error { pod2.ResourceVersion = ""; return k8sClient.Create(ctx, pod2) }).Should(Succeed()) + EventuallyCreation(func() error { pod3.ResourceVersion = ""; return k8sClient.Create(ctx, pod3) }).Should(Succeed()) + EventuallyCreation(func() error { pod4.ResourceVersion = ""; return k8sClient.Create(ctx, pod4) }).Should(Succeed()) + + expectLedgerSettled(ctx, ControllerNamespace, broad.GetName()) + expectLedgerSettled(ctx, ControllerNamespace, narrow.GetName()) + expectGlobalQuotaUsedAndClaims(ctx, broad.GetName(), "4", 4) + expectGlobalQuotaUsedAndClaims(ctx, narrow.GetName(), "2", 2) + + expectPodCreationDeniedContaining(func(name string) *corev1.Pod { + return MakePod(testNamespace, name, map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") + }, `GlobalCustomQuota "gq-pod-track-frontend"`) + + expectGlobalQuotaUsedAndClaims(ctx, broad.GetName(), "4", 4) + expectGlobalQuotaUsedAndClaims(ctx, narrow.GetName(), "2", 2) + + EventuallyCreation(func() error { + pod := MakePod(testNamespace, "track-backend-3", map[string]string{"track": "yes", "tier": "backend"}, nil, "nginx:1.27.0", "", "") + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectLedgerSettled(ctx, ControllerNamespace, broad.GetName()) + expectLedgerSettled(ctx, ControllerNamespace, narrow.GetName()) + expectGlobalQuotaUsedAndClaims(ctx, broad.GetName(), "5", 5) + expectGlobalQuotaUsedAndClaims(ctx, narrow.GetName(), "2", 2) + }) + + It("tracks different paths independently when multiple global quotas match the same pod gvk", func() { + cpuQuota := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-pod-path-cpu", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("400m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + emptyDirQuota := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-pod-path-emptydir", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, cpuQuota) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, emptyDirQuota) }).Should(Succeed()) + + awaitAllGlobalQuotasReady(cpuQuota.GetName(), emptyDirQuota.GetName()) + + pod1 := MakePod(testNamespace, "path-pod-1", nil, nil, "nginx:1.27.0", "100m", "1Gi") + pod2 := MakePod(testNamespace, "path-pod-2", nil, nil, "nginx:1.27.0", "100m", "1Gi") + + EventuallyCreation(func() error { pod1.ResourceVersion = ""; return k8sClient.Create(ctx, pod1) }).Should(Succeed()) + EventuallyCreation(func() error { pod2.ResourceVersion = ""; return k8sClient.Create(ctx, pod2) }).Should(Succeed()) + + expectLedgerSettled(ctx, ControllerNamespace, cpuQuota.GetName()) + expectLedgerSettled(ctx, ControllerNamespace, emptyDirQuota.GetName()) + expectGlobalQuotaUsedAndClaims(ctx, cpuQuota.GetName(), "200m", 2) + expectGlobalQuotaUsedAndClaims(ctx, emptyDirQuota.GetName(), "2Gi", 2) + + expectPodCreationDeniedContaining(func(name string) *corev1.Pod { + return MakePod(testNamespace, name, nil, nil, "nginx:1.27.0", "100m", "1Gi") + }, `GlobalCustomQuota "gq-pod-path-emptydir"`) + + expectGlobalQuotaUsedAndClaims(ctx, cpuQuota.GetName(), "200m", 2) + expectGlobalQuotaUsedAndClaims(ctx, emptyDirQuota.GetName(), "2Gi", 2) + }) + + It("accounts only the quotas that actually match when multiple global quotas share the same gvk", func() { + labelQuota := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-pod-track-only", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + fieldQuota := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-pod-nginx-only", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{ + `.spec.containers[?(@.image=="nginx:1.27.0")]`, + }, + }, + }, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, labelQuota) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, fieldQuota) }).Should(Succeed()) + + awaitAllGlobalQuotasReady(labelQuota.GetName(), fieldQuota.GetName()) + + matchBoth := MakePod(testNamespace, "subset-both", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "") + matchLabelOnly := MakePod(testNamespace, "subset-label-only", map[string]string{"track": "yes"}, nil, "busybox:1.36.1", "", "") + matchFieldOnly := MakePod(testNamespace, "subset-field-only", nil, nil, "nginx:1.27.0", "", "") + + EventuallyCreation(func() error { matchBoth.ResourceVersion = ""; return k8sClient.Create(ctx, matchBoth) }).Should(Succeed()) + EventuallyCreation(func() error { matchLabelOnly.ResourceVersion = ""; return k8sClient.Create(ctx, matchLabelOnly) }).Should(Succeed()) + EventuallyCreation(func() error { matchFieldOnly.ResourceVersion = ""; return k8sClient.Create(ctx, matchFieldOnly) }).Should(Succeed()) + + expectLedgerSettled(ctx, ControllerNamespace, labelQuota.GetName()) + expectLedgerSettled(ctx, ControllerNamespace, fieldQuota.GetName()) + expectGlobalQuotaUsedAndClaims(ctx, labelQuota.GetName(), "2", 2) + expectGlobalQuotaUsedAndClaims(ctx, fieldQuota.GetName(), "2", 2) + }) + + It("uses deterministic tie-breaking when multiple global quotas have the same remaining availability", func() { + quotaA := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-tie-a", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("3"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + quotaB := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-tie-b", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("4"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, quotaA) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, quotaB) }).Should(Succeed()) + + awaitAllGlobalQuotasReady(quotaA.GetName(), quotaB.GetName()) + + // Drive them to equal remaining availability: + // gq-tie-a limit 3, used 1 => available 2 + // gq-tie-b limit 4, used 2 => available 2 + pod1 := MakePod(testNamespace, "tie-1", nil, nil, "nginx:1.27.0", "", "") + pod2 := MakePod(testNamespace, "tie-2", nil, nil, "nginx:1.27.0", "", "") + + EventuallyCreation(func() error { pod1.ResourceVersion = ""; return k8sClient.Create(ctx, pod1) }).Should(Succeed()) + EventuallyCreation(func() error { pod2.ResourceVersion = ""; return k8sClient.Create(ctx, pod2) }).Should(Succeed()) + + expectLedgerSettled(ctx, ControllerNamespace, quotaA.GetName()) + expectLedgerSettled(ctx, ControllerNamespace, quotaB.GetName()) + expectGlobalQuotaUsedAndClaims(ctx, quotaA.GetName(), "2", 2) + expectGlobalQuotaUsedAndClaims(ctx, quotaB.GetName(), "2", 2) + + // Next pod is still allowed, because both have 1 and 2 available respectively after pod3. + pod3 := MakePod(testNamespace, "tie-3", nil, nil, "nginx:1.27.0", "", "") + EventuallyCreation(func() error { pod3.ResourceVersion = ""; return k8sClient.Create(ctx, pod3) }).Should(Succeed()) + + expectLedgerSettled(ctx, ControllerNamespace, quotaA.GetName()) + expectLedgerSettled(ctx, ControllerNamespace, quotaB.GetName()) + expectGlobalQuotaUsedAndClaims(ctx, quotaA.GetName(), "3", 3) + expectGlobalQuotaUsedAndClaims(ctx, quotaB.GetName(), "3", 3) + + // Now gq-tie-a is exhausted first and should be authoritative. + expectPodCreationDeniedContaining(func(name string) *corev1.Pod { + return MakePod(testNamespace, name, nil, nil, "nginx:1.27.0", "", "") + }, `GlobalCustomQuota "gq-tie-a"`) + + expectGlobalQuotaUsedAndClaims(ctx, quotaA.GetName(), "3", 3) + expectGlobalQuotaUsedAndClaims(ctx, quotaB.GetName(), "3", 3) + }) + + It("aggregates the same successful pod into multiple quotas with different paths on the same gvk", func() { + cpuQuota := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-multi-path-cpu", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("300m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + emptyDirQuota := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-multi-path-emptydir", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "globalcustomquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("3Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, cpuQuota) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, emptyDirQuota) }).Should(Succeed()) + + awaitAllGlobalQuotasReady(cpuQuota.GetName(), emptyDirQuota.GetName()) + + pod := MakePod(testNamespace, "multi-path-shared-1", nil, nil, "nginx:1.27.0", "100m", "1Gi") + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectLedgerSettled(ctx, ControllerNamespace, cpuQuota.GetName()) + expectLedgerSettled(ctx, ControllerNamespace, emptyDirQuota.GetName()) + expectGlobalQuotaUsedAndClaims(ctx, cpuQuota.GetName(), "100m", 1) + expectGlobalQuotaUsedAndClaims(ctx, emptyDirQuota.GetName(), "1Gi", 1) + + pod2 := MakePod(testNamespace, "multi-path-shared-2", nil, nil, "nginx:1.27.0", "100m", "1Gi") + EventuallyCreation(func() error { + pod2.ResourceVersion = "" + return k8sClient.Create(ctx, pod2) + }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, cpuQuota.GetName(), "200m", 2) + expectGlobalQuotaUsedAndClaims(ctx, emptyDirQuota.GetName(), "2Gi", 2) + }) +}) diff --git a/e2e/customquota_namespaced_test.go b/e2e/customquota_namespaced_test.go new file mode 100644 index 00000000..cbba61bb --- /dev/null +++ b/e2e/customquota_namespaced_test.go @@ -0,0 +1,2226 @@ +package e2e + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/gvk" + "github.com/projectcapsule/capsule/pkg/runtime/quota" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes/scheme" +) + +var _ = Describe("when CustomQuota uses ledger-backed reconciliation", Ordered, Label("namespaced", "namespacedcustomquota", "customquota", "ledger"), Ordered, func() { + const ( + testNamespace = "custom-quota-e2e-test" + tenantLabel = "e2e.capsule.dev/test-suite" + tenantValue = "custom-quota-e2e" + ) + + var ( + ctx context.Context + ns *corev1.Namespace + ) + + BeforeAll(func() { + ctx = context.Background() + utilruntime.Must(capsulev1beta2.AddToScheme(scheme.Scheme)) + }) + + BeforeEach(func() { + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + Labels: map[string]string{ + tenantLabel: tenantValue, + "env": "e2e", + }, + }, + } + + EventuallyCreation(func() error { + ns.ResourceVersion = "" + return k8sClient.Create(ctx, ns) + }).Should(Succeed()) + }) + + AfterEach(func() { + ForceDeleteNamespace(ctx, testNamespace) + // delete all global quotas used by tests + quotaList := &capsulev1beta2.CustomQuotaList{} + if err := k8sClient.List(ctx, quotaList); err == nil { + for i := range quotaList.Items { + item := quotaList.Items[i] + if item.Name == "" { + continue + } + if item.Labels["e2e.capsule.dev/test-suite"] == "customquota-ledger" { + EventuallyDeletion(&item) + } + } + } + + // delete all global quotas used by tests + gquotaList := &capsulev1beta2.GlobalCustomQuotaList{} + if err := k8sClient.List(ctx, gquotaList); err == nil { + for i := range gquotaList.Items { + item := gquotaList.Items[i] + if item.Name == "" { + continue + } + if item.Labels["e2e.capsule.dev/test-suite"] == "customquota-ledger" { + EventuallyDeletion(&item) + } + } + } + }) + + It("marks the quota not ready when an existing matching object has no value at the configured quantity path", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-missing-path-not-ready", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + }, + }, + } + + pod := MakePod(testNamespace, "cq-missing-emptydir-size", nil, nil, "nginx:1.27.0", "", "") + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + + Eventually(func(g Gomega) { + obj := &capsulev1beta2.CustomQuota{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: q.GetName(), + Namespace: testNamespace, + }, obj)).To(Succeed()) + + cond := obj.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(cond).NotTo(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Message).To(ContainSubstring("did not resolve to any value")) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + It("denies creating a matching object when the configured quantity path resolves to no value", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-missing-path-deny-create", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + Eventually(func() error { + pod := MakePod(testNamespace, "cq-missing-path-denied", nil, nil, "nginx:1.27.0", "", "") + return k8sClient.Create(ctx, pod) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring("did not resolve to any value")), + ) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "0", 0) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + }) + + It("remains consistent under concurrent pod creations for a CustomQuota", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-concurrent-pod-count", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + const total = 100 + type result struct { + name string + err error + } + + results := make(chan result, total) + + for i := 0; i < total; i++ { + i := i + go func() { + name := fmt.Sprintf("cq-concurrent-pod-%02d", i) + pod := MakePod( + testNamespace, + name, + map[string]string{"track": "yes"}, + nil, + "nginx:1.27.0", + "", + "", + ) + + err := k8sClient.Create(ctx, pod) + results <- result{name: name, err: err} + }() + } + + var succeeded, failed int + for i := 0; i < total; i++ { + res := <-results + if res.err == nil { + succeeded++ + } else { + failed++ + } + } + + Expect(succeeded).To(Equal(10)) + Expect(failed).To(Equal(90)) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "10", 10) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + }) + + It("tracks different paths independently when global and namespaced quotas match the same pod gvk", func() { + gq := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-mixed-path-cpu-from-cq-suite", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("500m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + cq := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-mixed-path-emptydir-from-cq-suite", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gq) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, cq) }).Should(Succeed()) + + awaitGlobalQuotaReady(ctx, gq.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, cq.GetName()) + + p1 := MakePod(testNamespace, "mixed-path-cq-suite-1", nil, nil, "nginx:1.27.0", "100m", "1Gi") + p2 := MakePod(testNamespace, "mixed-path-cq-suite-2", nil, nil, "nginx:1.27.0", "100m", "1Gi") + + EventuallyCreation(func() error { p1.ResourceVersion = ""; return k8sClient.Create(ctx, p1) }).Should(Succeed()) + EventuallyCreation(func() error { p2.ResourceVersion = ""; return k8sClient.Create(ctx, p2) }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "200m", 2) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, cq.GetName(), "2Gi", 2) + expectLedgerSettled(ctx, ControllerNamespace, gq.GetName()) + expectLedgerSettled(ctx, testNamespace, cq.GetName()) + + Eventually(func() error { + p3 := MakePod(testNamespace, "mixed-path-cq-suite-3", nil, nil, "nginx:1.27.0", "100m", "1Gi") + return k8sClient.Create(ctx, p3) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring(`CustomQuota "cq-mixed-path-emptydir-from-cq-suite"`)), + ) + }) + + It("treats missing quantity paths as error", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-wrong-path-zero", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].doesNotExist.sizeLimit", + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.resources.requests.thisDoesNotExist", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pod := MakePod(testNamespace, "wrong-path-pod", nil, nil, "nginx:1.27.0", "", "1Gi") + pvc := MakePVC(testNamespace, "wrong-path-pvc", "2Gi") + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).ShouldNot(Succeed()) + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).ShouldNot(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "0", 0) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + }) + + It("aggregates a custom pod quantity path and settles the corresponding ledger", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-pod-cpu-requests", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("500m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + dep := MakeDeployment(testNamespace, "cpu-requests", 2, map[string]string{ + "track": "yes", + }, "100m") + EventuallyCreation(func() error { + dep.ResourceVersion = "" + return k8sClient.Create(ctx, dep) + }).Should(Succeed()) + ExpectPodsForDeployment(ctx, testNamespace, "cpu-requests", 2) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "200m", 2) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + + ScaleDeployment(ctx, testNamespace, "cpu-requests", 4) + ExpectPodsForDeployment(ctx, testNamespace, "cpu-requests", 4) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "400m", 4) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + + ledger := getLedger(ctx, testNamespace, q.GetName()) + Expect(ledger.Spec.TargetRef.Kind).To(Equal("CustomQuota")) + Expect(ledger.Spec.TargetRef.Name).To(Equal(q.GetName())) + Expect(ledger.Spec.TargetRef.Namespace).To(Equal(testNamespace)) + }) + + It("counts pods correctly while scaling a deployment", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-pod-count", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("5"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + dep := MakeDeployment(testNamespace, "counted", 1, nil, "") + EventuallyCreation(func() error { + dep.ResourceVersion = "" + return k8sClient.Create(ctx, dep) + }).Should(Succeed()) + + ExpectPodsForDeployment(ctx, testNamespace, "counted", 1) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "1", 1) + + ScaleDeployment(ctx, testNamespace, "counted", 3) + ExpectPodsForDeployment(ctx, testNamespace, "counted", 3) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "3", 3) + + ScaleDeployment(ctx, testNamespace, "counted", 2) + ExpectPodsForDeployment(ctx, testNamespace, "counted", 2) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "2", 2) + + expectLedgerSettled(ctx, testNamespace, q.GetName()) + }) + + It("tracks count with a single MatchLabels selector and updates when the pod no longer matches", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-count-single-matchlabel", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pod := MakePod(testNamespace, "single-matchlabel", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "") + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "1", 1) + + UpdatePodLabels(ctx, testNamespace, "single-matchlabel", map[string]string{"track": "no"}) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "0", 0) + }) + + It("tracks count with multiple MatchLabels and updates when the pod no longer matches", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-count-multi-matchlabel", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + "tier": "frontend", + }, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pod := MakePod(testNamespace, "multi-matchlabel", map[string]string{ + "track": "yes", + "tier": "frontend", + }, nil, "nginx:1.27.0", "", "") + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "1", 1) + + UpdatePodLabels(ctx, testNamespace, "multi-matchlabel", map[string]string{ + "track": "yes", + "tier": "backend", + }) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "0", 0) + }) + + It("tracks count with a single field selector and updates when the pod no longer matches", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-count-single-fieldselector", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{ + `.spec.containers[?(@.image=="nginx:1.27.0")]`, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pod := MakePod(testNamespace, "single-fieldselector", nil, nil, "nginx:1.27.0", "", "") + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "1", 1) + + UpdatePodImage(ctx, testNamespace, "single-fieldselector", "nginx:1.26.0") + expectLedgerSettled(ctx, testNamespace, q.GetName()) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "0", 0) + }) + + It("tracks count with multiple field selectors and updates when the pod no longer matches", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-count-multi-fieldselector", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{ + `.spec.containers[?(@.image=="nginx:1.27.0")]`, + `.spec.containers[?(@.name=="main")]`, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pod := MakePod(testNamespace, "multi-fieldselector", nil, nil, "nginx:1.27.0", "", "") + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "1", 1) + + UpdatePodImage(ctx, testNamespace, "multi-fieldselector", "nginx:1.26.0") + expectLedgerSettled(ctx, testNamespace, q.GetName()) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "0", 0) + }) + + It("aggregates multiple sources across pod emptyDir size and pvc storage size", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-multi-source-storage", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("3Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.resources.requests.storage", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pod := MakePod(testNamespace, "multi-source-pod", nil, nil, "nginx:1.27.0", "", "1Gi") + pvc := MakePVC(testNamespace, "multi-source-pvc", "2Gi") + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "3Gi", 2) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + }) + + It("aggregates multiple sources with selectors across pod emptyDir and pvc storage", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-multi-source-selectors-storage", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Path: ".spec.volumes[*].emptyDir.sizeLimit", + Operation: quota.OpAdd, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Path: ".spec.resources.requests.storage", + Operation: quota.OpAdd, + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{ + `.spec.accessModes[?(@=="ReadWriteOnce")]`, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + matchingPod := MakePod(testNamespace, "matching-emptydir", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "1Gi") + nonMatchingPod := MakePod(testNamespace, "ignored-emptydir", map[string]string{"track": "no"}, nil, "nginx:1.27.0", "", "5Gi") + + matchingPVC := MakePVC(testNamespace, "matching-pvc", "2Gi") + nonMatchingPVC := MakePVC(testNamespace, "ignored-pvc", "4Gi") + nonMatchingPVC.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} + + EventuallyCreation(func() error { + matchingPod.ResourceVersion = "" + return k8sClient.Create(ctx, matchingPod) + }).Should(Succeed()) + + EventuallyCreation(func() error { + nonMatchingPod.ResourceVersion = "" + return k8sClient.Create(ctx, nonMatchingPod) + }).Should(Succeed()) + + EventuallyCreation(func() error { + matchingPVC.ResourceVersion = "" + return k8sClient.Create(ctx, matchingPVC) + }).Should(Succeed()) + + EventuallyCreation(func() error { + nonMatchingPVC.ResourceVersion = "" + return k8sClient.Create(ctx, nonMatchingPVC) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "3Gi", 2) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + }) + + It("clamps usage to zero for a pure subtraction source", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-sub-only-clamps-zero", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.resources.requests.storage", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pvc := MakePVC(testNamespace, "sub-only-pvc", "2Gi") + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "0", 1) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + }) + + It("subtracts matching pvc storage from added pod emptyDir storage", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-add-sub-storage", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.resources.requests.storage", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pod := MakePod(testNamespace, "add-sub-pod", nil, nil, "nginx:1.27.0", "", "3Gi") + pvc := MakePVC(testNamespace, "add-sub-pvc", "1Gi") + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "2Gi", 2) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + }) + + It("clamps mixed add and subtraction result to zero when subtraction exceeds additions", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-add-sub-clamp-zero", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.resources.requests.storage", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pod := MakePod(testNamespace, "add-sub-clamp-pod", nil, nil, "nginx:1.27.0", "", "1Gi") + pvc := MakePVC(testNamespace, "add-sub-clamp-pvc", "2Gi") + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "0", 2) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + }) + + It("supports subtraction with label selectors and removes the subtraction when the object no longer matches", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-sub-label-selector", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.resources.requests.storage", + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "discount": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pod := MakePod(testNamespace, "sub-label-pod", nil, nil, "nginx:1.27.0", "", "3Gi") + pvc := MakePVC(testNamespace, "sub-label-pvc", "1Gi") + pvc.Labels = map[string]string{"discount": "yes"} + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "2Gi", 2) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + + Eventually(func() error { + obj := &corev1.PersistentVolumeClaim{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace}, obj); err != nil { + return err + } + obj.Labels = map[string]string{"discount": "no"} + return k8sClient.Update(ctx, obj) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + expectLedgerSettled(ctx, testNamespace, q.GetName()) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "3Gi", 1) + }) + + It("reconciles subtraction correctly when the subtracting resource is deleted", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-sub-delete-reconcile", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpSub, + Path: ".spec.resources.requests.storage", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pod := MakePod(testNamespace, "sub-delete-pod", nil, nil, "nginx:1.27.0", "", "3Gi") + pvc := MakePVC(testNamespace, "sub-delete-pvc", "1Gi") + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "2Gi", 2) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + + EventuallyDeletion(pvc) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "3Gi", 1) + }) + + It("uses the smallest matching custom quota as authoritative while accounting successful pod count in both quotas", func() { + small := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-pod-count-small", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + } + + large := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-pod-count-large", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("5"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, small) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, large) }).Should(Succeed()) + + awaitCustomQuotaReady(ctx, testNamespace, small.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, large.GetName()) + + pod1 := MakePod(testNamespace, "multi-cq-count-1", nil, nil, "nginx:1.27.0", "", "") + pod2 := MakePod(testNamespace, "multi-cq-count-2", nil, nil, "nginx:1.27.0", "", "") + + EventuallyCreation(func() error { + pod1.ResourceVersion = "" + return k8sClient.Create(ctx, pod1) + }).Should(Succeed()) + EventuallyCreation(func() error { + pod2.ResourceVersion = "" + return k8sClient.Create(ctx, pod2) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, small.GetName(), "2", 2) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, large.GetName(), "2", 2) + expectLedgerSettled(ctx, testNamespace, small.GetName()) + expectLedgerSettled(ctx, testNamespace, large.GetName()) + + Eventually(func() error { + pod3 := MakePod(testNamespace, "multi-cq-count-3", nil, nil, "nginx:1.27.0", "", "") + return k8sClient.Create(ctx, pod3) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring(`CustomQuota "cq-pod-count-small"`)), + ) + }) + + It("uses the smallest matching custom quota as authoritative while accounting successful cpu usage in both quotas", func() { + small := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-pod-cpu-small", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("200m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + } + + large := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-pod-cpu-large", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("500m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, small) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, large) }).Should(Succeed()) + + awaitCustomQuotaReady(ctx, testNamespace, small.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, large.GetName()) + + pod1 := MakePod(testNamespace, "multi-cq-cpu-1", nil, nil, "nginx:1.27.0", "100m", "") + pod2 := MakePod(testNamespace, "multi-cq-cpu-2", nil, nil, "nginx:1.27.0", "100m", "") + + EventuallyCreation(func() error { + pod1.ResourceVersion = "" + return k8sClient.Create(ctx, pod1) + }).Should(Succeed()) + EventuallyCreation(func() error { + pod2.ResourceVersion = "" + return k8sClient.Create(ctx, pod2) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, small.GetName(), "200m", 2) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, large.GetName(), "200m", 2) + expectLedgerSettled(ctx, testNamespace, small.GetName()) + expectLedgerSettled(ctx, testNamespace, large.GetName()) + + Eventually(func() error { + pod3 := MakePod(testNamespace, "multi-cq-cpu-3", nil, nil, "nginx:1.27.0", "100m", "") + return k8sClient.Create(ctx, pod3) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring(`CustomQuota "cq-pod-cpu-small"`)), + ) + }) + + It("accounts only the matching subset for overlapping selectors on the same pod gvk", func() { + broad := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-pod-track-broad", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("5"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + } + + narrow := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-pod-track-frontend", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + "tier": "frontend", + }, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, broad) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, narrow) }).Should(Succeed()) + + awaitCustomQuotaReady(ctx, testNamespace, broad.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, narrow.GetName()) + + p1 := MakePod(testNamespace, "cq-overlap-1", map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") + p2 := MakePod(testNamespace, "cq-overlap-2", map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") + p3 := MakePod(testNamespace, "cq-overlap-3", map[string]string{"track": "yes", "tier": "backend"}, nil, "nginx:1.27.0", "", "") + p4 := MakePod(testNamespace, "cq-overlap-4", map[string]string{"track": "yes", "tier": "backend"}, nil, "nginx:1.27.0", "", "") + + EventuallyCreation(func() error { p1.ResourceVersion = ""; return k8sClient.Create(ctx, p1) }).Should(Succeed()) + EventuallyCreation(func() error { p2.ResourceVersion = ""; return k8sClient.Create(ctx, p2) }).Should(Succeed()) + EventuallyCreation(func() error { p3.ResourceVersion = ""; return k8sClient.Create(ctx, p3) }).Should(Succeed()) + EventuallyCreation(func() error { p4.ResourceVersion = ""; return k8sClient.Create(ctx, p4) }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, broad.GetName(), "4", 4) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, narrow.GetName(), "2", 2) + expectLedgerSettled(ctx, testNamespace, broad.GetName()) + expectLedgerSettled(ctx, testNamespace, narrow.GetName()) + + Eventually(func() error { + p5 := MakePod(testNamespace, "cq-overlap-5", map[string]string{"track": "yes", "tier": "frontend"}, nil, "nginx:1.27.0", "", "") + return k8sClient.Create(ctx, p5) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring(`CustomQuota "cq-pod-track-frontend"`)), + ) + + EventuallyCreation(func() error { + p6 := MakePod(testNamespace, "cq-overlap-6", map[string]string{"track": "yes", "tier": "backend"}, nil, "nginx:1.27.0", "", "") + return k8sClient.Create(ctx, p6) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, broad.GetName(), "5", 5) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, narrow.GetName(), "2", 2) + }) + + It("tracks different paths independently when multiple custom quotas match the same pod gvk", func() { + cpuQuota := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-pod-path-cpu", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("400m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + } + + emptyDirQuota := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-pod-path-emptydir", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, cpuQuota) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, emptyDirQuota) }).Should(Succeed()) + + awaitCustomQuotaReady(ctx, testNamespace, cpuQuota.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, emptyDirQuota.GetName()) + + p1 := MakePod(testNamespace, "cq-path-pod-1", nil, nil, "nginx:1.27.0", "100m", "1Gi") + p2 := MakePod(testNamespace, "cq-path-pod-2", nil, nil, "nginx:1.27.0", "100m", "1Gi") + + EventuallyCreation(func() error { p1.ResourceVersion = ""; return k8sClient.Create(ctx, p1) }).Should(Succeed()) + EventuallyCreation(func() error { p2.ResourceVersion = ""; return k8sClient.Create(ctx, p2) }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, cpuQuota.GetName(), "200m", 2) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, emptyDirQuota.GetName(), "2Gi", 2) + expectLedgerSettled(ctx, testNamespace, cpuQuota.GetName()) + expectLedgerSettled(ctx, testNamespace, emptyDirQuota.GetName()) + + Eventually(func() error { + p3 := MakePod(testNamespace, "cq-path-pod-3", nil, nil, "nginx:1.27.0", "100m", "1Gi") + return k8sClient.Create(ctx, p3) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring(`CustomQuota "cq-pod-path-emptydir"`)), + ) + }) + + It("accounts deployment scaling in multiple custom quotas and denies when the smaller quota is exceeded", func() { + small := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-scale-small", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("3"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + } + + large := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-scale-large", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, small) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, large) }).Should(Succeed()) + + awaitCustomQuotaReady(ctx, testNamespace, small.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, large.GetName()) + + dep := MakeDeployment(testNamespace, "cq-scale", 1, nil, "") + EventuallyCreation(func() error { + dep.ResourceVersion = "" + return k8sClient.Create(ctx, dep) + }).Should(Succeed()) + ExpectPodsForDeployment(ctx, testNamespace, "cq-scale", 1) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, small.GetName(), "1", 1) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, large.GetName(), "1", 1) + + ScaleDeployment(ctx, testNamespace, "cq-scale", 3) + ExpectPodsForDeployment(ctx, testNamespace, "cq-scale", 3) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, small.GetName(), "3", 3) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, large.GetName(), "3", 3) + + ScaleDeployment(ctx, testNamespace, "cq-scale", 4) + + Eventually(func(g Gomega) { + obj := &capsulev1beta2.CustomQuota{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: small.GetName(), + Namespace: testNamespace, + }, obj)).To(Succeed()) + g.Expect(obj.Status.Usage.Used.Cmp(resource.MustParse("3"))).To(Equal(0)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + ExpectPodsForDeployment(ctx, testNamespace, "cq-scale", 3) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, large.GetName(), "3", 3) + }) + + It("uses the smallest matching quota as authoritative while accounting successful pod count in both global and namespaced quotas", func() { + gq := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-mixed-pod-count-from-cq-suite", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("5"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + cq := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-mixed-pod-count-from-cq-suite", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("2"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gq) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, cq) }).Should(Succeed()) + + awaitGlobalQuotaReady(ctx, gq.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, cq.GetName()) + + p1 := MakePod(testNamespace, "mixed-cq-suite-1", nil, nil, "nginx:1.27.0", "", "") + p2 := MakePod(testNamespace, "mixed-cq-suite-2", nil, nil, "nginx:1.27.0", "", "") + + EventuallyCreation(func() error { p1.ResourceVersion = ""; return k8sClient.Create(ctx, p1) }).Should(Succeed()) + EventuallyCreation(func() error { p2.ResourceVersion = ""; return k8sClient.Create(ctx, p2) }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "2", 2) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, cq.GetName(), "2", 2) + expectLedgerSettled(ctx, ControllerNamespace, gq.GetName()) + expectLedgerSettled(ctx, testNamespace, cq.GetName()) + + Eventually(func() error { + p3 := MakePod(testNamespace, "mixed-cq-suite-3", nil, nil, "nginx:1.27.0", "", "") + return k8sClient.Create(ctx, p3) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring(`CustomQuota "cq-mixed-pod-count-from-cq-suite"`)), + ) + }) + + It("uses the smallest matching quota as authoritative while accounting successful cpu usage in both global and namespaced quotas", func() { + gq := &capsulev1beta2.GlobalCustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gq-mixed-pod-cpu-from-cq-suite", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.GlobalCustomQuotaSpec{ + CustomQuotaSpec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("500m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + NamespaceSelectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + tenantLabel: tenantValue, + }, + }, + }, + }, + }, + } + + cq := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-mixed-pod-cpu-from-cq-suite", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("200m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gq) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, cq) }).Should(Succeed()) + + awaitGlobalQuotaReady(ctx, gq.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, cq.GetName()) + + p1 := MakePod(testNamespace, "mixed-cpu-cq-suite-1", nil, nil, "nginx:1.27.0", "100m", "") + p2 := MakePod(testNamespace, "mixed-cpu-cq-suite-2", nil, nil, "nginx:1.27.0", "100m", "") + + EventuallyCreation(func() error { p1.ResourceVersion = ""; return k8sClient.Create(ctx, p1) }).Should(Succeed()) + EventuallyCreation(func() error { p2.ResourceVersion = ""; return k8sClient.Create(ctx, p2) }).Should(Succeed()) + + expectGlobalQuotaUsedAndClaims(ctx, gq.GetName(), "200m", 2) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, cq.GetName(), "200m", 2) + expectLedgerSettled(ctx, ControllerNamespace, gq.GetName()) + expectLedgerSettled(ctx, testNamespace, cq.GetName()) + + Eventually(func() error { + p3 := MakePod(testNamespace, "mixed-cpu-cq-suite-3", nil, nil, "nginx:1.27.0", "100m", "") + return k8sClient.Create(ctx, p3) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring(`CustomQuota "cq-mixed-pod-cpu-from-cq-suite"`)), + ) + }) + + It("reconciles multiple sources when objects are deleted", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-multi-source-reconcile", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.resources.requests.storage", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pod := MakePod(testNamespace, "reconcile-emptydir", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "1Gi") + pvc := MakePVC(testNamespace, "reconcile-pvc", "2Gi") + + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + EventuallyCreation(func() error { + pvc.ResourceVersion = "" + return k8sClient.Create(ctx, pvc) + }).Should(Succeed()) + + expectLedgerSettled(ctx, testNamespace, q.GetName()) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "3Gi", 2) + + EventuallyDeletion(pod) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "2Gi", 1) + + EventuallyDeletion(pvc) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "0", 0) + }) + + It("does not produce negative usage when a matching pod is relabeled to no longer match", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-no-negative-on-relabel", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pod := MakePod(testNamespace, "no-negative-on-relabel", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "") + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "1", 1) + + UpdatePodLabels(ctx, testNamespace, "no-negative-on-relabel", map[string]string{"track": "no"}) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "0", 0) + }) + + It("retracts emptyDir usage when a pod no longer matches source selectors", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-emptydir-relabel", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + pod := MakePod(testNamespace, "emptydir-relabel", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "1Gi") + EventuallyCreation(func() error { + pod.ResourceVersion = "" + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "1Gi", 1) + + UpdatePodLabels(ctx, testNamespace, "emptydir-relabel", map[string]string{"track": "no"}) + expectLedgerSettled(ctx, testNamespace, q.GetName()) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, q.GetName(), "0", 0) + }) + + It("accounts only the quotas that actually match when multiple custom quotas share the same gvk", func() { + labelQuota := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-track-only", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "track": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + } + + fieldQuota := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-nginx-only", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{ + `.spec.containers[?(@.image=="nginx:1.27.0")]`, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, labelQuota) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, fieldQuota) }).Should(Succeed()) + + awaitCustomQuotaReady(ctx, testNamespace, labelQuota.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, fieldQuota.GetName()) + + matchBoth := MakePod(testNamespace, "subset-both", map[string]string{"track": "yes"}, nil, "nginx:1.27.0", "", "") + matchLabelOnly := MakePod(testNamespace, "subset-label-only", map[string]string{"track": "yes"}, nil, "busybox:1.36.1", "", "") + matchFieldOnly := MakePod(testNamespace, "subset-field-only", nil, nil, "nginx:1.27.0", "", "") + + EventuallyCreation(func() error { matchBoth.ResourceVersion = ""; return k8sClient.Create(ctx, matchBoth) }).Should(Succeed()) + EventuallyCreation(func() error { matchLabelOnly.ResourceVersion = ""; return k8sClient.Create(ctx, matchLabelOnly) }).Should(Succeed()) + EventuallyCreation(func() error { matchFieldOnly.ResourceVersion = ""; return k8sClient.Create(ctx, matchFieldOnly) }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, labelQuota.GetName(), "2", 2) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, fieldQuota.GetName(), "2", 2) + expectLedgerSettled(ctx, testNamespace, labelQuota.GetName()) + expectLedgerSettled(ctx, testNamespace, fieldQuota.GetName()) + }) + + It("uses deterministic tie-breaking when multiple custom quotas have the same remaining availability", func() { + quotaA := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-tie-a", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("3"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + } + + quotaB := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-tie-b", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("4"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, quotaA) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, quotaB) }).Should(Succeed()) + + awaitCustomQuotaReady(ctx, testNamespace, quotaA.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, quotaB.GetName()) + + p1 := MakePod(testNamespace, "tie-1", nil, nil, "nginx:1.27.0", "", "") + p2 := MakePod(testNamespace, "tie-2", nil, nil, "nginx:1.27.0", "", "") + p3 := MakePod(testNamespace, "tie-3", nil, nil, "nginx:1.27.0", "", "") + + EventuallyCreation(func() error { p1.ResourceVersion = ""; return k8sClient.Create(ctx, p1) }).Should(Succeed()) + EventuallyCreation(func() error { p2.ResourceVersion = ""; return k8sClient.Create(ctx, p2) }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, quotaA.GetName(), "2", 2) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, quotaB.GetName(), "2", 2) + + EventuallyCreation(func() error { p3.ResourceVersion = ""; return k8sClient.Create(ctx, p3) }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, quotaA.GetName(), "3", 3) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, quotaB.GetName(), "3", 3) + + Eventually(func() error { + p4 := MakePod(testNamespace, "tie-4", nil, nil, "nginx:1.27.0", "", "") + return k8sClient.Create(ctx, p4) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring(`CustomQuota "cq-tie-a"`)), + ) + }) + + It("aggregates the same successful pod into multiple custom quotas with different paths on the same gvk", func() { + cpuQuota := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-multi-path-cpu", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("300m"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.containers[*].resources.requests.cpu", + }, + }, + }, + }, + } + + emptyDirQuota := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-multi-path-emptydir", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("3Gi"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpAdd, + Path: ".spec.volumes[*].emptyDir.sizeLimit", + }, + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, cpuQuota) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, emptyDirQuota) }).Should(Succeed()) + + awaitCustomQuotaReady(ctx, testNamespace, cpuQuota.GetName()) + awaitCustomQuotaReady(ctx, testNamespace, emptyDirQuota.GetName()) + + p1 := MakePod(testNamespace, "multi-path-shared-1", nil, nil, "nginx:1.27.0", "100m", "1Gi") + p2 := MakePod(testNamespace, "multi-path-shared-2", nil, nil, "nginx:1.27.0", "100m", "1Gi") + + EventuallyCreation(func() error { p1.ResourceVersion = ""; return k8sClient.Create(ctx, p1) }).Should(Succeed()) + EventuallyCreation(func() error { p2.ResourceVersion = ""; return k8sClient.Create(ctx, p2) }).Should(Succeed()) + + expectCustomQuotaUsedAndClaims(ctx, testNamespace, cpuQuota.GetName(), "200m", 2) + expectCustomQuotaUsedAndClaims(ctx, testNamespace, emptyDirQuota.GetName(), "2Gi", 2) + expectLedgerSettled(ctx, testNamespace, cpuQuota.GetName()) + expectLedgerSettled(ctx, testNamespace, emptyDirQuota.GetName()) + }) + + It("rejects admission when a field selector uses an invalid jsonpath filter on a scalar", func() { + q := &capsulev1beta2.CustomQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cq-invalid-fieldselector", + Namespace: testNamespace, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "customquota-ledger", + }, + }, + Spec: capsulev1beta2.CustomQuotaSpec{ + Limit: resource.MustParse("10"), + Sources: []capsulev1beta2.CustomQuotaSpecSource{ + { + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Pod", + }, + CustomQuotaSpecSourceConfig: capsulev1beta2.CustomQuotaSpecSourceConfig{ + Operation: quota.OpCount, + Selectors: []selectors.SelectorWithFields{ + { + FieldSelectors: []string{ + `.spec.restartPolicy[?(@=="Always")]`, + }, + }, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, q) + }).Should(Succeed()) + awaitCustomQuotaReady(ctx, testNamespace, q.GetName()) + + Eventually(func() error { + pod := MakePod(testNamespace, "invalid-selector-pod", nil, nil, "nginx:1.27.0", "", "") + return k8sClient.Create(ctx, pod) + }, defaultTimeoutInterval, defaultPollInterval).Should( + MatchError(ContainSubstring("is not array or slice and cannot be filtered")), + ) + }) +}) diff --git a/e2e/device_class_test.go b/e2e/device_class_test.go index a2987b35..487fbfcd 100644 --- a/e2e/device_class_test.go +++ b/e2e/device_class_test.go @@ -11,6 +11,8 @@ import ( . "github.com/onsi/gomega" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" resources "k8s.io/api/resource/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,13 +23,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes", "device"), func() { +var _ = Describe("when Tenant handles Device classes", Ordered, Label("tenant", "classes", "deviceclass"), func() { erm := "nvidia.com/gpu" authorized := &resources.DeviceClass{ ObjectMeta: metav1.ObjectMeta{ Name: "gpu.example.com", Labels: map[string]string{ - "env": "authorized", + "environment": "authorized", + "env": "e2e", }, }, Spec: resources.DeviceClassSpec{ @@ -45,7 +48,8 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" ObjectMeta: metav1.ObjectMeta{ Name: "gpu2.example.com", Labels: map[string]string{ - "env": "authorized", + "environment": "authorized", + "env": "e2e", }, }, Spec: resources.DeviceClassSpec{ @@ -63,7 +67,8 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" ObjectMeta: metav1.ObjectMeta{ Name: "gpu3.example.com", Labels: map[string]string{ - "env": "unauthorized", + "environment": "unauthorized", + "env": "e2e", }, }, Spec: resources.DeviceClassSpec{ @@ -81,12 +86,15 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" tntWithAuthorized := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-authorized-deviceclass", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: []api.OwnerSpec{ + Owners: []rbac.OwnerSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "authorized-deviceclass", Kind: "User", }, @@ -96,7 +104,7 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" DeviceClasses: &api.SelectorAllowedListSpec{ LabelSelector: v1.LabelSelector{ MatchLabels: map[string]string{ - "env": "authorized", + "environment": "authorized", }, }, }, @@ -105,12 +113,15 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" tntWithUnauthorized := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-unauthorized-deviceclass", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: []api.OwnerSpec{ + Owners: []rbac.OwnerSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "unauthorized-deviceclass", Kind: "User", }, @@ -120,7 +131,7 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" DeviceClasses: &api.SelectorAllowedListSpec{ LabelSelector: v1.LabelSelector{ MatchLabels: map[string]string{ - "env": "production", + "environment": "production", }, }, }, @@ -133,6 +144,7 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) } if err := k8sClient.List(context.Background(), &resources.DeviceClassList{}); err != nil { @@ -150,9 +162,7 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" }) JustAfterEach(func() { for _, tnt := range []*capsulev1beta2.Tenant{tntWithAuthorized, tntWithUnauthorized} { - EventuallyCreation(func() error { - return ignoreNotFound(k8sClient.Delete(context.TODO(), tnt)) - }).Should(Succeed()) + EventuallyDeletion(tnt) } if err := k8sClient.List(context.Background(), &resources.DeviceClassList{}); err != nil { @@ -161,16 +171,23 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" } } - Eventually(func() (err error) { - req, _ := labels.NewRequirement("env", selection.Exists, nil) + req, err := labels.NewRequirement("env", selection.Equals, []string{"e2e"}) + Expect(err).NotTo(HaveOccurred()) - return k8sClient.DeleteAllOf(context.TODO(), &resources.DeviceClass{}, &client.DeleteAllOfOptions{ - ListOptions: client.ListOptions{ - LabelSelector: labels.NewSelector().Add(*req), - }, - }) - }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + var list resources.DeviceClassList + Expect(k8sClient.List( + context.TODO(), + &list, + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*req), + }, + )).Should(Succeed()) + + for i := range list.Items { + EventuallyDeletion(&list.Items[i]) + } }) + It("ResourceClaims", func() { if err := k8sClient.List(context.Background(), &resources.DeviceClassList{}); err != nil { if utils.IsUnsupportedAPI(err) { @@ -194,9 +211,12 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" Should(ConsistOf(authorized.GetName(), authorized2.GetName())) }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithAuthorized.GetName(), + }) + NamespaceCreation(ns, tntWithAuthorized.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithAuthorized, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithAuthorized, ns).Should(Succeed()) By("providing authorized device class", func() { for _, class := range []*resources.DeviceClass{authorized} { @@ -305,7 +325,7 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" By("Verify Status (Deletion)", func() { for _, class := range []*resources.DeviceClass{authorized} { - Expect(ignoreNotFound(k8sClient.Delete(context.TODO(), class))).To(Succeed()) + EventuallyDeletion(class) } Eventually(func() ([]string, error) { @@ -320,7 +340,7 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" return t.Status.Classes.DeviceClasses, nil }, defaultTimeoutInterval, defaultPollInterval). - ShouldNot(ConsistOf(authorized.GetName(), authorized2.GetName())) + Should(ConsistOf(authorized2.GetName())) }) }) It("ResourceClaimTemplates", func() { @@ -330,9 +350,11 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" } } - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithAuthorized.GetName(), + }) NamespaceCreation(ns, tntWithAuthorized.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithAuthorized, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithAuthorized, ns).Should(Succeed()) By("providing authorized device class", func() { for _, class := range []*resources.DeviceClass{authorized} { @@ -557,5 +579,26 @@ var _ = Describe("when Tenant handles Device classes", Label("tenant", "classes" }, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed()) } }) + + By("Verify Status", func() { + for _, class := range []*resources.DeviceClass{authorized} { + Expect(ignoreNotFound(k8sClient.Delete(context.TODO(), class))).To(Succeed()) + } + + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntWithAuthorized.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.DeviceClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + ShouldNot(ConsistOf(authorized.GetName(), authorized2.GetName())) + }) + }) }) diff --git a/e2e/forbidden_annotations_regex_test.go b/e2e/forbidden_annotations_regex_test.go deleted file mode 100644 index 5e6a9535..00000000 --- a/e2e/forbidden_annotations_regex_test.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2020-2023 Project Capsule Authors. -// SPDX-License-Identifier: Apache-2.0 - -package e2e - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" -) - -var _ = Describe("creating a tenant with various forbidden regexes", Label("tenant"), func() { - //errorRegexes := []string{ - // "(.*gitops|.*nsm).[k8s.io/((?!(resource)).*|trusted)](http://k8s.io/((?!(resource)).*%7Ctrusted))", - //} - // - //for _, annotationValue := range errorRegexes { - // It("should fail using a non-valid the regex on the annotation", func() { - // tnt := &capsulev1beta2.Tenant{ - // ObjectMeta: metav1.ObjectMeta{ - // Name: "namespace", - // }, - // Spec: capsulev1beta2.TenantSpec{ - // Owners: capsulev1beta2.OwnerListSpec{ - // { - // Name: "alice", - // Kind: "User", - // }, - // }, - // }, - // } - // - // EventuallyCreation(func() error { - // tnt.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{ - // ForbiddenLabels: api.ForbiddenListSpec{ - // Regex: annotationValue, - // }, - // } - // return k8sClient.Create(context.TODO(), tnt) - // }).ShouldNot(Succeed()) - // - // EventuallyCreation(func() error { - // tnt.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{ - // ForbiddenAnnotations: api.ForbiddenListSpec{ - // Regex: annotationValue, - // }, - // } - // return k8sClient.Create(context.TODO(), tnt) - // }).ShouldNot(Succeed()) - // }) - //} - - successRegexes := []string{ - "", - "(.*gitops|.*nsm)", - } - for _, annotationValue := range successRegexes { - It("should succeed using a valid regex on the annotation", func() { - tnt := &capsulev1beta2.Tenant{ - ObjectMeta: metav1.ObjectMeta{ - Name: "namespace", - }, - Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ - { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "alice", - Kind: "User", - }, - }, - }, - }, - }, - } - - EventuallyCreation(func() error { - tnt.SetResourceVersion("") - - tnt.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{ - ForbiddenLabels: api.ForbiddenListSpec{ - Regex: annotationValue, - }, - } - return k8sClient.Create(context.TODO(), tnt) - }).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) - - EventuallyCreation(func() error { - tnt.SetResourceVersion("") - - tnt.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{ - ForbiddenAnnotations: api.ForbiddenListSpec{ - Regex: annotationValue, - }, - } - return k8sClient.Create(context.TODO(), tnt) - }).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) - }) - } -}) diff --git a/e2e/gateway_class_test.go b/e2e/gateway_class_test.go index c7b50f59..8245e3c2 100644 --- a/e2e/gateway_class_test.go +++ b/e2e/gateway_class_test.go @@ -22,10 +22,12 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" ) -var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes", "gateway"), func() { +var _ = Describe("when Tenant handles Gateway classes", Ordered, Label("tenant", "classes", "gatewayclass"), func() { authorized := &gatewayv1.GatewayClass{ ObjectMeta: metav1.ObjectMeta{ Name: "customer-class", @@ -77,12 +79,15 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes tntWithDefault := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-gateway-default-and-label-selector", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: []api.OwnerSpec{ + Owners: []rbac.OwnerSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "gateway-default-and-label-selector", Kind: "User", }, @@ -110,12 +115,15 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes tntWithoutDefault := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-gateway-label-selector-only", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: []api.OwnerSpec{ + Owners: []rbac.OwnerSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "gateway-with-label-selector-only", Kind: "User", }, @@ -144,10 +152,10 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes Name: "e2e-gateway-no-restrictions", }, Spec: capsulev1beta2.TenantSpec{ - Owners: []api.OwnerSpec{ + Owners: []rbac.OwnerSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "e2e-gateway-no-restrictions", Kind: "User", }, @@ -163,6 +171,7 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) } if err := k8sClient.List(context.Background(), &gatewayv1.GatewayClassList{}); err != nil { @@ -183,9 +192,7 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes JustAfterEach(func() { utilruntime.Must(gatewayv1.Install(scheme.Scheme)) for _, tnt := range []*capsulev1beta2.Tenant{tntWithDefault, tntWithoutDefault, tntNoRestrictions} { - EventuallyCreation(func() error { - return ignoreNotFound(k8sClient.Delete(context.TODO(), tnt)) - }).Should(Succeed()) + EventuallyDeletion(tnt) } if err := k8sClient.List(context.Background(), &gatewayv1.GatewayClassList{}); err != nil { @@ -227,9 +234,11 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes Should(ConsistOf(exact.GetName(), exactU.GetName(), authorized.GetName(), unauthorized.GetName())) }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoRestrictions.GetName(), + }) NamespaceCreation(ns, tntNoRestrictions.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoRestrictions, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntNoRestrictions, ns).Should(Succeed()) By("providing any storageclass", func() { for _, class := range []*gatewayv1.GatewayClass{authorized, unauthorized, exact, exactU} { @@ -283,7 +292,7 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes By("Verify Status (Deletion)", func() { for _, crd := range []*gatewayv1.GatewayClass{authorized} { - Expect(ignoreNotFound(k8sClient.Delete(context.TODO(), crd))).To(Succeed()) + EventuallyDeletion(crd) } Eventually(func() ([]string, error) { t := &capsulev1beta2.Tenant{} @@ -327,9 +336,11 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes Should(ConsistOf(exactU.GetName(), authorized.GetName())) }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) By("providing unauthorized gatewayClassName", func() { Eventually(func() (err error) { @@ -379,7 +390,7 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes By("Verify Status (Deletion)", func() { for _, crd := range []*gatewayv1.GatewayClass{authorized} { - Expect(ignoreNotFound(k8sClient.Delete(context.TODO(), crd))).To(Succeed()) + EventuallyDeletion(crd) } Eventually(func() ([]string, error) { t := &capsulev1beta2.Tenant{} @@ -398,7 +409,7 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes Should(ConsistOf(exactU.GetName())) for _, crd := range []*gatewayv1.GatewayClass{exactU} { - Expect(ignoreNotFound(k8sClient.Delete(context.TODO(), crd))).To(Succeed()) + EventuallyDeletion(crd) } Eventually(func() ([]string, error) { t := &capsulev1beta2.Tenant{} @@ -443,9 +454,12 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes Should(ConsistOf(exactU.GetName(), authorized.GetName())) }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) + By("providing authorized class", func() { Eventually(func() (err error) { g := &gatewayv1.Gateway{ @@ -541,9 +555,12 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes Should(ConsistOf(exact.GetName(), authorized.GetName())) }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithoutDefault.GetName(), + }) NamespaceCreation(ns, tntWithoutDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithoutDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithoutDefault, ns).Should(Succeed()) + By("providing empty GatewayClassName", func() { Eventually(func() (err error) { g := &gatewayv1.Gateway{ @@ -568,7 +585,7 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes By("Verify Status (Deletion)", func() { for _, crd := range []*gatewayv1.GatewayClass{authorized} { - Expect(ignoreNotFound(k8sClient.Delete(context.TODO(), crd))).To(Succeed()) + EventuallyDeletion(crd) } Eventually(func() ([]string, error) { t := &capsulev1beta2.Tenant{} @@ -587,7 +604,7 @@ var _ = Describe("when Tenant handles Gateway classes", Label("tenant", "classes Should(ConsistOf(exact.GetName())) for _, crd := range []*gatewayv1.GatewayClass{exact} { - Expect(ignoreNotFound(k8sClient.Delete(context.TODO(), crd))).To(Succeed()) + EventuallyDeletion(crd) } Eventually(func() ([]string, error) { t := &capsulev1beta2.Tenant{} diff --git a/e2e/globaltenantresource_test.go b/e2e/globaltenantresource_test.go deleted file mode 100644 index 5de9a2bd..00000000 --- a/e2e/globaltenantresource_test.go +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright 2020-2023 Project Capsule Authors. -// SPDX-License-Identifier: Apache-2.0 - -package e2e - -import ( - "context" - "fmt" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/selection" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" -) - -var _ = Describe("Creating a GlobalTenantResource object", func() { - solar := &capsulev1beta2.Tenant{ - ObjectMeta: metav1.ObjectMeta{ - Name: "energy-solar", - Labels: map[string]string{ - "replicate": "true", - }, - }, - Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ - { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "solar-user", - Kind: "User", - }, - }, - }, - }, - }, - } - - wind := &capsulev1beta2.Tenant{ - ObjectMeta: metav1.ObjectMeta{ - Name: "energy-wind", - Labels: map[string]string{ - "replicate": "true", - }, - }, - Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ - { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "wind-user", - Kind: "User", - }, - }, - }, - }, - }, - } - - namespacedItem := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "dummy-secret", - Namespace: "default", - Labels: map[string]string{ - "replicate": "true", - }, - }, - Type: corev1.SecretTypeOpaque, - } - - gtr := &capsulev1beta2.GlobalTenantResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "replicate-energies", - }, - Spec: capsulev1beta2.GlobalTenantResourceSpec{ - TenantSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "replicate": "true", - }, - }, - TenantResourceSpec: capsulev1beta2.TenantResourceSpec{ - ResyncPeriod: metav1.Duration{Duration: time.Minute}, - PruningOnDelete: ptr.To(true), - Resources: []capsulev1beta2.ResourceSpec{ - { - NamespacedItems: []capsulev1beta2.ObjectReference{ - { - ObjectReferenceAbstract: capsulev1beta2.ObjectReferenceAbstract{ - Kind: "Secret", - Namespace: "default", - APIVersion: "v1", - }, - Selector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "replicate": "true", - }, - }, - }, - }, - RawItems: []capsulev1beta2.RawExtension{ - { - RawExtension: runtime.RawExtension{ - Object: &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "raw-secret-1", - }, - Type: corev1.SecretTypeOpaque, - }, - }, - }, - { - RawExtension: runtime.RawExtension{ - Object: &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "raw-secret-2", - }, - Type: corev1.SecretTypeOpaque, - }, - }, - }, - - { - RawExtension: runtime.RawExtension{ - Object: &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "raw-secret-3", - }, - Type: corev1.SecretTypeOpaque, - }, - }, - }, - }, - AdditionalMetadata: &api.AdditionalMetadataSpec{ - Labels: map[string]string{ - "labels.energy.io": "replicate", - }, - Annotations: map[string]string{ - "annotations.energy.io": "replicate", - }, - }, - }, - }, - }, - }, - } - - JustBeforeEach(func() { - EventuallyCreation(func() error { - return k8sClient.Create(context.TODO(), solar) - }).Should(Succeed()) - - EventuallyCreation(func() error { - return k8sClient.Create(context.TODO(), wind) - }).Should(Succeed()) - - EventuallyCreation(func() error { - return k8sClient.Create(context.TODO(), gtr) - }).Should(Succeed()) - - EventuallyCreation(func() error { - return k8sClient.Create(context.TODO(), namespacedItem) - }).Should(Succeed()) - }) - - JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), solar)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), wind)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), gtr)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), namespacedItem)).Should(Succeed()) - }) - - It("should replicate resources to all Tenants", func() { - solarNs, windNs := []string{"solar-one", "solar-two", "solar-three"}, []string{"wind-one", "wind-two", "wind-three"} - - By("creating solar Namespaces", func() { - for _, ns := range solarNs { - NamespaceCreation(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, solar.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - } - }) - - By("creating wind Namespaces", func() { - for _, ns := range windNs { - NamespaceCreation(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, wind.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - } - }) - - for _, ns := range append(solarNs, windNs...) { - By(fmt.Sprintf("waiting for replicated resources in %s Namespace", ns), func() { - Eventually(func() []corev1.Secret { - r, err := labels.NewRequirement("labels.energy.io", selection.DoubleEquals, []string{"replicate"}) - if err != nil { - return nil - } - - secrets := corev1.SecretList{} - err = k8sClient.List(context.TODO(), &secrets, &client.ListOptions{LabelSelector: labels.NewSelector().Add(*r), Namespace: ns}) - if err != nil { - return nil - } - - return secrets.Items - }, defaultTimeoutInterval, defaultPollInterval).Should(HaveLen(4)) - }) - } - - By("removing a Namespace from labels", func() { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: wind.GetName()}, wind)).ToNot(HaveOccurred()) - - wind.SetLabels(nil) - Expect(k8sClient.Update(context.TODO(), wind)).ToNot(HaveOccurred()) - - By("expecting no more items in the wind Tenant namespaces due to label update", func() { - for _, ns := range windNs { - Eventually(func() []corev1.Secret { - r, err := labels.NewRequirement("labels.energy.io", selection.DoubleEquals, []string{"replicate"}) - if err != nil { - return nil - } - - secrets := corev1.SecretList{} - err = k8sClient.List(context.TODO(), &secrets, &client.ListOptions{LabelSelector: labels.NewSelector().Add(*r), Namespace: ns}) - if err != nil { - return nil - } - - return secrets.Items - }, defaultTimeoutInterval, defaultPollInterval).Should(HaveLen(0)) - } - }) - }) - - By("using a Namespace selector", func() { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: gtr.GetName()}, gtr)).ToNot(HaveOccurred()) - - gtr.Spec.Resources[0].NamespaceSelector = &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "kubernetes.io/metadata.name": "solar-three", - }, - } - - Expect(k8sClient.Update(context.TODO(), gtr)).ToNot(HaveOccurred()) - - checkFn := func(ns string) func() []corev1.Secret { - return func() []corev1.Secret { - r, err := labels.NewRequirement("labels.energy.io", selection.DoubleEquals, []string{"replicate"}) - if err != nil { - return nil - } - - secrets := corev1.SecretList{} - err = k8sClient.List(context.TODO(), &secrets, &client.ListOptions{LabelSelector: labels.NewSelector().Add(*r), Namespace: ns}) - if err != nil { - return nil - } - - return secrets.Items - } - } - - for _, ns := range []string{"solar-one", "solar-two"} { - Eventually(checkFn(ns), defaultTimeoutInterval, defaultPollInterval).Should(HaveLen(0)) - } - - Eventually(checkFn("solar-three"), defaultTimeoutInterval, defaultPollInterval).Should(HaveLen(4)) - }) - - By("checking if replicated object have annotations and labels", func() { - for _, name := range []string{"dummy-secret", "raw-secret-1", "raw-secret-2", "raw-secret-3"} { - secret := corev1.Secret{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: "solar-three"}, &secret)).ToNot(HaveOccurred()) - - for k, v := range gtr.Spec.Resources[0].AdditionalMetadata.Labels { - _, err := HaveKeyWithValue(k, v).Match(secret.GetLabels()) - Expect(err).ToNot(HaveOccurred()) - } - - for k, v := range gtr.Spec.Resources[0].AdditionalMetadata.Annotations { - _, err := HaveKeyWithValue(k, v).Match(secret.GetAnnotations()) - Expect(err).ToNot(HaveOccurred()) - } - } - }) - }) -}) diff --git a/e2e/ingress_class_extensions_test.go b/e2e/ingress_class_extensions_test.go index c184dc15..b10a4549 100644 --- a/e2e/ingress_class_extensions_test.go +++ b/e2e/ingress_class_extensions_test.go @@ -16,20 +16,25 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" ) -var _ = Describe("when Tenant handles Ingress classes with extensions/v1beta1", Label("ingress"), func() { +var _ = Describe("when Tenant handles Ingress classes with extensions/v1beta1", Ordered, Label("tenant", "networking", "ingress"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "ingress-class-extensions-v1beta1", + Name: "e2e-ingress-class-extensions-v1beta1", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "ingress", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-ingress-class-extensions-v1beta1", Kind: "User", }, }, @@ -59,17 +64,20 @@ var _ = Describe("when Tenant handles Ingress classes with extensions/v1beta1", tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should block a non allowed class for extensions/v1beta1", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) By("non-specifying at all", func() { if err := k8sClient.List(context.Background(), &extensionsv1beta1.IngressList{}); err != nil { @@ -147,11 +155,13 @@ var _ = Describe("when Tenant handles Ingress classes with extensions/v1beta1", }) It("should allow enabled class using the deprecated annotation", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) for _, c := range tnt.Spec.IngressOptions.AllowedClasses.Exact { Eventually(func() (err error) { @@ -192,11 +202,13 @@ var _ = Describe("when Tenant handles Ingress classes with extensions/v1beta1", Skip("Running test on Kubernetes " + version.String() + ", doesn't provide .spec.ingressClassName") } - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) for _, c := range tnt.Spec.IngressOptions.AllowedClasses.Exact { Eventually(func() (err error) { @@ -219,12 +231,14 @@ var _ = Describe("when Tenant handles Ingress classes with extensions/v1beta1", }) It("should allow enabled Ingress by regex using the deprecated annotation", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) ingressClass := "oil-ingress" NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) Eventually(func() (err error) { if err := k8sClient.List(context.Background(), &extensionsv1beta1.IngressList{}); err != nil { @@ -253,12 +267,14 @@ var _ = Describe("when Tenant handles Ingress classes with extensions/v1beta1", }) It("should allow enabled Ingress by regex using the ingressClassName field", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) ingressClass := "oil-haproxy" NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) Eventually(func() (err error) { if err := k8sClient.List(context.Background(), &extensionsv1beta1.IngressList{}); err != nil { diff --git a/e2e/ingress_class_networking_test.go b/e2e/ingress_class_networking_test.go index 77d7d317..1d7dae91 100644 --- a/e2e/ingress_class_networking_test.go +++ b/e2e/ingress_class_networking_test.go @@ -21,20 +21,25 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" ) -var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1", Label("ingress"), func() { +var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1", Ordered, Label("tenant", "networking", "ingress"), func() { tntNoDefault := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "ic-selector-networking-v1", + Name: "e2e-ic-selector-networking-v1", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: []api.OwnerSpec{ + Owners: []rbac.OwnerSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "ingress-selector", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-ic-selector-networking-v1", Kind: "User", }, }, @@ -60,14 +65,17 @@ var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1" tntWithDefault := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "ic-default-networking-v1", + Name: "e2e-ic-default-networking-v1", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: []api.OwnerSpec{ + Owners: []rbac.OwnerSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "ingress-default", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-ic-default-networking-v1", Kind: "User", }, }, @@ -140,14 +148,13 @@ var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) } }) JustAfterEach(func() { for _, tnt := range []*capsulev1beta2.Tenant{tntWithDefault, tntNoDefault} { - Eventually(func() error { - return k8sClient.Delete(context.TODO(), tnt) - }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + EventuallyDeletion(tnt) } Eventually(func() (err error) { @@ -168,11 +175,13 @@ var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1" } } - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefault.GetName(), + }) cs := ownerClient(tntNoDefault.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tntNoDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntNoDefault, ns).Should(Succeed()) By("non-specifying at all", func() { Eventually(func() (err error) { @@ -250,11 +259,13 @@ var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1" } } - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefault.GetName(), + }) cs := ownerClient(tntNoDefault.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tntNoDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntNoDefault, ns).Should(Succeed()) for _, c := range tntNoDefault.Spec.IngressOptions.AllowedClasses.Exact { Eventually(func() (err error) { @@ -289,11 +300,13 @@ var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1" } } - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefault.GetName(), + }) cs := ownerClient(tntNoDefault.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tntNoDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntNoDefault, ns).Should(Succeed()) for _, c := range tntNoDefault.Spec.IngressOptions.AllowedClasses.Exact { Eventually(func() (err error) { @@ -326,12 +339,14 @@ var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1" } } - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefault.GetName(), + }) cs := ownerClient(tntNoDefault.Spec.Owners[0].UserSpec) ingressClass := "oil-ingress" NamespaceCreation(ns, tntNoDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntNoDefault, ns).Should(Succeed()) Eventually(func() (err error) { i := &networkingv1.Ingress{ @@ -364,12 +379,14 @@ var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1" } } - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefault.GetName(), + }) cs := ownerClient(tntNoDefault.Spec.Owners[0].UserSpec) ingressClass := "oil-haproxy" NamespaceCreation(ns, tntNoDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntNoDefault, ns).Should(Succeed()) Eventually(func() (err error) { i := &networkingv1.Ingress{ @@ -435,11 +452,13 @@ var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1" }, } - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefault.GetName(), + }) cs := ownerClient(tntNoDefault.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tntNoDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntNoDefault, ns).Should(Succeed()) EventuallyCreation(func() error { _, err := cs.NetworkingV1().Ingresses(ns.GetName()).Create(context.TODO(), i, metav1.CreateOptions{}) @@ -488,11 +507,13 @@ var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1" }, } - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefault.GetName(), + }) cs := ownerClient(tntNoDefault.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tntNoDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntNoDefault, ns).Should(Succeed()) EventuallyCreation(func() error { _, err := cs.NetworkingV1().Ingresses(ns.GetName()).Create(context.TODO(), i, metav1.CreateOptions{}) @@ -502,9 +523,12 @@ var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1" }) It("should mutate to default tenant IngressClass (class not does not exist)", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) + NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) i := &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ @@ -541,9 +565,11 @@ var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1" class := tenantDefault Expect(k8sClient.Create(context.TODO(), &class)).Should(Succeed()) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) i := &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ @@ -583,9 +609,11 @@ var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1" Expect(k8sClient.Create(context.TODO(), &class)).Should(Succeed()) Expect(k8sClient.Create(context.TODO(), &global)).Should(Succeed()) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) i := &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ @@ -629,9 +657,11 @@ var _ = Describe("when Tenant handles Ingress classes with networking.k8s.io/v1" Expect(k8sClient.Create(context.TODO(), &class)).Should(Succeed()) Expect(k8sClient.Create(context.TODO(), &global)).Should(Succeed()) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) i := &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ diff --git a/e2e/disable_ingress_wildcard_test.go b/e2e/ingress_disable_wildcard_test.go similarity index 90% rename from e2e/disable_ingress_wildcard_test.go rename to e2e/ingress_disable_wildcard_test.go index 55fb67ef..fe6c20eb 100644 --- a/e2e/disable_ingress_wildcard_test.go +++ b/e2e/ingress_disable_wildcard_test.go @@ -16,24 +16,28 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" ) -var _ = Describe("creating an Ingress with a wildcard when it is denied for the Tenant", Label("tenant"), func() { +var _ = Describe("creating an Ingress with a wildcard when it is denied for the Tenant", Ordered, Label("tenant", "networking", "ingress"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "denied-ingress-wildcard", + Name: "e2e-denied-ingress-wildcard", + Labels: map[string]string{ + "env": "e2e", + }, Annotations: map[string]string{ "capsule.clastix.io/deny-wildcard": "true", }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "scott", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-denied-ingress-wildcard", Kind: "User", }, }, @@ -48,10 +52,11 @@ var _ = Describe("creating an Ingress with a wildcard when it is denied for the return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should fail creating an extensions/v1beta1 Ingress with a wildcard hostname", func() { @@ -61,8 +66,9 @@ var _ = Describe("creating an Ingress with a wildcard when it is denied for the } } - ns := NewNamespace("") - + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) ok := &extensionsv1beta1.Ingress{ @@ -141,9 +147,11 @@ var _ = Describe("creating an Ingress with a wildcard when it is denied for the } } - ns := NewNamespace("") - + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) ok := &networkingv1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ @@ -221,9 +229,11 @@ var _ = Describe("creating an Ingress with a wildcard when it is denied for the } } - ns := NewNamespace("") - + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) ok := &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ diff --git a/e2e/ingress_hostnames_collision_cluster_scope_test.go b/e2e/ingress_hostnames_collision_cluster_scope_test.go index 5c921e41..444c079d 100644 --- a/e2e/ingress_hostnames_collision_cluster_scope_test.go +++ b/e2e/ingress_hostnames_collision_cluster_scope_test.go @@ -16,20 +16,25 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" ) -var _ = Describe("when handling Cluster scoped Ingress hostnames collision", Label("ingress"), func() { +var _ = Describe("when handling Cluster scoped Ingress hostnames collision", Ordered, Label("tenant", "networking", "ingress"), func() { tnt1 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "hostnames-collision-cluster-one", + Name: "e2e-hostnames-collision-one", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "ingress-tenant-one", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-hostnames-collision-one", Kind: "User", }, }, @@ -42,14 +47,17 @@ var _ = Describe("when handling Cluster scoped Ingress hostnames collision", Lab } tnt2 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "hostnames-collision-cluster-two", + Name: "e2e-hostnames-collision-two", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "ingress-tenant-two", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-hostnames-collision-two", Kind: "User", }, }, @@ -134,30 +142,35 @@ var _ = Describe("when handling Cluster scoped Ingress hostnames collision", Lab return k8sClient.Create(context.TODO(), tnt1) }).Should(Succeed()) + TenantReady(tnt1, metav1.ConditionTrue, defaultTimeoutInterval) EventuallyCreation(func() error { tnt2.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt2) }).Should(Succeed()) + TenantReady(tnt2, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt1)).Should(Succeed()) - - Expect(k8sClient.Delete(context.TODO(), tnt2)).Should(Succeed()) + EventuallyDeletion(tnt1) + EventuallyDeletion(tnt2) }) It("should ensure Cluster scope for Ingress hostname and path collision", func() { - ns1 := NewNamespace("") + ns1 := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt1.GetName(), + }) cs1 := ownerClient(tnt1.Spec.Owners[0].UserSpec) NamespaceCreation(ns1, tnt1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt1, defaultTimeoutInterval).Should(ContainElement(ns1.GetName())) + NamespaceIsPartOfTenant(tnt1, ns1).Should(Succeed()) - ns2 := NewNamespace("") + ns2 := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt2.GetName(), + }) cs2 := ownerClient(tnt2.Spec.Owners[0].UserSpec) NamespaceCreation(ns2, tnt2.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt2, defaultTimeoutInterval).Should(ContainElement(ns2.GetName())) + NamespaceIsPartOfTenant(tnt2, ns2).Should(Succeed()) By("testing networking.k8s.io", func() { if err := k8sClient.List(context.Background(), &networkingv1.IngressList{}); err != nil { diff --git a/e2e/ingress_hostnames_collision_disabled_test.go b/e2e/ingress_hostnames_collision_disabled_test.go index 7526577c..bb4a3883 100644 --- a/e2e/ingress_hostnames_collision_disabled_test.go +++ b/e2e/ingress_hostnames_collision_disabled_test.go @@ -16,20 +16,25 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" ) -var _ = Describe("when disabling Ingress hostnames collision", Label("ingress"), func() { +var _ = Describe("when disabling Ingress hostnames collision", Ordered, Label("tenant", "networking", "ingress"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "hostnames-collision-disabled", + Name: "e2e-hostnames-collision-disabled", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "ingress-disabled", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-hostnames-collision-disabled", Kind: "User", }, }, @@ -114,20 +119,26 @@ var _ = Describe("when disabling Ingress hostnames collision", Label("ingress"), return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should not check any kind of collision", func() { - ns1 := NewNamespace("") - ns2 := NewNamespace("") + ns1 := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + ns2 := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns1, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns1).Should(Succeed()) + NamespaceCreation(ns2, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns1.GetName())) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns2.GetName())) + NamespaceIsPartOfTenant(tnt, ns2).Should(Succeed()) By("testing networking.k8s.io", func() { if err := k8sClient.List(context.Background(), &networkingv1.IngressList{}); err != nil { diff --git a/e2e/ingress_hostnames_collision_namespace_scope_test.go b/e2e/ingress_hostnames_collision_namespace_scope_test.go index 792c5c47..1e96d4f3 100644 --- a/e2e/ingress_hostnames_collision_namespace_scope_test.go +++ b/e2e/ingress_hostnames_collision_namespace_scope_test.go @@ -16,20 +16,25 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" ) -var _ = Describe("when handling Namespace scoped Ingress hostnames collision", Label("ingress"), func() { +var _ = Describe("when handling Namespace scoped Ingress hostnames collision", Ordered, Label("tenant", "networking", "ingress"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "hostnames-collision-namespace", + Name: "e2e-hostnames-collision-namespace", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "ingress-namespace", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-hostnames-collision-namespace", Kind: "User", }, }, @@ -114,20 +119,25 @@ var _ = Describe("when handling Namespace scoped Ingress hostnames collision", L return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should ensure Namespace scope for Ingress hostname and path collision", func() { - ns1 := NewNamespace("") - ns2 := NewNamespace("") + ns1 := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + ns2 := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns1, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns1).Should(Succeed()) NamespaceCreation(ns2, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns1.GetName())) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns2.GetName())) + NamespaceIsPartOfTenant(tnt, ns2).Should(Succeed()) By("testing networking.k8s.io", func() { if err := k8sClient.List(context.Background(), &networkingv1.IngressList{}); err != nil { diff --git a/e2e/ingress_hostnames_collision_tenant_scope_test.go b/e2e/ingress_hostnames_collision_tenant_scope_test.go index 6812b79a..1a7fbe97 100644 --- a/e2e/ingress_hostnames_collision_tenant_scope_test.go +++ b/e2e/ingress_hostnames_collision_tenant_scope_test.go @@ -16,20 +16,25 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" ) -var _ = Describe("when handling Tenant scoped Ingress hostnames collision", Label("ingress"), func() { +var _ = Describe("when handling Tenant scoped Ingress hostnames collision", Ordered, Label("tenant", "networking", "ingress"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "hostnames-collision-tenant", + Name: "e2e-hostnames-collision-tenant", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "ingress-tenant", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-hostnames-collision-tenant", Kind: "User", }, }, @@ -113,24 +118,30 @@ var _ = Describe("when handling Tenant scoped Ingress hostnames collision", Labe tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should ensure Tenant scope for Ingress hostname and path collision", func() { - ns1 := NewNamespace("") + ns1 := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) - ns2 := NewNamespace("") + ns2 := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns1, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns1).Should(Succeed()) + NamespaceCreation(ns2, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns1.GetName())) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns2.GetName())) + NamespaceIsPartOfTenant(tnt, ns2).Should(Succeed()) By("testing networking.k8s.io", func() { if err := k8sClient.List(context.Background(), &networkingv1.IngressList{}); err != nil { diff --git a/e2e/ingress_hostnames_test.go b/e2e/ingress_hostnames_test.go index 732a34bd..be8efacf 100644 --- a/e2e/ingress_hostnames_test.go +++ b/e2e/ingress_hostnames_test.go @@ -16,20 +16,25 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" ) -var _ = Describe("when Tenant handles Ingress hostnames", Label("ingress"), func() { +var _ = Describe("when Tenant handles Ingress hostnames", Ordered, Label("tenant", "networking", "ingress"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "ingress-hostnames", + Name: "e2e-ingress-hostnames", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "hostname", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-ingress-hostnames", Kind: "User", }, }, @@ -118,18 +123,21 @@ var _ = Describe("when Tenant handles Ingress hostnames", Label("ingress"), func tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should block an empty hostname", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) By("testing networking.k8s.io", func() { if err := k8sClient.List(context.Background(), &networkingv1.IngressList{}); err != nil { @@ -147,11 +155,13 @@ var _ = Describe("when Tenant handles Ingress hostnames", Label("ingress"), func }) It("should block a non allowed Hostname", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) By("testing networking.k8s.io", func() { if err := k8sClient.List(context.Background(), &networkingv1.IngressList{}); err != nil { @@ -169,11 +179,13 @@ var _ = Describe("when Tenant handles Ingress hostnames", Label("ingress"), func }) It("should block a non allowed Hostname", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) By("testing extensions", func() { if err := k8sClient.List(context.Background(), &extensionsv1beta1.IngressList{}); err != nil { @@ -191,11 +203,13 @@ var _ = Describe("when Tenant handles Ingress hostnames", Label("ingress"), func }) It("should allow Hostnames in list", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) By("testing networking.k8s.io", func() { if err := k8sClient.List(context.Background(), &networkingv1.IngressList{}); err != nil { @@ -215,11 +229,13 @@ var _ = Describe("when Tenant handles Ingress hostnames", Label("ingress"), func }) It("should allow Hostnames in list", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) By("testing extensions", func() { if err := k8sClient.List(context.Background(), &extensionsv1beta1.IngressList{}); err != nil { @@ -239,11 +255,13 @@ var _ = Describe("when Tenant handles Ingress hostnames", Label("ingress"), func }) It("should allow Hostnames in regex", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) By("testing networking.k8s.io", func() { if err := k8sClient.List(context.Background(), &networkingv1.IngressList{}); err != nil { @@ -263,11 +281,13 @@ var _ = Describe("when Tenant handles Ingress hostnames", Label("ingress"), func }) It("should allow Hostnames in regex", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) By("testing extensions", func() { if err := k8sClient.List(context.Background(), &extensionsv1beta1.IngressList{}); err != nil { diff --git a/e2e/namespace_additional_webhook_test.go b/e2e/namespace_additional_webhook_test.go index df3626bc..2bd911db 100644 --- a/e2e/namespace_additional_webhook_test.go +++ b/e2e/namespace_additional_webhook_test.go @@ -13,12 +13,17 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespace for a Tenant with additional metadata", Label("namespace"), func() { +var _ = Describe("creating a Namespace for a Tenant with additional metadata", Ordered, Label("namespace"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-metadata-webhook", + Name: "e2e-tenant-additional-metadata", + Labels: map[string]string{ + "env": "e2e", + }, OwnerReferences: []metav1.OwnerReference{ { APIVersion: "cap", @@ -29,11 +34,11 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "gatsby", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-additional-metadata", Kind: "User", }, }, @@ -61,15 +66,18 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should contain additional Namespace metadata", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) By("checking additional labels", func() { Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) diff --git a/e2e/namespace_capsule_label_test.go b/e2e/namespace_capsule_label_test.go index d69a6c12..c78b483e 100644 --- a/e2e/namespace_capsule_label_test.go +++ b/e2e/namespace_capsule_label_test.go @@ -13,21 +13,24 @@ import ( "k8s.io/apimachinery/pkg/types" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating several Namespaces for a Tenant", Label("namespace"), func() { +var _ = Describe("creating several Namespaces for a Tenant", Ordered, Label("namespace"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "capsule-labels", + Name: "e2e-managed-labels", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "charlie", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-managed-labels", Kind: "User", }, }, @@ -41,19 +44,29 @@ var _ = Describe("creating several Namespaces for a Tenant", Label("namespace"), tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) + JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should contains the default Capsule label", func() { namespaces := []*v1.Namespace{ - NewNamespace(""), - NewNamespace(""), - NewNamespace(""), + NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }), + NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }), + NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }), } for _, ns := range namespaces { NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + Eventually(func() (ok bool) { Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) ok, _ = HaveKeyWithValue(meta.TenantLabel, tnt.Name).Match(ns.Labels) diff --git a/e2e/new_namespace_test.go b/e2e/namespace_creation_test.go similarity index 61% rename from e2e/new_namespace_test.go rename to e2e/namespace_creation_test.go index 14119ce6..68026889 100644 --- a/e2e/new_namespace_test.go +++ b/e2e/namespace_creation_test.go @@ -11,35 +11,39 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespaces as different type of Tenant owners", Label("namespace"), func() { +var _ = Describe("creating a Namespaces as different type of Tenant owners", Ordered, Label("namespace", "permissions", "owners"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-assigned", + Name: "e2e-ns-creation", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "alice", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-ns-creation-1", Kind: "User", }, }, }, { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "bob", - Kind: "Group", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-ns-creation-2", + Kind: "User", }, }, }, { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "system:serviceaccount:new-namespace-sa:default", Kind: "ServiceAccount", }, @@ -54,33 +58,46 @@ var _ = Describe("creating a Namespaces as different type of Tenant owners", Lab tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should be available in Tenant namespaces list and RoleBindings should be present when created", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElements(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) for _, owner := range tnt.Spec.Owners { Eventually(CheckForOwnerRoleBindings(ns, owner, nil), defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } }) It("should be available in Tenant namespaces list and RoleBindings should present when created as Group", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[1].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElements(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) for _, owner := range tnt.Spec.Owners { Eventually(CheckForOwnerRoleBindings(ns, owner, nil), defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } + + c := impersonationClient(tnt.Spec.Owners[1].UserSpec.Name, withDefaultGroups(nil)) + + err := c.Delete(context.TODO(), ns) + Expect(err).ToNot(HaveOccurred()) + }) It("should be available in Tenant namespaces list and RoleBindings should present when created as ServiceAccount", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[2].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElements(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) for _, owner := range tnt.Spec.Owners { Eventually(CheckForOwnerRoleBindings(ns, owner, nil), defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) diff --git a/e2e/namespace_hijacking_test.go b/e2e/namespace_hijacking_test.go index 39ff0a9f..e278ac55 100644 --- a/e2e/namespace_hijacking_test.go +++ b/e2e/namespace_hijacking_test.go @@ -7,35 +7,54 @@ import ( "context" "fmt" "math/rand" - - corev1 "k8s.io/api/core/v1" + "sort" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apiserver/pkg/authentication/serviceaccount" + "k8s.io/utils/ptr" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" + clt "github.com/projectcapsule/capsule/pkg/runtime/client" + "github.com/projectcapsule/capsule/pkg/tenant" ) -var _ = Describe("creating several Namespaces for a Tenant", Label("namespace", "hijack"), func() { - tnt_1 := &capsulev1beta2.Tenant{ +var _ = Describe("creating several Namespaces for a Tenant", Ordered, Label("namespace", "hijack"), func() { + t1 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "capsule-ns-attack-1", + Name: "e2e-ns-attack-1", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "gatsby", Kind: "User", }, }, }, { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "oidc:group", + Kind: "Group", + }, + }, + }, + { + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Kind: "ServiceAccount", Name: "system:serviceaccount:attacker-system:attacker", }, @@ -45,54 +64,418 @@ var _ = Describe("creating several Namespaces for a Tenant", Label("namespace", }, } + t2 := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-ns-attack-2", + Labels: map[string]string{ + "env": "e2e", + }, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{ + { + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "gatsby", + Kind: "User", + }, + }, + }, + }, + }, + } + + t3 := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-ns-attack-3", + Labels: map[string]string{"env": "e2e"}, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{ + { + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "different-owner", + Kind: "User", + }, + }, + }, + }, + }, + } + kubeSystem := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "kube-system", }, } + getTenant := func(name string) *capsulev1beta2.Tenant { + tenant := &capsulev1beta2.Tenant{} + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: name}, tenant)).Should(Succeed()) + + return tenant + } + + getNamespace := func(name string) *corev1.Namespace { + ns := &corev1.Namespace{} + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: name}, ns)).Should(Succeed()) + + return ns + } + + hasTenantOwnerReference := func(ns *corev1.Namespace, tenant *capsulev1beta2.Tenant) bool { + for _, ownerRef := range ns.OwnerReferences { + if ownerRef.APIVersion == capsulev1beta2.GroupVersion.String() && + ownerRef.Kind == "Tenant" && + ownerRef.Name == tenant.GetName() && + ownerRef.UID == tenant.GetUID() { + return true + } + } + + return false + } + + hasTenantOwnerReferenceByNameAndUID := func(ns *corev1.Namespace, name string, uid types.UID) bool { + for _, ownerRef := range ns.OwnerReferences { + if ownerRef.APIVersion == capsulev1beta2.GroupVersion.String() && + ownerRef.Kind == "Tenant" && + ownerRef.Name == name && + ownerRef.UID == uid { + return true + } + } + + return false + } + + expectOriginalTenantOwnership := func(nsName string, tenant *capsulev1beta2.Tenant) { + retrievedNs := getNamespace(nsName) + + Expect(retrievedNs.Labels).To(HaveKeyWithValue(meta.TenantLabel, tenant.GetName())) + Expect(hasTenantOwnerReference(retrievedNs, tenant)).To(BeTrue(), "Namespace should keep original Tenant ownerReference") + } + + expectNoTenantOwnership := func(nsName string, tenant *capsulev1beta2.Tenant) { + retrievedNs := getNamespace(nsName) + + Expect(retrievedNs.Labels).NotTo(HaveKeyWithValue(meta.TenantLabel, tenant.GetName())) + Expect(hasTenantOwnerReference(retrievedNs, tenant)).To(BeFalse(), "Namespace should not have Tenant ownerReference") + } + + randomTenantReference := func() (string, types.UID) { + return fmt.Sprintf("random-tenant-%d", rand.Int()), types.UID(fmt.Sprintf("%d", rand.Int())) + } + JustBeforeEach(func() { - EventuallyCreation(func() (err error) { - tnt_1.ResourceVersion = "" - err = k8sClient.Create(context.TODO(), tnt_1) + EventuallyCreation(func() error { + t1.ResourceVersion = "" - return + return k8sClient.Create(context.TODO(), t1) }).Should(Succeed()) - }) - JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt_1)).Should(Succeed()) + TenantReady(t1, metav1.ConditionTrue, defaultTimeoutInterval) + EventuallyCreation(func() error { + t2.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), t2) + }).Should(Succeed()) + TenantReady(t2, metav1.ConditionTrue, defaultTimeoutInterval) + + EventuallyCreation(func() error { + t3.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), t3) + }).Should(Succeed()) + TenantReady(t3, metav1.ConditionTrue, defaultTimeoutInterval) + + }) + + JustAfterEach(func() { + EventuallyDeletion(t1) + EventuallyDeletion(t2) + EventuallyDeletion(t3) + }) + + It("Owners can not add a second Tenant ownerReference to a managed namespace", func() { + tenantA := getTenant(t1.Name) + tenantB := getTenant(t2.Name) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tenantA.GetName(), + }) + + NamespaceCreation(ns, owner.UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) + + current, err := cs.CoreV1().Namespaces().Get( + context.TODO(), + ns.Name, + metav1.GetOptions{}, + ) + Expect(err).ToNot(HaveOccurred()) + + current.OwnerReferences = append(current.OwnerReferences, metav1.OwnerReference{ + APIVersion: capsulev1beta2.GroupVersion.String(), + Kind: "Tenant", + Name: tenantB.GetName(), + UID: tenantB.GetUID(), + }) + + _, err = cs.CoreV1().Namespaces().Update( + context.TODO(), + current, + metav1.UpdateOptions{}, + ) + + Expect(err).To(HaveOccurred()) + + Eventually(func(g Gomega) { + updated := &corev1.Namespace{} + + err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: ns.Name}, + updated, + ) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(tenantOwnerReferences(updated)).To(Equal([]string{tenantA.GetName()})) + g.Expect(updated.Labels).To(HaveKeyWithValue(meta.TenantLabel, tenantA.GetName())) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + + It("Owners can not hijack unmanaged namespaces with multiple Tenant ownerReferences", func() { + tenantA := getTenant(t1.Name) + tenantB := getTenant(t2.Name) + + unmanaged := NewNamespace("") + Expect(k8sClient.Create(context.TODO(), unmanaged)).Should(Succeed()) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + patch := []byte(fmt.Sprintf(`{ + "metadata": { + "ownerReferences": [ + { + "apiVersion": "%s", + "kind": "Tenant", + "name": "%s", + "uid": "%s" + }, + { + "apiVersion": "%s", + "kind": "Tenant", + "name": "%s", + "uid": "%s" + } + ] + } + }`, + capsulev1beta2.GroupVersion.String(), + tenantA.GetName(), + tenantA.GetUID(), + capsulev1beta2.GroupVersion.String(), + tenantB.GetName(), + tenantB.GetUID(), + )) + + _, err := cs.CoreV1().Namespaces().Patch( + context.TODO(), + unmanaged.Name, + types.StrategicMergePatchType, + patch, + metav1.PatchOptions{}, + ) + + Expect(err).To(HaveOccurred()) + + Eventually(func(g Gomega) { + updated := &corev1.Namespace{} + + err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: unmanaged.Name}, + updated, + ) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(tenantOwnerReferences(updated)).To(BeEmpty()) + g.Expect(updated.Labels).ToNot(HaveKey(meta.TenantLabel)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + + It("Tenant A owners can not adopt unmanaged namespaces into Tenant B", func() { + tenantB := getTenant(t3.Name) + + unmanaged := NewNamespace("") + Expect(k8sClient.Create(context.TODO(), unmanaged)).Should(Succeed()) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + patch := []byte(fmt.Sprintf( + `{"metadata":{"labels":{"%s":"%s"},"ownerReferences":[{"apiVersion":"%s","kind":"Tenant","name":"%s","uid":"%s"}]}}`, + meta.TenantLabel, + tenantB.GetName(), + capsulev1beta2.GroupVersion.String(), + tenantB.GetName(), + tenantB.GetUID(), + )) + + _, err := cs.CoreV1().Namespaces().Patch( + context.TODO(), + unmanaged.Name, + types.StrategicMergePatchType, + patch, + metav1.PatchOptions{}, + ) + + Expect(err).To(HaveOccurred()) + expectNoTenantOwnership(unmanaged.Name, tenantB) + } + }) + + It("Owners can not hijack unmanaged namespaces with controller ownerReference flags", func() { + tenant := getTenant(t1.Name) + + unmanaged := NewNamespace("") + Expect(k8sClient.Create(context.TODO(), unmanaged)).Should(Succeed()) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + patch := []byte(fmt.Sprintf(`{ + "metadata":{ + "ownerReferences":[{ + "apiVersion":"%s", + "kind":"Tenant", + "name":"%s", + "uid":"%s", + "controller":true, + "blockOwnerDeletion":true + }] + } + }`, capsulev1beta2.GroupVersion.String(), tenant.GetName(), tenant.GetUID())) + + _, err := cs.CoreV1().Namespaces().Patch( + context.TODO(), + unmanaged.Name, + types.StrategicMergePatchType, + patch, + metav1.PatchOptions{}, + ) + + Expect(err).To(HaveOccurred()) + expectNoTenantOwnership(unmanaged.Name, tenant) + } + }) + + It("Owners can not smuggle Tenant ownerReference beside unrelated ownerReferences", func() { + tenant := getTenant(t1.Name) + + unmanaged := NewNamespace("") + Expect(k8sClient.Create(context.TODO(), unmanaged)).Should(Succeed()) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + patch := []byte(fmt.Sprintf(`{ + "metadata":{ + "ownerReferences":[ + {"apiVersion":"v1","kind":"ConfigMap","name":"dummy","uid":"%s"}, + {"apiVersion":"%s","kind":"Tenant","name":"%s","uid":"%s"} + ] + } + }`, types.UID("12345"), capsulev1beta2.GroupVersion.String(), tenant.GetName(), tenant.GetUID())) + + _, err := cs.CoreV1().Namespaces().Patch( + context.TODO(), + unmanaged.Name, + types.StrategicMergePatchType, + patch, + metav1.PatchOptions{}, + ) + + Expect(err).To(HaveOccurred()) + expectNoTenantOwnership(unmanaged.Name, tenant) + } + }) + + It("Owners can not create an ownership gap then patch managed namespace metadata", func() { + tenant := getTenant(t1.Name) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + ns := NewNamespace("", map[string]string{meta.TenantLabel: tenant.GetName()}) + NamespaceCreation(ns, owner.UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) + + remove := []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":null}}}`, meta.TenantLabel)) + _, err := cs.CoreV1().Namespaces().Patch( + context.TODO(), + ns.Name, + types.StrategicMergePatchType, + remove, + metav1.PatchOptions{}, + ) + Expect(err).ToNot(HaveOccurred()) + expectOriginalTenantOwnership(ns.Name, tenant) + + patch := []byte(`{"metadata":{"labels":{"attacker.example.com/touched":"true"}}}`) + _, err = cs.CoreV1().Namespaces().Patch( + context.TODO(), + ns.Name, + types.StrategicMergePatchType, + patch, + metav1.PatchOptions{}, + ) + Expect(err).ToNot(HaveOccurred()) + expectOriginalTenantOwnership(ns.Name, tenant) + } }) It("Can't hijack offlimits namespace (Ownerreferences)", func() { - tenant := &capsulev1beta2.Tenant{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt_1.Name}, tenant)).Should(Succeed()) + tenant := getTenant(t1.Name) - // Get the namespace Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: kubeSystem.GetName()}, kubeSystem)).Should(Succeed()) - for _, owner := range tnt_1.Spec.Owners { + for _, owner := range t1.Spec.Owners { cs := ownerClient(owner.UserSpec) - patch := []byte(fmt.Sprintf(`{"metadata":{"ownerReferences":[{"apiVersion":"%s/%s","kind":"Tenant","name":"%s","uid":"%s"}]}}`, capsulev1beta2.GroupVersion.Group, capsulev1beta2.GroupVersion.Version, tenant.GetName(), tenant.GetUID())) + patch := []byte(fmt.Sprintf( + `{"metadata":{"ownerReferences":[{"apiVersion":"%s","kind":"Tenant","name":"%s","uid":"%s"}]}}`, + capsulev1beta2.GroupVersion.String(), + tenant.GetName(), + tenant.GetUID(), + )) _, err := cs.CoreV1().Namespaces().Patch(context.TODO(), kubeSystem.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) Expect(err).To(HaveOccurred()) - } }) It("Can't hijack offlimits namespace (Labels)", func() { - tenant := &capsulev1beta2.Tenant{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt_1.Name}, tenant)).Should(Succeed()) + tenant := getTenant(t1.Name) - // Get the namespace Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: kubeSystem.GetName()}, kubeSystem)).Should(Succeed()) - for _, owner := range tnt_1.Spec.Owners { + for _, owner := range t1.Spec.Owners { cs := ownerClient(owner.UserSpec) - patch := []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, "capsule.clastix.io/tenant", tenant.GetName())) + patch := []byte(fmt.Sprintf( + `{"metadata":{"labels":{"%s":"%s"}}}`, + meta.TenantLabel, + tenant.GetName(), + )) _, err := cs.CoreV1().Namespaces().Patch(context.TODO(), kubeSystem.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) Expect(err).To(HaveOccurred()) @@ -100,63 +483,707 @@ var _ = Describe("creating several Namespaces for a Tenant", Label("namespace", }) It("Can't hijack offlimits namespace (Annotations)", func() { - tenant := &capsulev1beta2.Tenant{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt_1.Name}, tenant)).Should(Succeed()) + tenant := getTenant(t1.Name) - // Get the namespace Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: kubeSystem.GetName()}, kubeSystem)).Should(Succeed()) - for _, owner := range tnt_1.Spec.Owners { + for _, owner := range t1.Spec.Owners { cs := ownerClient(owner.UserSpec) - patch := []byte(fmt.Sprintf(`{"metadata":{"annotations":{"%s":"%s"}}}`, "capsule.clastix.io/tenant", tenant.GetName())) + patch := []byte(fmt.Sprintf( + `{"metadata":{"annotations":{"%s":"%s"}}}`, + meta.TenantLabel, + tenant.GetName(), + )) _, err := cs.CoreV1().Namespaces().Patch(context.TODO(), kubeSystem.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) Expect(err).To(HaveOccurred()) } }) - It("Owners can create and attempt to patch new namespaces but patches should not be applied", func() { - for _, owner := range tnt_1.Spec.Owners { + It("Owners can not hijack unmanaged namespaces using JSONPatch add label", func() { + tenant := getTenant(t1.Name) + + unmanaged := NewNamespace("") + Expect(k8sClient.Create(context.TODO(), unmanaged)).Should(Succeed()) + + for _, owner := range t1.Spec.Owners { cs := ownerClient(owner.UserSpec) - // Each owner creates a new namespace - ns := NewNamespace("") + patch := []byte(fmt.Sprintf(`[ + {"op":"add","path":"/metadata/labels","value":{}}, + {"op":"add","path":"/metadata/labels/%s","value":"%s"} + ]`, clt.EscapeJSONPointer(meta.TenantLabel), tenant.GetName())) + + _, err := cs.CoreV1().Namespaces().Patch( + context.TODO(), + unmanaged.Name, + types.JSONPatchType, + patch, + metav1.PatchOptions{}, + ) + + Expect(err).To(HaveOccurred()) + expectNoTenantOwnership(unmanaged.Name, tenant) + } + }) + + It("Owners can not hijack unmanaged namespaces using JSONPatch add ownerReference", func() { + tenant := getTenant(t1.Name) + + unmanaged := NewNamespace("") + Expect(k8sClient.Create(context.TODO(), unmanaged)).Should(Succeed()) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + patch := []byte(fmt.Sprintf(`[ + {"op":"add","path":"/metadata/ownerReferences","value":[ + {"apiVersion":"%s","kind":"Tenant","name":"%s","uid":"%s"} + ]} + ]`, capsulev1beta2.GroupVersion.String(), tenant.GetName(), tenant.GetUID())) + + _, err := cs.CoreV1().Namespaces().Patch( + context.TODO(), + unmanaged.Name, + types.JSONPatchType, + patch, + metav1.PatchOptions{}, + ) + + Expect(err).To(HaveOccurred()) + expectNoTenantOwnership(unmanaged.Name, tenant) + } + }) + + It("Owners can not hijack unmanaged namespaces with matching label and forged ownerReference UID", func() { + tenant := getTenant(t1.Name) + + unmanaged := NewNamespace("") + Expect(k8sClient.Create(context.TODO(), unmanaged)).Should(Succeed()) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + _, fakeUID := randomTenantReference() + patch := []byte(fmt.Sprintf( + `{"metadata":{"labels":{"%s":"%s"},"ownerReferences":[{"apiVersion":"%s","kind":"Tenant","name":"%s","uid":"%s"}]}}`, + meta.TenantLabel, + tenant.GetName(), + capsulev1beta2.GroupVersion.String(), + tenant.GetName(), + fakeUID, + )) + + _, err := cs.CoreV1().Namespaces().Patch( + context.TODO(), + unmanaged.Name, + types.StrategicMergePatchType, + patch, + metav1.PatchOptions{}, + ) + + Expect(err).To(HaveOccurred()) + expectNoTenantOwnership(unmanaged.Name, tenant) + } + }) + + It("Owners can not hijack unmanaged namespaces using server-side apply", func() { + tenant := getTenant(t1.Name) + + unmanaged := NewNamespace("") + Expect(k8sClient.Create(context.TODO(), unmanaged)).Should(Succeed()) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + apply := []byte(fmt.Sprintf(`{ + "apiVersion":"v1", + "kind":"Namespace", + "metadata":{ + "name":"%s", + "labels":{"%s":"%s"}, + "ownerReferences":[{ + "apiVersion":"%s", + "kind":"Tenant", + "name":"%s", + "uid":"%s" + }] + } + }`, + unmanaged.Name, + meta.TenantLabel, + tenant.GetName(), + capsulev1beta2.GroupVersion.String(), + tenant.GetName(), + tenant.GetUID(), + )) + + _, err := cs.CoreV1().Namespaces().Patch( + context.TODO(), + unmanaged.Name, + types.ApplyPatchType, + apply, + metav1.PatchOptions{ + FieldManager: "attacker", + Force: ptr.To(true), + }, + ) + + Expect(err).To(HaveOccurred()) + expectNoTenantOwnership(unmanaged.Name, tenant) + } + }) + + It("Owners can not combine status and metadata patches to adopt unmanaged namespaces", func() { + tenant := getTenant(t1.Name) + createNamespaceStatusRBACForOwner(tenant) + DeferCleanup(func(tnt *capsulev1beta2.Tenant) { + deleteNamespaceStatusRBACForOwner(tnt) + }, tenant) + + unmanaged := NewNamespace("") + Expect(k8sClient.Create(context.TODO(), unmanaged)).Should(Succeed()) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + statusNs, err := cs.CoreV1().Namespaces().Get(context.TODO(), unmanaged.Name, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + if statusNs.Labels == nil { + statusNs.Labels = map[string]string{} + } + + statusNs.Labels[meta.TenantLabel] = tenant.GetName() + statusNs.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: capsulev1beta2.GroupVersion.String(), + Kind: "Tenant", + Name: tenant.GetName(), + UID: tenant.GetUID(), + }} + + _, _ = cs.CoreV1().Namespaces().UpdateStatus(context.TODO(), statusNs, metav1.UpdateOptions{}) + + patch := []byte(fmt.Sprintf( + `{"metadata":{"labels":{"%s":"%s"}}}`, + meta.TenantLabel, + tenant.GetName(), + )) + + _, err = cs.CoreV1().Namespaces().Patch( + context.TODO(), + unmanaged.Name, + types.StrategicMergePatchType, + patch, + metav1.PatchOptions{}, + ) + + Expect(err).To(HaveOccurred()) + expectNoTenantOwnership(unmanaged.Name, tenant) + } + }) + + It("Owners can not hijack unmanaged namespaces with valid ownerReference and mismatching label", func() { + tenant := getTenant(t1.Name) + + unmanaged := NewNamespace("") + Expect(k8sClient.Create(context.TODO(), unmanaged)).Should(Succeed()) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + patch := []byte(fmt.Sprintf( + `{"metadata":{"labels":{"%s":"not-%s"},"ownerReferences":[{"apiVersion":"%s","kind":"Tenant","name":"%s","uid":"%s"}]}}`, + meta.TenantLabel, + tenant.GetName(), + capsulev1beta2.GroupVersion.String(), + tenant.GetName(), + tenant.GetUID(), + )) + + _, err := cs.CoreV1().Namespaces().Patch( + context.TODO(), + unmanaged.Name, + types.StrategicMergePatchType, + patch, + metav1.PatchOptions{}, + ) + + Expect(err).To(HaveOccurred()) + expectNoTenantOwnership(unmanaged.Name, tenant) + } + }) + + It("Owners can patch managed namespaces but ownerReference changes should be reverted", func() { + tenant := getTenant(t1.Name) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tenant.GetName(), + }) NamespaceCreation(ns, owner.UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) - // Attempt to patch the owner references of the new namespace - tenant := &capsulev1beta2.Tenant{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt_1.Name}, tenant)).Should(Succeed()) + randomName, randomUID := randomTenantReference() - randomUID := types.UID(fmt.Sprintf("%d", rand.Int())) - randomName := fmt.Sprintf("random-tenant-%d", rand.Int()) - patch := []byte(fmt.Sprintf(`{"metadata":{"ownerReferences":[{"apiVersion":"%s/%s","kind":"Tenant","name":"%s","uid":"%s"}]}}`, capsulev1beta2.GroupVersion.Group, capsulev1beta2.GroupVersion.Version, randomName, randomUID)) + patch := []byte(fmt.Sprintf( + `{"metadata":{"ownerReferences":[{"apiVersion":"%s","kind":"Tenant","name":"%s","uid":"%s"}]}}`, + capsulev1beta2.GroupVersion.String(), + randomName, + randomUID, + )) _, err := cs.CoreV1().Namespaces().Patch(context.TODO(), ns.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) Expect(err).ToNot(HaveOccurred()) - retrievedNs := &corev1.Namespace{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.Name}, retrievedNs)).Should(Succeed()) + retrievedNs := getNamespace(ns.Name) - // Check if the namespace has an owner reference with the specific UID and name - hasSpecificOwnerRef := false - for _, ownerRef := range retrievedNs.OwnerReferences { - if ownerRef.UID == randomUID && ownerRef.Name == randomName { - hasSpecificOwnerRef = true - break - } - } - Expect(hasSpecificOwnerRef).To(BeFalse(), "Namespace should not have owner reference with UID %s and name %s", randomUID, randomName) - - hasOriginReference := false - for _, ownerRef := range retrievedNs.OwnerReferences { - if ownerRef.UID == tenant.GetUID() && ownerRef.Name == tenant.GetName() { - hasOriginReference = true - break - } - } - Expect(hasOriginReference).To(BeTrue(), "Namespace should have origin reference", tenant.GetUID(), tenant.GetName()) + Expect(hasTenantOwnerReferenceByNameAndUID(retrievedNs, randomName, randomUID)).To(BeFalse(), "Namespace should not keep patched Tenant ownerReference") + Expect(hasTenantOwnerReference(retrievedNs, tenant)).To(BeTrue(), "Namespace should keep original Tenant ownerReference") } }) + It("Owners can patch managed namespaces but tenant label changes should be reverted", func() { + tenant := getTenant(t1.Name) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tenant.GetName(), + }) + NamespaceCreation(ns, owner.UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) + + randomName, _ := randomTenantReference() + + patch := []byte(fmt.Sprintf( + `{"metadata":{"labels":{"%s":"%s"}}}`, + meta.TenantLabel, + randomName, + )) + + _, err := cs.CoreV1().Namespaces().Patch(context.TODO(), ns.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + Expect(err).ToNot(HaveOccurred()) + + expectOriginalTenantOwnership(ns.Name, tenant) + } + }) + + It("Owners can patch managed namespaces but combined ownership changes should be reverted", func() { + tenant := getTenant(t1.Name) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tenant.GetName(), + }) + NamespaceCreation(ns, owner.UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) + + randomName, randomUID := randomTenantReference() + + patch := []byte(fmt.Sprintf( + `{"metadata":{"labels":{"%s":"%s"},"ownerReferences":[{"apiVersion":"%s","kind":"Tenant","name":"%s","uid":"%s"}]}}`, + meta.TenantLabel, + randomName, + capsulev1beta2.GroupVersion.String(), + randomName, + randomUID, + )) + + _, err := cs.CoreV1().Namespaces().Patch(context.TODO(), ns.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + Expect(err).ToNot(HaveOccurred()) + + retrievedNs := getNamespace(ns.Name) + + Expect(retrievedNs.Labels).To(HaveKeyWithValue(meta.TenantLabel, tenant.GetName())) + Expect(hasTenantOwnerReferenceByNameAndUID(retrievedNs, randomName, randomUID)).To(BeFalse()) + Expect(hasTenantOwnerReference(retrievedNs, tenant)).To(BeTrue()) + } + }) + + It("Owners can not migrate managed namespaces to another Tenant", func() { + tenantA := getTenant(t1.Name) + tenantB := getTenant(t2.Name) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tenantA.GetName(), + }) + NamespaceCreation(ns, owner.UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) + + patch := []byte(fmt.Sprintf( + `{"metadata":{"labels":{"%s":"%s"},"ownerReferences":[{"apiVersion":"%s","kind":"Tenant","name":"%s","uid":"%s"}]}}`, + meta.TenantLabel, + tenantB.GetName(), + capsulev1beta2.GroupVersion.String(), + tenantB.GetName(), + tenantB.GetUID(), + )) + + _, err := cs.CoreV1().Namespaces().Patch(context.TODO(), ns.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + Expect(err).ToNot(HaveOccurred()) + + retrievedNs := getNamespace(ns.Name) + + Expect(retrievedNs.Labels).To(HaveKeyWithValue(meta.TenantLabel, tenantA.GetName())) + Expect(hasTenantOwnerReference(retrievedNs, tenantA)).To(BeTrue()) + Expect(hasTenantOwnerReference(retrievedNs, tenantB)).To(BeFalse()) + } + }) + + It("Owners can not remove tenant ownership from managed namespaces", func() { + tenant := getTenant(t1.Name) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tenant.GetName(), + }) + NamespaceCreation(ns, owner.UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) + + patchRemoveOwnerReferences := []byte(`{"metadata":{"ownerReferences":[]}}`) + + _, err := cs.CoreV1().Namespaces().Patch(context.TODO(), ns.Name, types.StrategicMergePatchType, patchRemoveOwnerReferences, metav1.PatchOptions{}) + Expect(err).ToNot(HaveOccurred()) + + expectOriginalTenantOwnership(ns.Name, tenant) + + patchRemoveTenantLabel := []byte(fmt.Sprintf( + `{"metadata":{"labels":{"%s":null}}}`, + meta.TenantLabel, + )) + + _, err = cs.CoreV1().Namespaces().Patch(context.TODO(), ns.Name, types.StrategicMergePatchType, patchRemoveTenantLabel, metav1.PatchOptions{}) + Expect(err).ToNot(HaveOccurred()) + + expectOriginalTenantOwnership(ns.Name, tenant) + } + }) + + It("Owners can not patch unmanaged namespaces into a Tenant", func() { + tenant := getTenant(t1.Name) + + unmanaged := NewNamespace("") + Expect(k8sClient.Create(context.TODO(), unmanaged)).Should(Succeed()) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + patchLabel := []byte(fmt.Sprintf( + `{"metadata":{"labels":{"%s":"%s"}}}`, + meta.TenantLabel, + tenant.GetName(), + )) + + _, err := cs.CoreV1().Namespaces().Patch(context.TODO(), unmanaged.Name, types.StrategicMergePatchType, patchLabel, metav1.PatchOptions{}) + Expect(err).To(HaveOccurred()) + + patchOwnerReference := []byte(fmt.Sprintf( + `{"metadata":{"ownerReferences":[{"apiVersion":"%s","kind":"Tenant","name":"%s","uid":"%s"}]}}`, + capsulev1beta2.GroupVersion.String(), + tenant.GetName(), + tenant.GetUID(), + )) + + _, err = cs.CoreV1().Namespaces().Patch(context.TODO(), unmanaged.Name, types.StrategicMergePatchType, patchOwnerReference, metav1.PatchOptions{}) + Expect(err).To(HaveOccurred()) + + expectNoTenantOwnership(unmanaged.Name, tenant) + } + }) + + It("Namespace status updates by owners can not change tenant ownerReferences", func() { + tenant := getTenant(t1.Name) + + createNamespaceStatusRBACForOwner(tenant) + DeferCleanup(func(tnt *capsulev1beta2.Tenant) { + deleteNamespaceStatusRBACForOwner(tnt) + }, tenant) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tenant.GetName(), + }) + NamespaceCreation(ns, owner.UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) + + randomName, randomUID := randomTenantReference() + + statusNs, err := cs.CoreV1().Namespaces().Get(context.TODO(), ns.Name, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + statusNs.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: capsulev1beta2.GroupVersion.String(), + Kind: "Tenant", + Name: randomName, + UID: randomUID, + }, + } + + _, err = cs.CoreV1().Namespaces().UpdateStatus(context.TODO(), statusNs, metav1.UpdateOptions{}) + if err != nil { + expectOriginalTenantOwnership(ns.Name, tenant) + + continue + } + + retrievedNs := getNamespace(ns.Name) + + Expect(hasTenantOwnerReferenceByNameAndUID(retrievedNs, randomName, randomUID)).To(BeFalse(), "Namespace status update must not change Tenant ownerReference") + Expect(hasTenantOwnerReference(retrievedNs, tenant)).To(BeTrue(), "Namespace should keep original Tenant ownerReference") + } + }) + + It("Namespace status updates by owners can not change tenant labels", func() { + tenant := getTenant(t1.Name) + + createNamespaceStatusRBACForOwner(tenant) + DeferCleanup(func(tnt *capsulev1beta2.Tenant) { + deleteNamespaceStatusRBACForOwner(tnt) + }, tenant) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tenant.GetName(), + }) + NamespaceCreation(ns, owner.UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) + + randomName, _ := randomTenantReference() + + statusNs, err := cs.CoreV1().Namespaces().Get(context.TODO(), ns.Name, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + if statusNs.Labels == nil { + statusNs.Labels = map[string]string{} + } + + statusNs.Labels[meta.TenantLabel] = randomName + + _, err = cs.CoreV1().Namespaces().UpdateStatus(context.TODO(), statusNs, metav1.UpdateOptions{}) + if err != nil { + expectOriginalTenantOwnership(ns.Name, tenant) + + continue + } + + expectOriginalTenantOwnership(ns.Name, tenant) + } + }) + + It("Namespace status updates by owners can not migrate namespaces to another Tenant", func() { + tenantA := getTenant(t1.Name) + tenantB := getTenant(t2.Name) + + createNamespaceStatusRBACForOwner(tenantA) + DeferCleanup(func(tnt *capsulev1beta2.Tenant) { + deleteNamespaceStatusRBACForOwner(tnt) + }, tenantA) + + createNamespaceStatusRBACForOwner(tenantB) + DeferCleanup(func(tnt *capsulev1beta2.Tenant) { + deleteNamespaceStatusRBACForOwner(tnt) + }, tenantB) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tenantA.GetName(), + }) + NamespaceCreation(ns, owner.UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) + + statusNs, err := cs.CoreV1().Namespaces().Get(context.TODO(), ns.Name, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + if statusNs.Labels == nil { + statusNs.Labels = map[string]string{} + } + + statusNs.Labels[meta.TenantLabel] = tenantB.GetName() + statusNs.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: capsulev1beta2.GroupVersion.String(), + Kind: "Tenant", + Name: tenantB.GetName(), + UID: tenantB.GetUID(), + }, + } + + _, err = cs.CoreV1().Namespaces().UpdateStatus(context.TODO(), statusNs, metav1.UpdateOptions{}) + if err != nil { + retrievedNs := getNamespace(ns.Name) + + Expect(retrievedNs.Labels).To(HaveKeyWithValue(meta.TenantLabel, tenantA.GetName())) + Expect(hasTenantOwnerReference(retrievedNs, tenantA)).To(BeTrue()) + Expect(hasTenantOwnerReference(retrievedNs, tenantB)).To(BeFalse()) + + continue + } + + retrievedNs := getNamespace(ns.Name) + + Expect(retrievedNs.Labels).To(HaveKeyWithValue(meta.TenantLabel, tenantA.GetName())) + Expect(hasTenantOwnerReference(retrievedNs, tenantA)).To(BeTrue()) + Expect(hasTenantOwnerReference(retrievedNs, tenantB)).To(BeFalse()) + } + }) + + It("Namespace status updates by owners can not patch unmanaged namespaces into a Tenant", func() { + tenant := getTenant(t1.Name) + + createNamespaceStatusRBACForOwner(tenant) + DeferCleanup(func(tnt *capsulev1beta2.Tenant) { + deleteNamespaceStatusRBACForOwner(tnt) + }, tenant) + + unmanaged := NewNamespace("") + Expect(k8sClient.Create(context.TODO(), unmanaged)).Should(Succeed()) + + for _, owner := range t1.Spec.Owners { + cs := ownerClient(owner.UserSpec) + + statusNs, err := cs.CoreV1().Namespaces().Get(context.TODO(), unmanaged.Name, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + if statusNs.Labels == nil { + statusNs.Labels = map[string]string{} + } + + statusNs.Labels[meta.TenantLabel] = tenant.GetName() + statusNs.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: capsulev1beta2.GroupVersion.String(), + Kind: "Tenant", + Name: tenant.GetName(), + UID: tenant.GetUID(), + }, + } + + _, err = cs.CoreV1().Namespaces().UpdateStatus(context.TODO(), statusNs, metav1.UpdateOptions{}) + if err != nil { + expectNoTenantOwnership(unmanaged.Name, tenant) + + continue + } + + expectNoTenantOwnership(unmanaged.Name, tenant) + } + }) }) + +func createNamespaceStatusRBACForOwner(tnt *capsulev1beta2.Tenant) { + name := "namespace-status-patch-" + tnt.GetName() + + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{ + "namespaces", + "namespaces/status", + }, + Verbs: []string{ + "get", + "patch", + "update", + }, + }, + }, + } + + err := k8sClient.Create(context.TODO(), clusterRole) + if err != nil && !apierrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + + clusterRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: name, + }, + } + + for _, sub := range tnt.Spec.Owners { + if sub.Kind != rbac.ServiceAccountOwner { + clusterRoleBinding.Subjects = append(clusterRoleBinding.Subjects, rbacv1.Subject{ + Kind: string(sub.Kind), + Name: sub.Name, + }) + } else { + namespace, name, err := serviceaccount.SplitUsername(sub.Name) + Expect(err).NotTo(HaveOccurred()) + + clusterRoleBinding.Subjects = append(clusterRoleBinding.Subjects, rbacv1.Subject{ + Kind: string(sub.Kind), + Name: name, + Namespace: namespace, + }) + } + } + + err = k8sClient.Create(context.TODO(), clusterRoleBinding) + if err != nil && !apierrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } +} + +func deleteNamespaceStatusRBACForOwner(tnt *capsulev1beta2.Tenant) { + name := "namespace-status-patch-" + tnt.GetName() + + err := k8sClient.Delete(context.TODO(), &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + }) + if err != nil && !apierrors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred()) + } + + err = k8sClient.Delete(context.TODO(), &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + }) + if err != nil && !apierrors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred()) + } +} + +func tenantOwnerReferences(ns *corev1.Namespace) []string { + var refs []string + + for _, ref := range ns.GetOwnerReferences() { + if tenant.IsTenantOwnerReference(ref) { + refs = append(refs, ref.Name) + } + } + + sort.Strings(refs) + + return refs +} diff --git a/e2e/namespace_metadata_controller_test.go b/e2e/namespace_metadata_controller_test.go index b43c015c..28e076ed 100644 --- a/e2e/namespace_metadata_controller_test.go +++ b/e2e/namespace_metadata_controller_test.go @@ -8,17 +8,24 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespace for a Tenant with additional metadata", Label("namespace"), func() { +var _ = Describe("creating a Namespace for a Tenant with additional metadata", Ordered, Label("namespace", "metadata"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-metadata-controller", + Name: "e2e-tenant-metadata", + Labels: map[string]string{ + "env": "e2e", + }, + OwnerReferences: []metav1.OwnerReference{ { APIVersion: "cap", @@ -29,11 +36,11 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "gatsby", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-metadata", Kind: "User", }, }, @@ -58,43 +65,85 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should contain Namespace metadata after tenant update", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) By("checking labels", func() { Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) Expect(ns.Labels).ShouldNot(HaveKeyWithValue("newlabel", "foobazbar")) - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, tnt)).Should(Succeed()) - tnt.Spec.NamespaceOptions.AdditionalMetadata.Labels["newlabel"] = "foobazbar" - Expect(k8sClient.Update(context.TODO(), tnt)).Should(Succeed()) + Eventually(func() error { + current := &capsulev1beta2.Tenant{} + if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, current); err != nil { + return err + } - Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) - ok, _ = Equal(ns.Labels["newlabel"]).Match("foobazbar") - return - }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + if current.Spec.NamespaceOptions == nil { + current.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{} + } + + if current.Spec.NamespaceOptions.AdditionalMetadata == nil { + current.Spec.NamespaceOptions.AdditionalMetadata = &api.AdditionalMetadataSpec{} + } + + if current.Spec.NamespaceOptions.AdditionalMetadata.Labels == nil { + current.Spec.NamespaceOptions.AdditionalMetadata.Labels = map[string]string{} + } + + current.Spec.NamespaceOptions.AdditionalMetadata.Labels["newlabel"] = "foobazbar" + + return k8sClient.Update(context.TODO(), current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Eventually(func(g Gomega) { + current := &corev1.Namespace{} + g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, current)).Should(Succeed()) + g.Expect(current.Labels).Should(HaveKeyWithValue("newlabel", "foobazbar")) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) + By("checking annotations", func() { Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) - Expect(ns.Labels).ShouldNot(HaveKeyWithValue("newannotation", "foobazbar")) + Expect(ns.Annotations).ShouldNot(HaveKeyWithValue("newannotation", "foobazbar")) - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, tnt)).Should(Succeed()) - tnt.Spec.NamespaceOptions.AdditionalMetadata.Annotations["newannotation"] = "foobazbar" - Expect(k8sClient.Update(context.TODO(), tnt)).Should(Succeed()) + Eventually(func() error { + current := &capsulev1beta2.Tenant{} + if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, current); err != nil { + return err + } - Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) - ok, _ = Equal(ns.Annotations["newannotation"]).Match("foobazbar") - return - }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + if current.Spec.NamespaceOptions == nil { + current.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{} + } + + if current.Spec.NamespaceOptions.AdditionalMetadata == nil { + current.Spec.NamespaceOptions.AdditionalMetadata = &api.AdditionalMetadataSpec{} + } + + if current.Spec.NamespaceOptions.AdditionalMetadata.Annotations == nil { + current.Spec.NamespaceOptions.AdditionalMetadata.Annotations = map[string]string{} + } + + current.Spec.NamespaceOptions.AdditionalMetadata.Annotations["newannotation"] = "foobazbar" + + return k8sClient.Update(context.TODO(), current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Eventually(func(g Gomega) { + current := &corev1.Namespace{} + g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, current)).Should(Succeed()) + g.Expect(current.Annotations).Should(HaveKeyWithValue("newannotation", "foobazbar")) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) }) }) diff --git a/e2e/namespace_user_metadata_test.go b/e2e/namespace_metadata_forbidden_test.go similarity index 72% rename from e2e/namespace_user_metadata_test.go rename to e2e/namespace_metadata_forbidden_test.go index c9e92d7a..4d143189 100644 --- a/e2e/namespace_user_metadata_test.go +++ b/e2e/namespace_metadata_forbidden_test.go @@ -15,12 +15,17 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespace with user-specified labels and annotations", Label("namespace"), func() { +var _ = Describe("creating a Namespace with user-specified labels and annotations", Ordered, Label("namespace", "metadata", "forbidden"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-user-metadata-forbidden", + Name: "e2e-user-metadata-forbidden", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ NamespaceOptions: &capsulev1beta2.NamespaceOptions{ @@ -33,11 +38,11 @@ var _ = Describe("creating a Namespace with user-specified labels and annotation Regex: "^gatsby-.*$", }, }, - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "gatsby", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-user-metadata-forbidden", Kind: "User", }, }, @@ -51,44 +56,68 @@ var _ = Describe("creating a Namespace with user-specified labels and annotation tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should allow", func() { By("specifying non-forbidden labels", func() { - ns := NewNamespace("") - ns.SetLabels(map[string]string{"bim": "baz"}) + ns := NewNamespace("", map[string]string{ + "bim": "baz", + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + }) By("specifying non-forbidden annotations", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) ns.SetAnnotations(map[string]string{"bim": "baz"}) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + }) }) It("should fail when creating a Namespace", func() { By("specifying forbidden labels using exact match", func() { - ns := NewNamespace("") - ns.SetLabels(map[string]string{"foo": "bar"}) + ns := NewNamespace("", map[string]string{ + "foo": "bar", + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + NamespaceIsNotPartOfTenant(tnt, ns).Should(Succeed()) + }) By("specifying forbidden labels using regex match", func() { - ns := NewNamespace("") - ns.SetLabels(map[string]string{"gatsby-foo": "bar"}) + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "gatsby-foo": "bar", + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + NamespaceIsNotPartOfTenant(tnt, ns).Should(Succeed()) }) By("specifying forbidden annotations using exact match", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) ns.SetAnnotations(map[string]string{"foo": "bar"}) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + NamespaceIsNotPartOfTenant(tnt, ns).Should(Succeed()) + }) By("specifying forbidden annotations using regex match", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) ns.SetAnnotations(map[string]string{"gatsby-foo": "bar"}) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + NamespaceIsNotPartOfTenant(tnt, ns).Should(Succeed()) + }) }) @@ -137,9 +166,12 @@ var _ = Describe("creating a Namespace with user-specified labels and annotation cs := ownerClient(tnt.Spec.Owners[0].UserSpec) By("specifying forbidden labels using exact match", func() { - ns := NewNamespace("forbidden-labels-exact-match") - + ns := NewNamespace("forbidden-labels-exact-match", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + rbacPatch(ns.GetName()) Consistently(func() error { if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: ns.GetName()}, ns); err != nil { @@ -154,9 +186,12 @@ var _ = Describe("creating a Namespace with user-specified labels and annotation }, 10*time.Second, time.Second).ShouldNot(Succeed()) }) By("specifying forbidden labels using regex match", func() { - ns := NewNamespace("forbidden-labels-regex-match") - + ns := NewNamespace("forbidden-labels-regex-match", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + rbacPatch(ns.GetName()) Consistently(func() error { if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: ns.GetName()}, ns); err != nil { @@ -171,9 +206,12 @@ var _ = Describe("creating a Namespace with user-specified labels and annotation }, 3*time.Second, time.Second).ShouldNot(Succeed()) }) By("specifying forbidden annotations using exact match", func() { - ns := NewNamespace("forbidden-annotations-exact-match") - + ns := NewNamespace("forbidden-annotations-exact-match", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + rbacPatch(ns.GetName()) Consistently(func() error { if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: ns.GetName()}, ns); err != nil { @@ -188,9 +226,12 @@ var _ = Describe("creating a Namespace with user-specified labels and annotation }, 10*time.Second, time.Second).ShouldNot(Succeed()) }) By("specifying forbidden annotations using regex match", func() { - ns := NewNamespace("forbidden-annotations-regex-match") - + ns := NewNamespace("forbidden-annotations-regex-match", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + rbacPatch(ns.GetName()) Consistently(func() error { if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: ns.GetName()}, ns); err != nil { diff --git a/e2e/namespace_required_metadata_test.go b/e2e/namespace_metadata_required_test.go similarity index 71% rename from e2e/namespace_required_metadata_test.go rename to e2e/namespace_metadata_required_test.go index 3e08137c..ab4b08e2 100644 --- a/e2e/namespace_required_metadata_test.go +++ b/e2e/namespace_metadata_required_test.go @@ -12,20 +12,24 @@ import ( "k8s.io/apimachinery/pkg/types" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespace for a Tenant with required metadata", Label("namespace", "metadata", "me"), func() { +var _ = Describe("creating a Namespace for a Tenant with required metadata", Ordered, Label("namespace", "metadata", "forbidden"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-metadata-required", + Name: "e2e-metadata-required", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "gatsby", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-metadata-required", Kind: "User", }, }, @@ -48,39 +52,46 @@ var _ = Describe("creating a Namespace for a Tenant with required metadata", Lab EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should contain required Namespace metadata", func() { By("creating without required label", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).ShouldNot(ContainElement(ns.GetName())) + NamespaceIsNotPartOfTenant(tnt, ns).Should(Succeed()) + }) By("creating with required label, without annotation", func() { ns := NewNamespace("", map[string]string{ - "environment": "prod", + "environment": "prod", + meta.TenantLabel: tnt.GetName(), }) ns.SetAnnotations(map[string]string{}) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).ShouldNot(ContainElement(ns.GetName())) + NamespaceIsNotPartOfTenant(tnt, ns).Should(Succeed()) + }) By("creating with required label and annotation", func() { ns := NewNamespace("", map[string]string{ - "environment": "prod", + "environment": "prod", + meta.TenantLabel: tnt.GetName(), }) ns.SetAnnotations(map[string]string{ "example.corp/cost-center": "INV-1234", }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) ns.SetLabels(map[string]string{ @@ -100,27 +111,28 @@ var _ = Describe("creating a Namespace for a Tenant with required metadata", Lab By("creating with required label (wrong value) and annotation", func() { ns := NewNamespace("", map[string]string{ - "environment": "UAT", + "environment": "UAT", + meta.TenantLabel: tnt.GetName(), }) ns.SetAnnotations(map[string]string{ "example.corp/cost-center": "INV-1234", }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).ShouldNot(ContainElement(ns.GetName())) + NamespaceIsNotPartOfTenant(tnt, ns).Should(Succeed()) }) By("creating with required label and annotation (wrong value)", func() { ns := NewNamespace("", map[string]string{ - "environment": "prod", + "environment": "prod", + meta.TenantLabel: tnt.GetName(), }) ns.SetAnnotations(map[string]string{ "example.corp/cost-center": "INV-1", }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).ShouldNot(ContainElement(ns.GetName())) + NamespaceIsNotPartOfTenant(tnt, ns).Should(Succeed()) }) - }) }) diff --git a/e2e/namespace_additional_metadata_test.go b/e2e/namespace_metadata_test.go similarity index 58% rename from e2e/namespace_additional_metadata_test.go rename to e2e/namespace_metadata_test.go index 6da9e40f..d44af2a2 100644 --- a/e2e/namespace_additional_metadata_test.go +++ b/e2e/namespace_metadata_test.go @@ -11,17 +11,20 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespace for a Tenant with additional metadata", Label("namespace", "metadata"), func() { +var _ = Describe("creating a Namespace for a Tenant with additional metadata", Ordered, Label("namespace", "metadata"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-metadata", + Name: "e2e-tenant-metadata-admission", + Labels: map[string]string{ + "env": "e2e", + }, OwnerReferences: []metav1.OwnerReference{ { APIVersion: "cap", @@ -32,11 +35,11 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "gatsby", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-metadata-admission", Kind: "User", }, }, @@ -46,7 +49,45 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L } JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) + }) + + It("should reject invalid additional metadata on tenant create", func() { + tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-invalid-metadata-create", + Labels: map[string]string{ + "env": "e2e", + }, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{ + { + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-invalid-metadata-create", + Kind: "User", + }, + }, + }, + }, + NamespaceOptions: &capsulev1beta2.NamespaceOptions{ + ManagedMetadataOnly: false, + AdditionalMetadataList: []api.AdditionalMetadataSelectorSpec{ + { + Labels: map[string]string{ + "clastix.io???custom-label": "bar", + }, + }, + }, + }, + }, + } + + Eventually(func() error { + tnt.ResourceVersion = "" + return k8sClient.Create(context.TODO(), tnt) + }, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed()) }) It("should contain additional Namespace metadata", func() { @@ -70,17 +111,20 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, tnt)).Should(Succeed()) + tnt = GetTenantEventually(tnt) }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) By("checking additional labels", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) for k, v := range tnt.Spec.NamespaceOptions.AdditionalMetadata.Labels { if k == "capsule.clastix.io/tenant" || k == "kubernetes.io/metadata.name" { continue // this label is managed and shouldn't be set by the user @@ -94,7 +138,7 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }) By("checking managed labels", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) if ok, _ = HaveKeyWithValue("capsule.clastix.io/tenant", tnt.GetName()).Match(ns.Labels); !ok { return } @@ -107,7 +151,7 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L By("checking additional annotations", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) for k, v := range tnt.Spec.NamespaceOptions.AdditionalMetadata.Annotations { if ok, _ = HaveKeyWithValue(k, v).Match(ns.Annotations); !ok { return @@ -184,19 +228,19 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, tnt)).Should(Succeed()) + tnt = GetTenantEventually(tnt) }) - labels := map[string]string{ + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), "matching_namespace_label": "matching_namespace_label_value", - } - ns := NewNamespace("", labels) + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) By("checking templated annotations", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) if ok, _ = HaveKeyWithValue("projectcapsule.dev/templated-tenant-annotation", tnt.Name).Match(ns.Annotations); !ok { return } @@ -208,7 +252,7 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }) By("checking templated labels", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) if ok, _ = HaveKeyWithValue("projectcapsule.dev/templated-tenant-label", tnt.Name).Match(ns.Labels); !ok { return } @@ -220,7 +264,7 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }) By("checking additional labels from entry without node selector", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) for k, v := range tnt.Spec.NamespaceOptions.AdditionalMetadataList[0].Labels { if ok, _ = HaveKeyWithValue(k, v).Match(ns.Labels); !ok { return @@ -231,7 +275,7 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }) By("checking additional labels from entry with matching node selector", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) for k, v := range tnt.Spec.NamespaceOptions.AdditionalMetadataList[1].Labels { if ok, _ = HaveKeyWithValue(k, v).Match(ns.Labels); !ok { return @@ -242,7 +286,7 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }) By("checking additional labels from entry with non-matching node selector", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) for k, v := range tnt.Spec.NamespaceOptions.AdditionalMetadataList[2].Labels { if ok, _ = HaveKeyWithValue(k, v).Match(ns.Labels); !ok { return @@ -253,7 +297,7 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }) By("checking additional annotations from entry without node selector", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) for k, v := range tnt.Spec.NamespaceOptions.AdditionalMetadataList[0].Annotations { if ok, _ = HaveKeyWithValue(k, v).Match(ns.Annotations); !ok { return @@ -264,7 +308,7 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }) By("checking additional annotations from entry with matching node selector", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) for k, v := range tnt.Spec.NamespaceOptions.AdditionalMetadataList[1].Annotations { if ok, _ = HaveKeyWithValue(k, v).Match(ns.Annotations); !ok { return @@ -275,7 +319,7 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }) By("checking additional annotations from entry with non-matching node selector", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) for k, v := range tnt.Spec.NamespaceOptions.AdditionalMetadataList[2].Annotations { if ok, _ = HaveKeyWithValue(k, v).Match(ns.Annotations); !ok { return @@ -317,17 +361,16 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, tnt)).Should(Succeed()) }) - labels := map[string]string{ + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), "matching_namespace_label": "matching_namespace_label_value", - } - - ns := NewNamespace("", labels) + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) By("checking additional labels", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) for _, mv := range tnt.Spec.NamespaceOptions.AdditionalMetadataList { for k, v := range mv.Labels { if k == "capsule.clastix.io/tenant" || k == "kubernetes.io/metadata.name" { @@ -344,7 +387,7 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }) By("checking managed labels", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) if ok, _ = HaveKeyWithValue("capsule.clastix.io/tenant", tnt.GetName()).Match(ns.Labels); !ok { return } @@ -357,7 +400,7 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L By("checking additional annotations", func() { Eventually(func() (ok bool) { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) for _, mv := range tnt.Spec.NamespaceOptions.AdditionalMetadataList { for k, v := range mv.Annotations { if ok, _ = HaveKeyWithValue(k, v).Match(ns.Annotations); !ok { @@ -371,20 +414,25 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }) By("patching labels and annotations on the Namespace", func() { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).To(Succeed()) + PatchNamespaceEventually(ns, func(current *corev1.Namespace) { + if current.Labels == nil { + current.Labels = map[string]string{} + } + if current.Annotations == nil { + current.Annotations = map[string]string{} + } - before := ns.DeepCopy() - ns.Labels["test-label"] = "test-value" - ns.Labels["k8s.io/custom-label"] = "foo-value" - ns.Annotations["test-annotation"] = "test-value" - ns.Annotations["k8s.io/custom-annotation"] = "bizz-value" + current.Labels["test-label"] = "test-value" + current.Labels["k8s.io/custom-label"] = "foo-value" + current.Annotations["test-annotation"] = "test-value" + current.Annotations["k8s.io/custom-annotation"] = "bizz-value" + }) - Expect(k8sClient.Patch(context.TODO(), ns, client.MergeFrom(before))).To(Succeed()) - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, tnt)).Should(Succeed()) + tnt = GetTenantEventually(tnt) }) By("Add additional annotations (Tenant Owner)", func() { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) + ns = GetNamespaceEventually(ns.GetName()) expectedLabels := map[string]string{ "test-label": "test-value", @@ -426,60 +474,71 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }, defaultTimeoutInterval, defaultPollInterval).Should(Equal(expectedAnnotations)) By("verify tenant status", func() { - condition := tnt.Status.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + Eventually(func(g Gomega) { + tnt = GetTenantEventually(tnt) - Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected tenant condition status to be True") - Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected tenant condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected tenant condition reason to be Succeeded") + condition := tnt.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected tenant condition status to be True") + g.Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected tenant condition type to be Ready") + g.Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected tenant condition reason to be Succeeded") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("verify namespace status", func() { - instance := tnt.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{Name: ns.GetName(), UID: ns.GetUID()}) - Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") + Eventually(func(g Gomega) { + currentTenant := GetTenantEventually(tnt) + currentNamespace := GetNamespaceEventually(ns.GetName()) - condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + instance := currentTenant.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ + Name: currentNamespace.GetName(), + UID: currentNamespace.GetUID(), + }) + g.Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") - Expect(instance.Name).To(Equal(ns.GetName())) - Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected namespace condition status to be True") - Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected namespace condition reason to be Succeeded") + condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") - expectedMetadata := &capsulev1beta2.TenantStatusNamespaceMetadata{ - Labels: map[string]string{ - "clastix.io/custom-label": "bar", - "k8s.io/custom-label": "foo", - }, - Annotations: map[string]string{ - "clastix.io/custom-annotation": "buzz", - "k8s.io/custom-annotation": "bizz", - }, - } + g.Expect(instance.Name).To(Equal(currentNamespace.GetName())) + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(condition.Type).To(Equal(meta.ReadyCondition)) + g.Expect(condition.Reason).To(Equal(meta.SucceededReason)) - Expect(instance.Metadata).To(Equal(expectedMetadata)) + expectedMetadata := &capsulev1beta2.TenantStatusNamespaceMetadata{ + Labels: map[string]string{ + "clastix.io/custom-label": "bar", + "k8s.io/custom-label": "foo", + }, + Annotations: map[string]string{ + "clastix.io/custom-annotation": "buzz", + "k8s.io/custom-annotation": "bizz", + }, + } + + g.Expect(instance.Metadata).To(Equal(expectedMetadata)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) }) By("change managed additional metadata", func() { - tnt.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{ - ManagedMetadataOnly: false, - AdditionalMetadataList: []api.AdditionalMetadataSelectorSpec{ - { - Labels: map[string]string{ - "clastix.io/custom-label": "bar", + UpdateTenantEventually(tnt, func(t *capsulev1beta2.Tenant) { + t.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{ + ManagedMetadataOnly: false, + AdditionalMetadataList: []api.AdditionalMetadataSelectorSpec{ + { + Labels: map[string]string{ + "clastix.io/custom-label": "bar", + }, + }, + { + Annotations: map[string]string{ + "k8s.io/custom-annotation": "bizz", + }, }, }, - { - Annotations: map[string]string{ - "k8s.io/custom-annotation": "bizz", - }, - }, - }, - } - - Expect(k8sClient.Update(context.TODO(), tnt)).Should(Succeed()) - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, tnt)).Should(Succeed()) + } + }) }) By("verify metadata lifecycle (valid update)", func() { @@ -522,54 +581,79 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }, defaultTimeoutInterval, defaultPollInterval).Should(Equal(expectedAnnotations)) By("verify tenant status", func() { - condition := tnt.Status.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + Eventually(func(g Gomega) { + current := &capsulev1beta2.Tenant{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, current) + g.Expect(err).NotTo(HaveOccurred()) - Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected tenant condition status to be True") - Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected tenant condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected tenant condition reason to be Succeeded") + condition := current.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected tenant condition status to be True") + g.Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected tenant condition type to be Ready") + g.Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected tenant condition reason to be Succeeded") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("verify namespace status", func() { - instance := tnt.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{Name: ns.GetName(), UID: ns.GetUID()}) - Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") + Eventually(func(g Gomega) { + current := &capsulev1beta2.Tenant{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, current) + g.Expect(err).NotTo(HaveOccurred()) - condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + currentNS := &corev1.Namespace{} + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, currentNS) + g.Expect(err).NotTo(HaveOccurred()) - Expect(instance.Name).To(Equal(ns.GetName())) - Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected namespace condition status to be True") - Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected namespace condition reason to be Succeeded") + instance := current.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ + Name: currentNS.GetName(), + UID: currentNS.GetUID(), + }) + g.Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") - expectedMetadata := &capsulev1beta2.TenantStatusNamespaceMetadata{ - Labels: map[string]string{ - "clastix.io/custom-label": "bar", - }, - Annotations: map[string]string{ - "k8s.io/custom-annotation": "bizz", - }, - } + condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") - Expect(instance.Metadata).To(Equal(expectedMetadata)) + g.Expect(instance.Name).To(Equal(currentNS.GetName())) + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected namespace condition status to be True") + g.Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") + g.Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected namespace condition reason to be Succeeded") + + expectedMetadata := &capsulev1beta2.TenantStatusNamespaceMetadata{ + Labels: map[string]string{ + "clastix.io/custom-label": "bar", + }, + Annotations: map[string]string{ + "k8s.io/custom-annotation": "bizz", + }, + } + + g.Expect(instance.Metadata).To(Equal(expectedMetadata)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) }) By("change managed additional metadata (provoke an error)", func() { - tnt.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{ - ManagedMetadataOnly: false, - AdditionalMetadataList: []api.AdditionalMetadataSelectorSpec{ - { - Labels: map[string]string{ - "clastix.io???custom-label": "bar", + Eventually(func() error { + t := &capsulev1beta2.Tenant{} + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).To(Succeed()) + + t.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{ + ManagedMetadataOnly: false, + AdditionalMetadataList: []api.AdditionalMetadataSelectorSpec{ + { + Labels: map[string]string{ + "clastix.io???custom-label": "bar", + }, }, }, - }, - } + } - Expect(k8sClient.Update(context.TODO(), tnt)).Should(Succeed()) - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, tnt)).Should(Succeed()) + return k8sClient.Update(context.TODO(), t) + }, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed()) + + TenantReadyTrue(tnt) }) By("verify metadata lifecycle (faulty update)", func() { @@ -612,47 +696,66 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }, defaultTimeoutInterval, defaultPollInterval).Should(Equal(expectedAnnotations)) By("verify tenant status", func() { - condition := tnt.Status.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + Eventually(func(g Gomega) { + t := &capsulev1beta2.Tenant{} - Expect(condition.Status).To(Equal(metav1.ConditionFalse), "Expected tenant condition status to be True") - Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected tenant condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.FailedReason), "Expected tenant condition reason to be Succeeded") + g.Expect(k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tnt.GetName()}, + t, + )).To(Succeed()) + + condition := t.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + + g.Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected tenant condition type to be Ready") + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected tenant condition status to be False") + g.Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected tenant condition reason to be Failed") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("verify namespace status", func() { - instance := tnt.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{Name: ns.GetName(), UID: ns.GetUID()}) - Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") + Eventually(func(g Gomega) { + t := &capsulev1beta2.Tenant{} + g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).To(Succeed()) - condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + instance := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{Name: ns.GetName(), UID: ns.GetUID()}) + g.Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") - Expect(instance.Name).To(Equal(ns.GetName())) - Expect(condition.Status).To(Equal(metav1.ConditionFalse), "Expected namespace condition status to be True") - Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.FailedReason), "Expected namespace condition reason to be Succeeded") + condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") - expectedMetadata := &capsulev1beta2.TenantStatusNamespaceMetadata{ - Labels: map[string]string{ - "clastix.io/custom-label": "bar", - }, - Annotations: map[string]string{ - "k8s.io/custom-annotation": "bizz", - }, - } + g.Expect(instance.Name).To(Equal(ns.GetName())) + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected namespace condition status to be True") + g.Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") + g.Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected namespace condition reason to be Succeeded") - Expect(instance.Metadata).To(Equal(expectedMetadata)) + expectedMetadata := &capsulev1beta2.TenantStatusNamespaceMetadata{ + Labels: map[string]string{ + "clastix.io/custom-label": "bar", + }, + Annotations: map[string]string{ + "k8s.io/custom-annotation": "bizz", + }, + } + + g.Expect(instance.Metadata).To(Equal(expectedMetadata)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) }) By("change managed additional metadata (empty update)", func() { - tnt.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{ - ManagedMetadataOnly: false, - AdditionalMetadataList: []api.AdditionalMetadataSelectorSpec{}, - } + Eventually(func() error { + t := &capsulev1beta2.Tenant{} + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).To(Succeed()) - Expect(k8sClient.Update(context.TODO(), tnt)).Should(Succeed()) - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, tnt)).Should(Succeed()) + t.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{ + ManagedMetadataOnly: false, + AdditionalMetadataList: []api.AdditionalMetadataSelectorSpec{}, + } + + return k8sClient.Update(context.TODO(), t) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("verify metadata lifecycle (empty update)", func() { @@ -693,28 +796,43 @@ var _ = Describe("creating a Namespace for a Tenant with additional metadata", L }, defaultTimeoutInterval, defaultPollInterval).Should(Equal(expectedAnnotations)) By("verify tenant status", func() { - condition := tnt.Status.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + Eventually(func(g Gomega) { + t := &capsulev1beta2.Tenant{} - Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected tenant condition status to be True") - Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected tenant condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected tenant condition reason to be Succeeded") + g.Expect(k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tnt.GetName()}, + t, + )).To(Succeed()) + + condition := t.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + + g.Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected tenant condition type to be Ready") + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected tenant condition status to be True") + g.Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected tenant condition reason to be Succeeded") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("verify namespace status", func() { - instance := tnt.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{Name: ns.GetName(), UID: ns.GetUID()}) - Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") + Eventually(func(g Gomega) { + t := &capsulev1beta2.Tenant{} + g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).To(Succeed()) - condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + instance := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{Name: ns.GetName(), UID: ns.GetUID()}) + g.Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") - Expect(instance.Name).To(Equal(ns.GetName())) - Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected namespace condition status to be True") - Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected namespace condition reason to be Succeeded") + condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") - expectedMetadata := &capsulev1beta2.TenantStatusNamespaceMetadata{} - Expect(instance.Metadata).To(Equal(expectedMetadata)) + g.Expect(instance.Name).To(Equal(ns.GetName())) + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected namespace condition status to be True") + g.Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") + g.Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected namespace condition reason to be Succeeded") + + expectedMetadata := &capsulev1beta2.TenantStatusNamespaceMetadata{} + g.Expect(instance.Metadata).To(Equal(expectedMetadata)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) }) }) diff --git a/e2e/overquota_namespace_test.go b/e2e/namespace_overprovisioning_test.go similarity index 59% rename from e2e/overquota_namespace_test.go rename to e2e/namespace_overprovisioning_test.go index 06434bdd..51ebdad3 100644 --- a/e2e/overquota_namespace_test.go +++ b/e2e/namespace_overprovisioning_test.go @@ -12,20 +12,24 @@ import ( "k8s.io/utils/ptr" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespace in over-quota of three", Label("namespace"), func() { +var _ = Describe("creating a Namespace in over-quota of three", Ordered, Label("namespace"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "over-quota-tenant", + Name: "e2e-ns-overprovision", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "bob", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-ns-overprovision", Kind: "User", }, }, @@ -41,22 +45,28 @@ var _ = Describe("creating a Namespace in over-quota of three", Label("namespace EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should fail", func() { By("creating three Namespaces", func() { - for _, name := range []string{"bob-dev", "bob-staging", "bob-production"} { - ns := NewNamespace(name) + for _, name := range []string{"e2e-ns-overprovision-dev", "e2e-ns-overprovision-prod", "e2e-ns-overprovision-test"} { + ns := NewNamespace(name, map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) } }) By("creating additional namespace", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) _, err := cs.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) Expect(err).ShouldNot(Succeed()) diff --git a/e2e/selecting_non_owned_tenant_test.go b/e2e/namespace_selecting_non_owned_tenant_test.go similarity index 54% rename from e2e/selecting_non_owned_tenant_test.go rename to e2e/namespace_selecting_non_owned_tenant_test.go index 19a80c19..4684cea8 100644 --- a/e2e/selecting_non_owned_tenant_test.go +++ b/e2e/namespace_selecting_non_owned_tenant_test.go @@ -6,28 +6,30 @@ package e2e import ( "context" - "github.com/projectcapsule/capsule/pkg/api" - "github.com/projectcapsule/capsule/pkg/utils" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" ) -var _ = Describe("creating a Namespace trying to select a third Tenant", Label("tenant"), func() { +var _ = Describe("creating a Namespace trying to select a third Tenant", Ordered, Label("namespace", "tenant", "assignment"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-non-owned", + Name: "e2e-tenant-non-owned", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "undefined", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-non-owned", Kind: "User", }, }, @@ -40,25 +42,18 @@ var _ = Describe("creating a Namespace trying to select a third Tenant", Label(" EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should fail", func() { - var ns *corev1.Namespace - - By("assigning to the Namespace the Capsule Tenant label", func() { - l, err := utils.GetTypeLabel(&capsulev1beta2.Tenant{}) - Expect(err).ToNot(HaveOccurred()) - - ns := NewNamespace("") - ns.SetLabels(map[string]string{ - l: tnt.Name, - }) + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.Name, }) - cs := ownerClient(api.UserSpec{Name: "dale", Kind: "User"}) + cs := ownerClient(rbac.UserSpec{Name: "e2e-tenant-non-owned-fail", Kind: "User"}) _, err := cs.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) Expect(err).To(HaveOccurred()) }) diff --git a/e2e/selecting_tenant_fail_test.go b/e2e/namespace_selecting_tenant_fail_test.go similarity index 62% rename from e2e/selecting_tenant_fail_test.go rename to e2e/namespace_selecting_tenant_fail_test.go index e941f1c6..2fe53aff 100644 --- a/e2e/selecting_tenant_fail_test.go +++ b/e2e/namespace_selecting_tenant_fail_test.go @@ -11,20 +11,23 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespace without a Tenant selector when user owns multiple Tenants", Label("tenant", "assignment"), func() { +var _ = Describe("creating a Namespace without a Tenant selector when user owns multiple Tenants", Ordered, Label("config", "tenant", "assignment"), func() { t1 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-one", + Name: "e2e-tenant-fail-one", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "john", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-fail", Kind: "User", }, }, @@ -34,14 +37,17 @@ var _ = Describe("creating a Namespace without a Tenant selector when user owns } t2 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-two", + Name: "e2e-tenant-fail-two", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "john", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-fail", Kind: "User", }, }, @@ -51,14 +57,17 @@ var _ = Describe("creating a Namespace without a Tenant selector when user owns } t3 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-three", + Name: "e2e-tenant-fail-three", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "john", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-fail", Kind: "Group", }, }, @@ -68,14 +77,17 @@ var _ = Describe("creating a Namespace without a Tenant selector when user owns } t4 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-four", + Name: "e2e-tenant-fail-four", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "john", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-fail", Kind: "Group", }, }, @@ -88,34 +100,42 @@ var _ = Describe("creating a Namespace without a Tenant selector when user owns ns := NewNamespace("") By("user owns 2 tenants", func() { EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), t1) }).Should(Succeed()) + TenantReady(t1, metav1.ConditionTrue, defaultTimeoutInterval) EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), t2) }).Should(Succeed()) + TenantReady(t2, metav1.ConditionTrue, defaultTimeoutInterval) NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) NamespaceCreation(ns, t2.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) - Expect(k8sClient.Delete(context.TODO(), t1)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), t2)).Should(Succeed()) + EventuallyDeletion(t1) + EventuallyDeletion(t2) }) By("group owns 2 tenants", func() { EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), t3) }).Should(Succeed()) + TenantReady(t3, metav1.ConditionTrue, defaultTimeoutInterval) EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), t4) }).Should(Succeed()) + TenantReady(t4, metav1.ConditionTrue, defaultTimeoutInterval) NamespaceCreation(ns, t3.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) NamespaceCreation(ns, t4.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) - Expect(k8sClient.Delete(context.TODO(), t3)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), t4)).Should(Succeed()) + EventuallyDeletion(t3) + EventuallyDeletion(t4) }) By("user and group owns 4 tenants", func() { t1.ResourceVersion, t2.ResourceVersion, t3.ResourceVersion, t4.ResourceVersion = "", "", "", "" EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), t1) }).Should(Succeed()) + TenantReady(t1, metav1.ConditionTrue, defaultTimeoutInterval) EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), t2) }).Should(Succeed()) + TenantReady(t2, metav1.ConditionTrue, defaultTimeoutInterval) EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), t3) }).Should(Succeed()) + TenantReady(t3, metav1.ConditionTrue, defaultTimeoutInterval) EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), t4) }).Should(Succeed()) + TenantReady(t4, metav1.ConditionTrue, defaultTimeoutInterval) NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) NamespaceCreation(ns, t2.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) NamespaceCreation(ns, t3.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) NamespaceCreation(ns, t4.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) - Expect(k8sClient.Delete(context.TODO(), t1)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), t2)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), t3)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), t4)).Should(Succeed()) + EventuallyDeletion(t1) + EventuallyDeletion(t2) + EventuallyDeletion(t3) + EventuallyDeletion(t4) }) }) }) diff --git a/e2e/selecting_tenant_with_label_test.go b/e2e/namespace_selecting_tenant_with_label_test.go similarity index 76% rename from e2e/selecting_tenant_with_label_test.go rename to e2e/namespace_selecting_tenant_with_label_test.go index 8835bb13..b5f68ad4 100644 --- a/e2e/selecting_tenant_with_label_test.go +++ b/e2e/namespace_selecting_tenant_with_label_test.go @@ -6,8 +6,8 @@ package e2e import ( "context" - "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -18,17 +18,20 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" ) -var _ = Describe("creating a Namespace with Tenant selector when user owns multiple tenants", Label("tenant", "assignment"), func() { +var _ = Describe("creating a Namespace with Tenant selector when user owns multiple tenants", Ordered, Label("tenant", "assignment"), func() { t1 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-one", + Name: "e2e-tenant-label-one", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "john", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-label", Kind: "User", }, }, @@ -38,14 +41,17 @@ var _ = Describe("creating a Namespace with Tenant selector when user owns multi } t2 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-two", + Name: "e2e-tenant-label-two", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "john", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-label", Kind: "User", }, }, @@ -59,25 +65,26 @@ var _ = Describe("creating a Namespace with Tenant selector when user owns multi t1.ResourceVersion = "" return k8sClient.Create(context.TODO(), t1) }).Should(Succeed()) + TenantReady(t1, metav1.ConditionTrue, defaultTimeoutInterval) + EventuallyCreation(func() error { t2.ResourceVersion = "" return k8sClient.Create(context.TODO(), t2) }).Should(Succeed()) + TenantReady(t2, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), t1)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), t2)).Should(Succeed()) + EventuallyDeletion(t1) + EventuallyDeletion(t2) }) It("should be assigned to the selected Tenant", func() { - ns := NewNamespace("") - By("assigning to the Namespace the Capsule Tenant label", func() { - ns.Labels = map[string]string{ - meta.TenantLabel: t2.Name, - } + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: t2.Name, }) + NamespaceCreation(ns, t2.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - NamespaceIsPartOfTenant(t2, ns) + NamespaceIsPartOfTenant(t2, ns).Should(Succeed()) }) It("prevent reassignment via labels from owners", func() { @@ -88,8 +95,7 @@ var _ = Describe("creating a Namespace with Tenant selector when user owns multi } NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(t1, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - NamespaceIsPartOfTenant(t1, ns) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) }) By("assigning to the Namespace the Capsule Tenant label (Attempt Label Patch)", func() { @@ -107,8 +113,7 @@ var _ = Describe("creating a Namespace with Tenant selector when user owns multi new := &corev1.Namespace{} k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, new) - TenantNamespaceList(t1, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - NamespaceIsPartOfTenant(t1, new) + NamespaceIsPartOfTenant(t1, new).Should(Succeed()) }) By("assigning to the Namespace the Capsule Tenant label (Attempt Ownerreference Patch)", func() { @@ -130,8 +135,7 @@ var _ = Describe("creating a Namespace with Tenant selector when user owns multi new := &corev1.Namespace{} k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, new) - TenantNamespaceList(t1, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - NamespaceIsPartOfTenant(t1, new) + NamespaceIsPartOfTenant(t1, new).Should(Succeed()) }) By("assigning to the Namespace the Capsule Tenant label (Attempt Ownerreference Patch) - Without Label", func() { @@ -151,8 +155,7 @@ var _ = Describe("creating a Namespace with Tenant selector when user owns multi new := &corev1.Namespace{} k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, new) - TenantNamespaceList(t1, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - NamespaceIsPartOfTenant(t1, ns) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) }) By("assigning to the Namespace the Capsule Tenant label (Empty Ownerreferences)", func() { @@ -171,8 +174,7 @@ var _ = Describe("creating a Namespace with Tenant selector when user owns multi new := &corev1.Namespace{} k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, new) - TenantNamespaceList(t1, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - NamespaceIsPartOfTenant(t1, new) + NamespaceIsPartOfTenant(t1, new).Should(Succeed()) }) By("assigning to the Namespace the Capsule Tenant label (Empty Ownerreferences) - Without Label", func() { @@ -191,8 +193,7 @@ var _ = Describe("creating a Namespace with Tenant selector when user owns multi new := &corev1.Namespace{} k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, new) - TenantNamespaceList(t1, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - NamespaceIsPartOfTenant(t1, ns) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) }) By("assigning to the Namespace the Capsule Tenant label (2nd Tenant Label + Ownerreference)", func() { @@ -214,8 +215,7 @@ var _ = Describe("creating a Namespace with Tenant selector when user owns multi new := &corev1.Namespace{} k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, new) - TenantNamespaceList(t1, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - NamespaceIsPartOfTenant(t1, new) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) }) }) }) diff --git a/e2e/namespace_status_test.go b/e2e/namespace_status_test.go index a3db8d9a..5ec9ca43 100644 --- a/e2e/namespace_status_test.go +++ b/e2e/namespace_status_test.go @@ -12,21 +12,24 @@ import ( "k8s.io/apimachinery/pkg/types" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating namespace with status lifecycle", Label("namespace", "status"), func() { +var _ = Describe("creating namespace with status lifecycle", Ordered, Label("namespace", "status"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-status", + Name: "e2e-tenant-status", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "gatsby", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-status", Kind: "User", }, }, @@ -40,54 +43,32 @@ var _ = Describe("creating namespace with status lifecycle", Label("namespace", tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("verify namespace lifecycle (functionality)", func() { - ns1 := NewNamespace("") + ns1 := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + By("creating first namespace", func() { NamespaceCreation(ns1, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElements(ns1.GetName())) - - t := &capsulev1beta2.Tenant{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).Should(Succeed()) - - Expect(t.Status.Size).To(Equal(uint(1))) - - instance := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{Name: ns1.GetName(), UID: ns1.GetUID()}) - Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") - - condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + NamespaceIsPartOfTenant(tnt, ns1).Should(Succeed()) + TenantNamespaceReady(tnt, ns1, 1) + }) - Expect(instance.Name).To(Equal(ns1.GetName())) - Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected namespace condition status to be True") - Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected namespace condition reason to be Succeeded") + ns2 := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), }) - ns2 := NewNamespace("") By("creating second namespace", func() { NamespaceCreation(ns2, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElements(ns2.GetName())) - - t := &capsulev1beta2.Tenant{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).Should(Succeed()) - - Expect(t.Status.Size).To(Equal(uint(2))) - - instance := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{Name: ns2.GetName(), UID: ns2.GetUID()}) - Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") - - condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") - - Expect(instance.Name).To(Equal(ns2.GetName())) - Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected namespace condition status to be True") - Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected namespace condition reason to be Succeeded") + NamespaceIsPartOfTenant(tnt, ns2).Should(Succeed()) + TenantNamespaceReady(tnt, ns2, 2) }) By("removing first namespace", func() { @@ -116,13 +97,25 @@ var _ = Describe("creating namespace with status lifecycle", Label("namespace", By("removing second namespace", func() { Expect(k8sClient.Delete(context.TODO(), ns2)).Should(Succeed()) - t := &capsulev1beta2.Tenant{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).Should(Succeed()) + Eventually(func(g Gomega) { + t := &capsulev1beta2.Tenant{} - Expect(t.Status.Size).To(Equal(uint(0))) + err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tnt.GetName()}, + t, + ) + g.Expect(err).ToNot(HaveOccurred()) + + instance := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ + Name: ns2.GetName(), + UID: ns2.GetUID(), + }) + g.Expect(instance).To(BeNil(), "Namespace instance should be nil") + + g.Expect(t.Status.Size).To(Equal(uint(0))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) - instance := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{Name: ns2.GetName(), UID: ns2.GetUID()}) - Expect(instance).To(BeNil(), "Namespace instance should be nil") }) }) }) diff --git a/e2e/namespace_termination_test.go b/e2e/namespace_termination_test.go new file mode 100644 index 00000000..a88cfddd --- /dev/null +++ b/e2e/namespace_termination_test.go @@ -0,0 +1,188 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" +) + +var _ = Describe("terminating namespace with guardrails", Ordered, Label("namespace", "termination"), func() { + ctx := context.TODO() + + tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-termination", + Labels: map[string]string{ + "env": "e2e", + }, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{ + { + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-termination", + Kind: "User", + }, + }, + }, + }, + }, + } + + var ( + nsName string + podKey types.NamespacedName + ) + + JustBeforeEach(func() { + EventuallyCreation(func() error { + tnt.ResourceVersion = "" + return k8sClient.Create(ctx, tnt) + }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + nsName = ns.GetName() + + NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + + // Create a pod with a finalizer so the namespace can't complete deletion + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "finalizer-pod", + Namespace: nsName, + Finalizers: []string{"e2e.capsule.io/block-delete"}, + }, + Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + Containers: []corev1.Container{ + { + Name: "pause", + Image: "registry.k8s.io/pause:3.9", + Command: []string{"/pause"}, + SecurityContext: restrictedContainerSecurityContext(), + }, + }, + }, + } + podKey = types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace} + + EventuallyCreation(func() error { + return k8sClient.Create(ctx, pod) + }).Should(Succeed()) + }) + + JustAfterEach(func() { + EventuallyDeletion(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: podKey.Name, Namespace: podKey.Namespace}}) + EventuallyDeletion(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName}}) + EventuallyDeletion(tnt) + }) + + It("keeps managed rolebindings during namespace termination and cleans up after finalizer removal", func() { + By("deleting the namespace (it should get stuck terminating due to pod finalizer)", func() { + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName}} + Expect(k8sClient.Delete(ctx, ns)).To(Succeed()) + }) + + By("verifying namespace is terminating", func() { + Eventually(func(g Gomega) { + ns := &corev1.Namespace{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: nsName}, ns) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(ns.DeletionTimestamp).ToNot(BeNil(), "namespace should have deletionTimestamp") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + By("while namespace is terminating, verify rolebindings are still present", func() { + Consistently(func(g Gomega) { + // Namespace likely still exists during this window + ns := &corev1.Namespace{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: nsName}, ns) + g.Expect(err).ToNot(HaveOccurred()) + + VerifyTenantRoleBindings(tnt) + }, 10*time.Second, defaultPollInterval).Should(Succeed()) + }) + + By("while namespace is terminating, verify tenant still exists and has controller finalizer", func() { + Consistently(func(g Gomega) { + cur := &capsulev1beta2.Tenant{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: tnt.GetName()}, cur)).To(Succeed()) + g.Expect(controllerutil.ContainsFinalizer(cur, meta.ControllerFinalizer)).To(BeTrue(), + "tenant should still have controller finalizer while a namespace is terminating", + ) + }, 10*time.Second, defaultPollInterval).Should(Succeed()) + }) + + By("removing the pod finalizer to unblock namespace deletion", func() { + Eventually(func(g Gomega) error { + p := &corev1.Pod{} + if err := k8sClient.Get(ctx, podKey, p); err != nil { + // If it's already gone, we're done + if apierrors.IsNotFound(err) { + return nil + } + return err + } + + // Already cleared + if len(p.Finalizers) == 0 { + return nil + } + + p.Finalizers = nil + return k8sClient.Update(ctx, p) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + By("verifying the pod is eventually deleted", func() { + Eventually(func() bool { + p := &corev1.Pod{} + err := k8sClient.Get(ctx, podKey, p) + return apierrors.IsNotFound(err) + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue(), + "expected pod %s/%s to be deleted", podKey.Namespace, podKey.Name, + ) + }) + + By("verifying the namespace is eventually deleted", func() { + Eventually(func() bool { + ns := &corev1.Namespace{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: nsName}, ns) + return apierrors.IsNotFound(err) + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue(), + "expected namespace %q to be deleted", nsName, + ) + }) + + By("verifying tenant still exists (and finalizer cleanup depending on policy)", func() { + Eventually(func(g Gomega) { + cur := &capsulev1beta2.Tenant{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: tnt.GetName()}, cur) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(cur.Status.Size).Should(Equal(uint(0))) + g.Expect(controllerutil.ContainsFinalizer(cur, meta.ControllerFinalizer)).To(BeFalse()) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + }) +}) diff --git a/e2e/node_user_metadata_test.go b/e2e/node_user_metadata_test.go index 243dac96..7ca62e52 100644 --- a/e2e/node_user_metadata_test.go +++ b/e2e/node_user_metadata_test.go @@ -15,23 +15,27 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" + "github.com/projectcapsule/capsule/pkg/utils" ) -var _ = Describe("modifying node labels and annotations", Label("config", "nodes"), func() { +var _ = Describe("modifying node labels and annotations", Ordered, Label("config", "nodes"), func() { originConfig := &capsulev1beta2.CapsuleConfiguration{} tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-node-user-metadata-forbidden", + Name: "e2e-node-user-metadata-forbidden", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "gatsby", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-node-user-metadata-forbidden", Kind: "User", }, }, @@ -66,7 +70,7 @@ var _ = Describe("modifying node labels and annotations", Label("config", "nodes { Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, - Name: "gatsby", + Name: "e2e-node-user-metadata-forbidden", }, }, } @@ -85,6 +89,8 @@ var _ = Describe("modifying node labels and annotations", Label("config", "nodes tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) + EventuallyCreation(func() error { cr.ResourceVersion = "" return k8sClient.Create(context.TODO(), cr) @@ -95,9 +101,10 @@ var _ = Describe("modifying node labels and annotations", Label("config", "nodes }).Should(Succeed()) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), crb)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), cr)).Should(Succeed()) + EventuallyDeletion(tnt) + EventuallyDeletion(crb) + EventuallyDeletion(cr) + EventuallyCreation(func() error { return ModifyNode(func(node *corev1.Node) error { annotations := node.GetAnnotations() diff --git a/e2e/dynamic_tenant_owner_clusterroles_test.go b/e2e/owner_dynamic_clusterroles_test.go similarity index 52% rename from e2e/dynamic_tenant_owner_clusterroles_test.go rename to e2e/owner_dynamic_clusterroles_test.go index c6e99caa..ff3dc4a8 100644 --- a/e2e/dynamic_tenant_owner_clusterroles_test.go +++ b/e2e/owner_dynamic_clusterroles_test.go @@ -11,32 +11,36 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("defining dynamic Tenant Owner Cluster Roles", Label("tenant"), func() { +var _ = Describe("defining dynamic Tenant Owner Cluster Roles", Ordered, Label("tenant", "permissions", "owners", "rolebindings"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "dynamic-tenant-owner-clusterroles", + Name: "e2e-dynamic-to-clusterroles", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Kind: "User", - Name: "michonne", + Name: "e2e-dynamic-to-clusterroles", }, - ClusterRoles: []string{"editor", "manager"}, + ClusterRoles: []string{"edit", "admin"}, }, }, { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "kingdom", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "group:e2e-dynamic-to-clusterroles", Kind: "Group", }, - ClusterRoles: []string{"readonly"}, + ClusterRoles: []string{"view"}, }, }, }, @@ -49,20 +53,24 @@ var _ = Describe("defining dynamic Tenant Owner Cluster Roles", Label("tenant"), return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("namespace should contains the dynamic rolebindings", func() { for _, ns := range []string{"dynamnic-roles-1", "dynamnic-roles-2", "dynamnic-roles-3"} { - ns := NewNamespace(ns) + ns := NewNamespace(ns, map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) - Eventually(CheckForOwnerRoleBindings(ns, tnt.Spec.Owners[0], map[string]bool{"editor": false, "manager": false}), defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) - Eventually(CheckForOwnerRoleBindings(ns, tnt.Spec.Owners[1], map[string]bool{"readonly": false}), defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + Eventually(CheckForOwnerRoleBindings(ns, tnt.Spec.Owners[0], map[string]bool{"edit": false, "admin": false}), defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + Eventually(CheckForOwnerRoleBindings(ns, tnt.Spec.Owners[1], map[string]bool{"view": false}), defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } }) }) diff --git a/e2e/missing_tenant_test.go b/e2e/owner_missing_tenant_test.go similarity index 57% rename from e2e/missing_tenant_test.go rename to e2e/owner_missing_tenant_test.go index f1e532be..5d519128 100644 --- a/e2e/missing_tenant_test.go +++ b/e2e/owner_missing_tenant_test.go @@ -11,18 +11,25 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespace creation with no Tenant assigned", Label("tenant"), func() { +var _ = Describe("creating a Namespace creation with no Tenant assigned", Ordered, Label("tenant", "permissions", "owners"), func() { It("should fail", func() { tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-missing-user", + Labels: map[string]string{ + "env": "e2e", + }, + }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "missing", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-missing-user", Kind: "User", }, }, @@ -30,7 +37,9 @@ var _ = Describe("creating a Namespace creation with no Tenant assigned", Label( }, }, } - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) _, err := cs.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) Expect(err).ShouldNot(Succeed()) diff --git a/e2e/owner_webhooks_test.go b/e2e/owner_webhooks_test.go index 69e96a18..c4c596f0 100644 --- a/e2e/owner_webhooks_test.go +++ b/e2e/owner_webhooks_test.go @@ -17,19 +17,24 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("when Tenant owner interacts with the webhooks", Label("tenant"), func() { +var _ = Describe("when Tenant owner interacts with the webhooks", Ordered, Label("tenant"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-owner", + Name: "e2e-owner-admission", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "ruby", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-owner-admission", Kind: "User", }, }, @@ -99,16 +104,20 @@ var _ = Describe("when Tenant owner interacts with the webhooks", Label("tenant" tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should disallow deletions", func() { By("blocking Capsule Limit ranges", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) lr := &corev1.LimitRange{} Eventually(func() error { @@ -120,9 +129,11 @@ var _ = Describe("when Tenant owner interacts with the webhooks", Label("tenant" Expect(cs.CoreV1().LimitRanges(ns.GetName()).Delete(context.TODO(), lr.Name, metav1.DeleteOptions{})).ShouldNot(Succeed()) }) By("blocking Capsule Network Policy", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) np := &networkingv1.NetworkPolicy{} Eventually(func() error { @@ -134,9 +145,11 @@ var _ = Describe("when Tenant owner interacts with the webhooks", Label("tenant" Expect(cs.NetworkingV1().NetworkPolicies(ns.GetName()).Delete(context.TODO(), np.Name, metav1.DeleteOptions{})).ShouldNot(Succeed()) }) By("blocking Capsule Resource Quota", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) rq := &corev1.ResourceQuota{} Eventually(func() error { @@ -151,9 +164,11 @@ var _ = Describe("when Tenant owner interacts with the webhooks", Label("tenant" It("should allow", func() { By("listing Limit Range", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) Eventually(func() (err error) { cs := ownerClient(tnt.Spec.Owners[0].UserSpec) @@ -162,9 +177,11 @@ var _ = Describe("when Tenant owner interacts with the webhooks", Label("tenant" }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("listing Network Policy", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) Eventually(func() (err error) { cs := ownerClient(tnt.Spec.Owners[0].UserSpec) @@ -173,9 +190,11 @@ var _ = Describe("when Tenant owner interacts with the webhooks", Label("tenant" }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("listing Resource Quota", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) Eventually(func() (err error) { cs := ownerClient(tnt.Spec.Owners[0].UserSpec) @@ -186,9 +205,11 @@ var _ = Describe("when Tenant owner interacts with the webhooks", Label("tenant" }) It("should allow all actions to Tenant owner Network Policy", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) np := &networkingv1.NetworkPolicy{ diff --git a/e2e/owners_test.go b/e2e/owners_test.go index b6309c4f..10c4a7b4 100644 --- a/e2e/owners_test.go +++ b/e2e/owners_test.go @@ -13,16 +13,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { +var _ = Describe("Owners", Ordered, Label("config", "tenant", "permissions", "owners"), func() { originConfig := &capsulev1beta2.CapsuleConfiguration{} tnt1 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-owners-1", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ Permissions: capsulev1beta2.Permissions{ @@ -39,26 +42,26 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { }, }, }, - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "e2e-owners-1", Kind: "User", }, }, }, { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "e2e-owners-1-group", Kind: "Group", }, }, }, { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "system:serviceaccount:capsule-system:capsule", Kind: "ServiceAccount", }, @@ -71,6 +74,9 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { tnt2 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-owners-2", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ Permissions: capsulev1beta2.Permissions{ @@ -87,26 +93,26 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { }, }, }, - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "e2e-owners-2", Kind: "User", }, }, }, { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "e2e-owners-2-group", Kind: "Group", }, }, }, { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "system:serviceaccount:capsule-system:capsule", Kind: "ServiceAccount", }, @@ -125,13 +131,13 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { }, Spec: capsulev1beta2.TenantOwnerSpec{ Aggregate: true, - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "oidc:comp:administrators", }, ClusterRoles: []string{ - "mega-admin", + "admin", }, }, }, @@ -146,13 +152,13 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { }, Spec: capsulev1beta2.TenantOwnerSpec{ Aggregate: true, - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "oidc:comp:devops", }, ClusterRoles: []string{ - "namespaced-admin", + "view", }, }, }, @@ -168,13 +174,13 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { }, Spec: capsulev1beta2.TenantOwnerSpec{ Aggregate: true, - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:capsule-system:capsule", }, ClusterRoles: []string{ - "service-admin", + "edit", }, }, }, @@ -189,13 +195,13 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { }, Spec: capsulev1beta2.TenantOwnerSpec{ Aggregate: true, - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "tnt-1-user", }, ClusterRoles: []string{ - "service-admin", + "edit", }, }, }, @@ -211,13 +217,13 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { }, Spec: capsulev1beta2.TenantOwnerSpec{ Aggregate: false, - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "some-user", }, ClusterRoles: []string{ - "service-admin", + "view", }, }, }, @@ -232,6 +238,8 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) } for _, tnt := range []*capsulev1beta2.TenantOwner{ownersInfra, ownersDevops, ownersCommon, userOwnersCommon, tnt1Owner} { @@ -245,13 +253,11 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { JustAfterEach(func() { for _, tnt := range []*capsulev1beta2.Tenant{tnt1, tnt2} { - err := k8sClient.Delete(context.TODO(), tnt) - Expect(client.IgnoreNotFound(err)).To(Succeed()) + EventuallyDeletion(tnt) } for _, owners := range []*capsulev1beta2.TenantOwner{ownersInfra, ownersDevops, ownersCommon, userOwnersCommon, tnt1Owner} { - err := k8sClient.Delete(context.TODO(), owners) - Expect(client.IgnoreNotFound(err)).To(Succeed()) + EventuallyDeletion(owners) } }) @@ -266,100 +272,98 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { ) g.Expect(err).ToNot(HaveOccurred()) - expected := api.UserListSpec{ + expected := rbac.UserListSpec{ {Kind: ownersInfra.Spec.Kind, Name: ownersInfra.Spec.Name}, {Kind: ownersDevops.Spec.Kind, Name: ownersDevops.Spec.Name}, {Kind: ownersCommon.Spec.Kind, Name: ownersCommon.Spec.Name}, {Kind: tnt1Owner.Spec.Kind, Name: tnt1Owner.Spec.Name}, - {Kind: api.GroupOwner, Name: "projectcapsule.dev"}, + {Kind: rbac.GroupOwner, Name: "projectcapsule.dev"}, } g.Expect(cfg.Status.Users).To(ConsistOf(expected)) g.Expect(cfg.Status.Users).NotTo(ContainElement( - api.UserSpec{Kind: userOwnersCommon.Spec.Kind, Name: userOwnersCommon.Spec.Name}, + rbac.UserSpec{Kind: userOwnersCommon.Spec.Kind, Name: userOwnersCommon.Spec.Name}, )) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("checking owners (e2e-owners-1)", func() { - t := &capsulev1beta2.Tenant{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt1.GetName()}, t)).Should(Succeed()) - - expectedOwners := api.OwnerStatusListSpec{ + expectedOwners := rbac.OwnerStatusListSpec{ { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "e2e-owners-1-group", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "oidc:comp:devops", }, - ClusterRoles: []string{"namespaced-admin"}, + ClusterRoles: []string{"view"}, }, { - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:capsule-system:capsule", }, - ClusterRoles: []string{"admin", "capsule-namespace-deleter", "service-admin"}, + ClusterRoles: []string{"admin", "capsule-namespace-deleter", "edit"}, }, { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "e2e-owners-1", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ + UserSpec: rbac.UserSpec{ Kind: userOwnersCommon.Spec.Kind, Name: userOwnersCommon.Spec.Name, }, - ClusterRoles: []string{"service-admin"}, + ClusterRoles: []string{"view"}, }, { - UserSpec: api.UserSpec{ + UserSpec: rbac.UserSpec{ Kind: tnt1Owner.Spec.Kind, Name: tnt1Owner.Spec.Name, }, - ClusterRoles: []string{"service-admin"}, + ClusterRoles: []string{"edit"}, }, } - Expect(normalizeOwners(t.Status.Owners)). - To(Equal(normalizeOwners(expectedOwners))) + t := &capsulev1beta2.Tenant{} + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt1.GetName()}, t)).Should(Succeed()) + t = ExpectTenantOwnersEventually(tnt1.GetName(), expectedOwners) VerifyTenantRoleBindings(t) }) By("creating namespaces (e2e-owners-1)", func() { - for _, u := range []api.UserSpec{ - api.UserSpec{ - Kind: api.GroupOwner, + for _, u := range []rbac.UserSpec{ + rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "e2e-owners-1-group", }, - api.UserSpec{ - Kind: api.GroupOwner, + rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "oidc:comp:devops", }, - api.UserSpec{ - Kind: api.ServiceAccountOwner, + rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:capsule-system:capsule", }, - api.UserSpec{ - Kind: api.UserOwner, + rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "e2e-owners-1", }, - api.UserSpec{ + rbac.UserSpec{ Kind: userOwnersCommon.Spec.Kind, Name: userOwnersCommon.Spec.Name, }, - api.UserSpec{ + rbac.UserSpec{ Kind: tnt1Owner.Spec.Kind, Name: tnt1Owner.Spec.Name, }, @@ -368,7 +372,7 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { meta.TenantLabel: tnt1.GetName(), }) NamespaceCreation(ns, u, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt1, defaultTimeoutInterval).Should(ContainElements(ns.GetName())) + NamespaceIsPartOfTenant(tnt1, ns).Should(Succeed()) } }) @@ -376,70 +380,68 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { t := &capsulev1beta2.Tenant{} Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt2.GetName()}, t)).Should(Succeed()) - expectedOwners := api.OwnerStatusListSpec{ + expectedOwners := rbac.OwnerStatusListSpec{ { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "e2e-owners-2-group", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "oidc:comp:administrators", }, - ClusterRoles: []string{"mega-admin"}, + ClusterRoles: []string{"admin"}, }, { - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:capsule-system:capsule", }, - ClusterRoles: []string{"admin", "capsule-namespace-deleter", "service-admin"}, + ClusterRoles: []string{"admin", "capsule-namespace-deleter", "edit"}, }, { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "e2e-owners-2", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ + UserSpec: rbac.UserSpec{ Kind: userOwnersCommon.Spec.Kind, Name: userOwnersCommon.Spec.Name, }, - ClusterRoles: []string{"service-admin"}, + ClusterRoles: []string{"view"}, }, } - Expect(normalizeOwners(t.Status.Owners)). - To(Equal(normalizeOwners(expectedOwners))) - + t = ExpectTenantOwnersEventually(tnt2.GetName(), expectedOwners) VerifyTenantRoleBindings(t) }) By("creating namespaces (e2e-owners-2)", func() { - for _, u := range []api.UserSpec{ - api.UserSpec{ - Kind: api.GroupOwner, + for _, u := range []rbac.UserSpec{ + rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "e2e-owners-2-group", }, - api.UserSpec{ - Kind: api.GroupOwner, + rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "oidc:comp:administrators", }, - api.UserSpec{ - Kind: api.ServiceAccountOwner, + rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:capsule-system:capsule", }, - api.UserSpec{ - Kind: api.UserOwner, + rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "e2e-owners-2", }, - api.UserSpec{ + rbac.UserSpec{ Kind: userOwnersCommon.Spec.Kind, Name: userOwnersCommon.Spec.Name, }, @@ -448,12 +450,12 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { meta.TenantLabel: tnt2.GetName(), }) NamespaceCreation(ns, u, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt2, defaultTimeoutInterval).Should(ContainElements(ns.GetName())) + NamespaceIsPartOfTenant(tnt2, ns).Should(Succeed()) } }) By("remove common tenant-owners", func() { - Expect(k8sClient.Delete(context.TODO(), ownersCommon)).Should(Succeed()) + EventuallyDeletion(ownersCommon) }) By("checking configuration", func() { @@ -466,11 +468,11 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { ) g.Expect(err).ToNot(HaveOccurred()) - expected := api.UserListSpec{ + expected := rbac.UserListSpec{ {Kind: ownersInfra.Spec.Kind, Name: ownersInfra.Spec.Name}, {Kind: ownersDevops.Spec.Kind, Name: ownersDevops.Spec.Name}, {Kind: tnt1Owner.Spec.Kind, Name: tnt1Owner.Spec.Name}, - {Kind: api.GroupOwner, Name: "projectcapsule.dev"}, + {Kind: rbac.GroupOwner, Name: "projectcapsule.dev"}, } g.Expect(cfg.Status.Users).To(ConsistOf(expected)) @@ -481,54 +483,52 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { t := &capsulev1beta2.Tenant{} Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt1.GetName()}, t)).Should(Succeed()) - expectedOwners := api.OwnerStatusListSpec{ + expectedOwners := rbac.OwnerStatusListSpec{ { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "e2e-owners-1-group", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "oidc:comp:devops", }, - ClusterRoles: []string{"namespaced-admin"}, + ClusterRoles: []string{"view"}, }, { - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:capsule-system:capsule", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "e2e-owners-1", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ + UserSpec: rbac.UserSpec{ Kind: userOwnersCommon.Spec.Kind, Name: userOwnersCommon.Spec.Name, }, - ClusterRoles: []string{"service-admin"}, + ClusterRoles: []string{"view"}, }, { - UserSpec: api.UserSpec{ + UserSpec: rbac.UserSpec{ Kind: tnt1Owner.Spec.Kind, Name: tnt1Owner.Spec.Name, }, - ClusterRoles: []string{"service-admin"}, + ClusterRoles: []string{"edit"}, }, } - Expect(normalizeOwners(t.Status.Owners)). - To(Equal(normalizeOwners(expectedOwners))) - + t = ExpectTenantOwnersEventually(tnt1.GetName(), expectedOwners) VerifyTenantRoleBindings(t) }) @@ -536,47 +536,45 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { t := &capsulev1beta2.Tenant{} Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt2.GetName()}, t)).Should(Succeed()) - expectedOwners := api.OwnerStatusListSpec{ + expectedOwners := rbac.OwnerStatusListSpec{ { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "e2e-owners-2-group", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "oidc:comp:administrators", }, - ClusterRoles: []string{"mega-admin"}, + ClusterRoles: []string{"admin"}, }, { - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:capsule-system:capsule", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "e2e-owners-2", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ + UserSpec: rbac.UserSpec{ Kind: userOwnersCommon.Spec.Kind, Name: userOwnersCommon.Spec.Name, }, - ClusterRoles: []string{"service-admin"}, + ClusterRoles: []string{"view"}, }, } - Expect(normalizeOwners(t.Status.Owners)). - To(Equal(normalizeOwners(expectedOwners))) - + t = ExpectTenantOwnersEventually(tnt2.GetName(), expectedOwners) VerifyTenantRoleBindings(t) }) @@ -594,10 +592,10 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { ) g.Expect(err).ToNot(HaveOccurred()) - expected := api.UserListSpec{ + expected := rbac.UserListSpec{ {Kind: ownersDevops.Spec.Kind, Name: ownersDevops.Spec.Name}, {Kind: tnt1Owner.Spec.Kind, Name: tnt1Owner.Spec.Name}, - {Kind: api.GroupOwner, Name: "projectcapsule.dev"}, + {Kind: rbac.GroupOwner, Name: "projectcapsule.dev"}, } g.Expect(cfg.Status.Users).To(ConsistOf(expected)) @@ -608,54 +606,52 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { t := &capsulev1beta2.Tenant{} Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt1.GetName()}, t)).Should(Succeed()) - expectedOwners := api.OwnerStatusListSpec{ + expectedOwners := rbac.OwnerStatusListSpec{ { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "e2e-owners-1-group", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "oidc:comp:devops", }, - ClusterRoles: []string{"namespaced-admin"}, + ClusterRoles: []string{"view"}, }, { - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:capsule-system:capsule", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "e2e-owners-1", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ + UserSpec: rbac.UserSpec{ Kind: userOwnersCommon.Spec.Kind, Name: userOwnersCommon.Spec.Name, }, - ClusterRoles: []string{"service-admin"}, + ClusterRoles: []string{"view"}, }, { - UserSpec: api.UserSpec{ + UserSpec: rbac.UserSpec{ Kind: tnt1Owner.Spec.Kind, Name: tnt1Owner.Spec.Name, }, - ClusterRoles: []string{"service-admin"}, + ClusterRoles: []string{"edit"}, }, } - Expect(normalizeOwners(t.Status.Owners)). - To(Equal(normalizeOwners(expectedOwners))) - + t = ExpectTenantOwnersEventually(tnt1.GetName(), expectedOwners) VerifyTenantRoleBindings(t) }) @@ -663,45 +659,43 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { t := &capsulev1beta2.Tenant{} Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt2.GetName()}, t)).Should(Succeed()) - expectedOwners := api.OwnerStatusListSpec{ + expectedOwners := rbac.OwnerStatusListSpec{ { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "e2e-owners-2-group", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:capsule-system:capsule", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "e2e-owners-2", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ + UserSpec: rbac.UserSpec{ Kind: userOwnersCommon.Spec.Kind, Name: userOwnersCommon.Spec.Name, }, - ClusterRoles: []string{"service-admin"}, + ClusterRoles: []string{"view"}, }, } - Expect(normalizeOwners(t.Status.Owners)). - To(Equal(normalizeOwners(expectedOwners))) - + t = ExpectTenantOwnersEventually(tnt2.GetName(), expectedOwners) VerifyTenantRoleBindings(t) }) By("remove admin tenant-owners", func() { - Expect(k8sClient.Delete(context.TODO(), ownersDevops)).Should(Succeed()) + EventuallyDeletion(ownersDevops) }) By("checking configuration", func() { @@ -714,9 +708,9 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { ) g.Expect(err).ToNot(HaveOccurred()) - expected := api.UserListSpec{ + expected := rbac.UserListSpec{ {Kind: tnt1Owner.Spec.Kind, Name: tnt1Owner.Spec.Name}, - {Kind: api.GroupOwner, Name: "projectcapsule.dev"}, + {Kind: rbac.GroupOwner, Name: "projectcapsule.dev"}, } g.Expect(cfg.Status.Users).To(ConsistOf(expected)) @@ -727,47 +721,45 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { t := &capsulev1beta2.Tenant{} Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt1.GetName()}, t)).Should(Succeed()) - expectedOwners := api.OwnerStatusListSpec{ + expectedOwners := rbac.OwnerStatusListSpec{ { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "e2e-owners-1-group", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:capsule-system:capsule", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "e2e-owners-1", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ + UserSpec: rbac.UserSpec{ Kind: userOwnersCommon.Spec.Kind, Name: userOwnersCommon.Spec.Name, }, - ClusterRoles: []string{"service-admin"}, + ClusterRoles: []string{"view"}, }, { - UserSpec: api.UserSpec{ + UserSpec: rbac.UserSpec{ Kind: tnt1Owner.Spec.Kind, Name: tnt1Owner.Spec.Name, }, - ClusterRoles: []string{"service-admin"}, + ClusterRoles: []string{"edit"}, }, } - Expect(normalizeOwners(t.Status.Owners)). - To(Equal(normalizeOwners(expectedOwners))) - + t = ExpectTenantOwnersEventually(tnt1.GetName(), expectedOwners) VerifyTenantRoleBindings(t) }) @@ -775,41 +767,61 @@ var _ = Describe("Owners", Label("tenant", "permissions", "owners"), func() { t := &capsulev1beta2.Tenant{} Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt2.GetName()}, t)).Should(Succeed()) - expectedOwners := api.OwnerStatusListSpec{ + expectedOwners := rbac.OwnerStatusListSpec{ { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "e2e-owners-2-group", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:capsule-system:capsule", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "e2e-owners-2", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, { - UserSpec: api.UserSpec{ + UserSpec: rbac.UserSpec{ Kind: userOwnersCommon.Spec.Kind, Name: userOwnersCommon.Spec.Name, }, - ClusterRoles: []string{"service-admin"}, + ClusterRoles: []string{"view"}, }, } - Expect(normalizeOwners(t.Status.Owners)). - To(Equal(normalizeOwners(expectedOwners))) - + t = ExpectTenantOwnersEventually(tnt2.GetName(), expectedOwners) VerifyTenantRoleBindings(t) }) }) }) + +func ExpectTenantOwnersEventually( + tenantName string, + expected rbac.OwnerStatusListSpec, +) *capsulev1beta2.Tenant { + var current *capsulev1beta2.Tenant + + Eventually(func(g Gomega) { + current = &capsulev1beta2.Tenant{} + + g.Expect(k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tenantName}, + current, + )).To(Succeed()) + + g.Expect(normalizeOwners(current.Status.Owners)). + To(Equal(normalizeOwners(expected))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + return current +} diff --git a/e2e/scalability_test.go b/e2e/performance_scalability_test.go similarity index 52% rename from e2e/scalability_test.go rename to e2e/performance_scalability_test.go index 3baef9bd..49d69f1c 100644 --- a/e2e/scalability_test.go +++ b/e2e/performance_scalability_test.go @@ -15,21 +15,21 @@ import ( "k8s.io/apimachinery/pkg/types" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("verify scalability", Label("scalability"), func() { +var _ = Describe("verify scalability", Ordered, Label("performance", "scalability"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-scalability", + Name: "e2e-perf-scalability", }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "gatsby", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-perf-scalability", Kind: "User", }, }, @@ -43,9 +43,11 @@ var _ = Describe("verify scalability", Label("scalability"), func() { tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("verify lifecycle (scalability)", func() { @@ -55,98 +57,86 @@ var _ = Describe("verify scalability", Label("scalability"), func() { getTenant := func() *capsulev1beta2.Tenant { t := &capsulev1beta2.Tenant{} Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).To(Succeed()) + return t } - waitSize := func(expected uint) { - Eventually(func() uint { - return getTenant().Status.Size - }, defaultTimeoutInterval, defaultPollInterval).Should(Equal(expected)) - } - - waitInstancePresent := func(ns *corev1.Namespace) { + waitTenantNamespacesReady := func(namespaces []*corev1.Namespace) { Eventually(func() error { t := getTenant() - inst := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ - Name: ns.GetName(), - UID: ns.GetUID(), - }) - if inst == nil { - return fmt.Errorf("instance not found for ns=%q uid=%q", ns.GetName(), ns.GetUID()) + + if t.Status.Size != uint(len(namespaces)) { + return fmt.Errorf("tenant size=%d, want %d", t.Status.Size, len(namespaces)) } - condition := inst.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") - if inst == nil { - return fmt.Errorf("instance not found for ns=%q uid=%q", ns.GetName(), ns.GetUID()) - } + for _, ns := range namespaces { + inst := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ + Name: ns.GetName(), + UID: ns.GetUID(), + }) + if inst == nil { + return fmt.Errorf("instance not found for ns=%q uid=%q", ns.GetName(), ns.GetUID()) + } - if inst.Name != ns.GetName() { - return fmt.Errorf("instance.Name=%q, want %q", inst.Name, ns.GetName()) - } + cond := inst.Conditions.GetConditionByType(meta.ReadyCondition) + if cond == nil { + return fmt.Errorf("namespace %q missing %q condition", ns.GetName(), meta.ReadyCondition) + } - cond := inst.Conditions.GetConditionByType(meta.ReadyCondition) - if cond == nil { - return fmt.Errorf("missing %q condition", meta.ReadyCondition) - } - if cond.Type != meta.ReadyCondition { - return fmt.Errorf("cond.Type=%q, want %q", cond.Type, meta.ReadyCondition) - } - if cond.Status != metav1.ConditionTrue { - return fmt.Errorf("cond.Status=%q, want %q", cond.Status, metav1.ConditionTrue) - } - if cond.Reason != meta.SucceededReason { - return fmt.Errorf("cond.Reason=%q, want %q", cond.Reason, meta.SucceededReason) + if cond.Status != metav1.ConditionTrue { + return fmt.Errorf("namespace %q ready status=%q, want %q", ns.GetName(), cond.Status, metav1.ConditionTrue) + } + + if cond.Reason != meta.SucceededReason { + return fmt.Errorf("namespace %q ready reason=%q, want %q", ns.GetName(), cond.Reason, meta.SucceededReason) + } } return nil }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } - waitInstanceAbsent := func(ns *corev1.Namespace) { - Eventually(func() bool { - t := getTenant() - inst := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ - Name: ns.GetName(), - UID: ns.GetUID(), - }) - return inst == nil - }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + waitTenantNamespacesGone := func() { + Eventually(func() uint { + return getTenant().Status.Size + }, defaultTimeoutInterval, defaultPollInterval).Should(Equal(uint(0))) } - // --- Scale up: create N namespaces and verify Tenant status each time --- namespaces := make([]*corev1.Namespace, 0, amount) + + By("creating namespaces") for i := 0; i < amount; i++ { - ns := NewNamespace(fmt.Sprintf("scale-%d", i)) + ns := NewNamespace(fmt.Sprintf("e2e-scale-%d", i), map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + namespaces = append(namespaces, ns) + } - // Expect size bumped to i+1 and instance present - waitSize(uint(i + 1)) - waitInstancePresent(ns) + By("waiting for all namespaces to be reflected in tenant status") + waitTenantNamespacesReady(namespaces) - // --- NEW: create low-impact traffic pods in the namespace --- + By("creating low-impact deployments") + for _, ns := range namespaces { dep := newTrafficDeployment(ns.GetName(), podsPerNamespace) EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), dep) }).Should(Succeed()) - - // Wait until pods are actually scheduled & ready - waitDeploymentReady(context.TODO(), ns.GetName(), dep.Name, podsPerNamespace) - - namespaces = append(namespaces, ns) } - // --- Scale down: delete N namespaces and verify Tenant status each time --- - for i := 0; i < amount; i++ { - ns := namespaces[i] + By("waiting for deployments") + for _, ns := range namespaces { + waitDeploymentReady(context.TODO(), ns.GetName(), "traffic-pause", podsPerNamespace) + } + + By("deleting namespaces") + for _, ns := range namespaces { Expect(k8sClient.Delete(context.TODO(), ns)).To(Succeed()) - - // Expect size decremented and instance absent - waitSize(uint(amount - i - 1)) - waitInstanceAbsent(ns) } + By("waiting for tenant status to be empty") + waitTenantNamespacesGone() }) }) @@ -165,13 +155,12 @@ func newTrafficDeployment(ns string, replicas int32) *appsv1.Deployment { Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{ - // pause container keeps footprint tiny + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "pause", - Image: "registry.k8s.io/pause:3.9", - // No resources specified => requests/limits default to zero. - // Resources: corev1.ResourceRequirements{}, + Name: "pause", + Image: "registry.k8s.io/pause:3.9", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, diff --git a/e2e/pod_metadata_test.go b/e2e/pod_metadata_test.go index 50146470..b8564bef 100644 --- a/e2e/pod_metadata_test.go +++ b/e2e/pod_metadata_test.go @@ -5,28 +5,32 @@ package e2e import ( "context" - "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) -var _ = Describe("adding metadata to Pod objects", Label("pod"), func() { +var _ = Describe("adding metadata to Pod objects", Ordered, Label("pod"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "pod-metadata", + Name: "e2e-pod-metadata", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "gatsby", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-pod-metadata", Kind: "User", }, }, @@ -53,29 +57,35 @@ var _ = Describe("adding metadata to Pod objects", Label("pod"), func() { tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should apply them to Pod", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - fmt.Sprint("namespace created") - //TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - fmt.Sprint("tenant contains list namespace") + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-metadata", Namespace: ns.GetName(), }, + Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "container", Image: "quay.io/google-containers/pause-amd64:3.0", ImagePullPolicy: "IfNotPresent", + SecurityContext: restrictedContainerSecurityContext(), }, }, RestartPolicy: "Always", diff --git a/e2e/pod_priority_class_test.go b/e2e/pod_priority_class_test.go index 228450fc..5d488472 100644 --- a/e2e/pod_priority_class_test.go +++ b/e2e/pod_priority_class_test.go @@ -20,19 +20,24 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { +var _ = Describe("enforcing a Priority Class", Ordered, Label("pod", "classes", "priorityclass"), func() { tntWithDefaults := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "priority-class-defaults", + Name: "e2e-priority-class-defaults", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "paul", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-pod-priority-1", Kind: "User", }, }, @@ -43,7 +48,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { SelectorAllowedListSpec: api.SelectorAllowedListSpec{ LabelSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ - "env": "customer", + "environment": "customer", }, }, }, @@ -53,14 +58,17 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { tntNoDefaults := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "priority-class-no-defaults", + Name: "e2e-priority-class-no-defaults", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "george", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-pod-priority-2", Kind: "User", }, }, @@ -74,7 +82,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { }, LabelSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ - "env": "customer", + "environment": "customer", }, }, }, @@ -85,12 +93,15 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { tntNoRestrictions := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-priority-class-no-restrictions", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: []api.OwnerSpec{ + Owners: []rbac.OwnerSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "e2e-priority-class-no-restrictions", Kind: "User", }, @@ -105,7 +116,8 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "tenant-default", Labels: map[string]string{ - "env": "e2e", + "environment": "shared", + "env": "e2e", }, }, Description: "tenant default priorityclass", @@ -118,7 +130,8 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "global-default", Labels: map[string]string{ - "env": "customer", + "environment": "customer", + "env": "e2e", }, }, Description: "global default priorityclass", @@ -130,7 +143,8 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "disallowed-global-default", Labels: map[string]string{ - "env": "e2e", + "environment": "internal", + "env": "e2e", }, }, Description: "global default priorityclass", @@ -142,7 +156,8 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "customer-bronze", Labels: map[string]string{ - "env": "customer", + "environment": "customer", + "env": "e2e", }, }, Description: "fake PriorityClass for e2e", @@ -153,7 +168,8 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "customer-silver", Labels: map[string]string{ - "env": "e2e", + "environment": "internal", + "env": "e2e", }, }, Description: "fake PriorityClass for e2e", @@ -164,7 +180,8 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "customer-gold", Labels: map[string]string{ - "env": "e2e", + "environment": "internal", + "env": "e2e", }, }, Description: "fake PriorityClass for e2e", @@ -178,6 +195,8 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) } for _, crd := range []*schedulingv1.PriorityClass{customerBronze, customerSilver, customerGold} { @@ -190,23 +209,35 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { JustAfterEach(func() { for _, tnt := range []*capsulev1beta2.Tenant{tntWithDefaults, tntNoDefaults, tntNoRestrictions} { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) } - Eventually(func() (err error) { - req, _ := labels.NewRequirement("env", selection.Exists, nil) + req, err := labels.NewRequirement("env", selection.Equals, []string{"e2e"}) + Expect(err).NotTo(HaveOccurred()) - return k8sClient.DeleteAllOf(context.TODO(), &schedulingv1.PriorityClass{}, &client.DeleteAllOfOptions{ - ListOptions: client.ListOptions{ - LabelSelector: labels.NewSelector().Add(*req), - }, - }) - }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + var list schedulingv1.PriorityClassList + Expect(k8sClient.List( + context.TODO(), + &list, + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*req), + }, + )).Should(Succeed()) + + for i := range list.Items { + EventuallyDeletion(&list.Items[i]) + } }) It("should allow all classes", Label("skip-on-openshift"), func() { all := []string{"system-cluster-critical", "system-node-critical", customerBronze.GetName(), customerSilver.GetName(), customerGold.GetName()} + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoRestrictions.GetName(), + }) + NamespaceCreation(ns, tntNoRestrictions.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tntNoRestrictions, ns).Should(Succeed()) + By("Verify Status (Creation)", func() { Eventually(func() ([]string, error) { t := &capsulev1beta2.Tenant{} @@ -223,10 +254,6 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { Should(ConsistOf(all)) }) - ns := NewNamespace("") - NamespaceCreation(ns, tntNoRestrictions.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoRestrictions, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - By("providing any priorityclass", func() { for _, class := range all { Eventually(func() (err error) { @@ -237,13 +264,16 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { Namespace: ns.GetName(), }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + PriorityClassName: class, Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, - PriorityClassName: class, }, } @@ -254,7 +284,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { By("Verify Status (Deletion)", func() { for _, crd := range []*schedulingv1.PriorityClass{customerGold} { - Expect(ignoreNotFound(k8sClient.Delete(context.TODO(), crd))).To(Succeed()) + EventuallyDeletion(crd) } Eventually(func() ([]string, error) { t := &capsulev1beta2.Tenant{} @@ -273,21 +303,43 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { }) It("should block non allowed Priority Class", Label("skip-on-openshift"), func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefaults.GetName(), + }) NamespaceCreation(ns, tntNoDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tntNoDefaults, ns).Should(Succeed()) + + By("Verify Status", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntNoDefaults.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.PriorityClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ConsistOf(customerGold.GetName(), customerBronze.GetName(), customerSilver.GetName())) + }) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + PriorityClassName: "system-node-critical", Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, - PriorityClassName: "system-node-critical", }, } @@ -305,7 +357,8 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: priorityName, Labels: map[string]string{ - "env": "internal", + "environment": "internal", + "env": "e2e", }, }, Description: "fake PriorityClass for e2e", @@ -318,29 +371,58 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { Name: pc, }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + PriorityClassName: class.GetName(), Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, - PriorityClassName: class.GetName(), }, } - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefaults.GetName(), + }) cs := ownerClient(tntNoDefaults.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tntNoDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoDefaults, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntNoDefaults, ns).Should(Succeed()) + + By("Verify Status", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntNoDefaults.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.PriorityClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ConsistOf(customerGold.GetName(), customerBronze.GetName(), customerSilver.GetName())) + }) EventuallyCreation(func() error { _, err := cs.CoreV1().Pods(ns.GetName()).Create(context.Background(), pod, metav1.CreateOptions{}) return err }).ShouldNot(Succeed()) } + }) - By("verify Status (Creation)", func() { + It("should allow exact match", func() { + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefaults.GetName(), + }) + NamespaceCreation(ns, tntNoDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tntNoDefaults, ns).Should(Succeed()) + + By("Verify Status", func() { Eventually(func() ([]string, error) { t := &capsulev1beta2.Tenant{} if err := k8sClient.Get( @@ -353,26 +435,24 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { return t.Status.Classes.PriorityClasses, nil }, defaultTimeoutInterval, defaultPollInterval). - Should(ConsistOf(customerGold.GetName(), customerBronze.GetName())) + Should(ConsistOf(customerGold.GetName(), customerBronze.GetName(), customerSilver.GetName())) }) - }) - - It("should allow exact match", func() { - ns := NewNamespace("") - NamespaceCreation(ns, tntNoDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + PriorityClassName: "customer-gold", Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, - PriorityClassName: "customer-gold", }, } @@ -384,8 +464,27 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { }) It("should allow regex match", Label("skip-on-openshift"), func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefaults.GetName(), + }) NamespaceCreation(ns, tntNoDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tntNoDefaults, ns).Should(Succeed()) + + By("Verify Status", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntNoDefaults.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.PriorityClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ConsistOf(customerGold.GetName(), customerBronze.GetName(), customerSilver.GetName())) + }) for _, class := range []string{customerBronze.GetName(), customerSilver.GetName(), customerGold.GetName()} { EventuallyCreation(func() error { @@ -395,13 +494,16 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { Namespace: ns.GetName(), }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + PriorityClassName: class, Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, - PriorityClassName: class, }, } @@ -418,7 +520,8 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: priorityName, Labels: map[string]string{ - "env": "customer", + "environment": "customer", + "env": "e2e", }, }, Description: "fake PriorityClass for e2e", @@ -426,27 +529,49 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { } Expect(k8sClient.Create(context.TODO(), class)).Should(Succeed()) + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefaults.GetName(), + }) + + NamespaceCreation(ns, tntNoDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tntNoDefaults, ns).Should(Succeed()) + + By("Verify Status", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntNoDefaults.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.PriorityClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ContainElement(class.GetName())) + }) + pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: pc, }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + PriorityClassName: class.GetName(), Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, - PriorityClassName: class.GetName(), }, } - ns := NewNamespace("") cs := ownerClient(tntNoDefaults.Spec.Owners[0].UserSpec) - NamespaceCreation(ns, tntNoDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoDefaults, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - EventuallyCreation(func() error { _, err := cs.CoreV1().Pods(ns.GetName()).Create(context.Background(), pod, metav1.CreateOptions{}) return err @@ -467,7 +592,7 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { return t.Status.Classes.PriorityClasses, nil }, defaultTimeoutInterval, defaultPollInterval). - Should(ConsistOf("internal-bronze-new-0", "internal-silver-new-1", "internal-gold-new-2", customerGold.GetName(), customerBronze.GetName())) + Should(ConsistOf("internal-bronze-new-0", "internal-silver-new-1", "internal-gold-new-2", customerGold.GetName(), customerBronze.GetName(), customerSilver.GetName())) }) }) @@ -478,20 +603,41 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { Name: "tenant-default", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, }, } - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefaults.GetName(), + }) cs := ownerClient(tntWithDefaults.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tntWithDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefaults, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithDefaults, ns).Should(Succeed()) + + By("verify Status (Creation)", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntWithDefaults.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.PriorityClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ConsistOf(customerBronze.GetName())) + }) EventuallyCreation(func() error { _, err := cs.CoreV1().Pods(ns.GetName()).Create(context.Background(), pod, metav1.CreateOptions{}) @@ -499,22 +645,6 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { return err }).ShouldNot(Succeed()) }) - - By("verify Status (Creation)", func() { - Eventually(func() ([]string, error) { - t := &capsulev1beta2.Tenant{} - if err := k8sClient.Get( - context.TODO(), - types.NamespacedName{Name: tntWithDefaults.GetName()}, - t, - ); err != nil { - return nil, err - } - - return t.Status.Classes.PriorityClasses, nil - }, defaultTimeoutInterval, defaultPollInterval). - Should(ConsistOf(customerBronze.GetName())) - }) }) It("should mutate to default tenant PriorityClass", Label("skip-on-openshift"), func() { @@ -523,9 +653,27 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { class.SetResourceVersion("") Expect(k8sClient.Create(context.TODO(), class)).Should(Succeed()) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefaults.GetName(), + }) NamespaceCreation(ns, tntWithDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefaults, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithDefaults, ns).Should(Succeed()) + + By("verify Status (Creation)", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntWithDefaults.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.PriorityClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ConsistOf(tenantDefault.GetName(), customerBronze.GetName())) + }) pod := corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -533,10 +681,13 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { Namespace: ns.GetName(), }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -550,22 +701,6 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { Expect(pod.Spec.Priority).To(Equal(&class.Value)) Expect(pod.Spec.PreemptionPolicy).To(Equal(class.PreemptionPolicy)) }) - - By("verify Status (Creation)", func() { - Eventually(func() ([]string, error) { - t := &capsulev1beta2.Tenant{} - if err := k8sClient.Get( - context.TODO(), - types.NamespacedName{Name: tntWithDefaults.GetName()}, - t, - ); err != nil { - return nil, err - } - - return t.Status.Classes.PriorityClasses, nil - }, defaultTimeoutInterval, defaultPollInterval). - Should(ConsistOf(tenantDefault.GetName(), customerBronze.GetName())) - }) }) It("should mutate to default tenant PriorityClass although the cluster global one is not allowed", Label("skip-on-openshift"), func() { @@ -578,32 +713,11 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { Expect(k8sClient.Create(context.TODO(), class)).Should(Succeed()) Expect(k8sClient.Create(context.TODO(), global)).Should(Succeed()) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefaults.GetName(), + }) NamespaceCreation(ns, tntWithDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefaults, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-default-global-default", - Namespace: ns.GetName(), - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", - }, - }, - }, - } - - EventuallyCreation(func() error { - return k8sClient.Create(context.Background(), pod) - }).Should(Succeed()) - // Check if correct applied - Expect(pod.Spec.PriorityClassName).To(Equal(class.GetName())) - Expect(pod.Spec.Priority).To(Equal(&class.Value)) - Expect(pod.Spec.PreemptionPolicy).To(Equal(class.PreemptionPolicy)) + NamespaceIsPartOfTenant(tntWithDefaults, ns).Should(Succeed()) By("verify Status (Creation)", func() { Eventually(func() ([]string, error) { @@ -620,6 +734,33 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { }, defaultTimeoutInterval, defaultPollInterval). Should(ConsistOf(tenantDefault.GetName(), customerBronze.GetName())) }) + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-default-global-default", + Namespace: ns.GetName(), + }, + Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + Containers: []corev1.Container{ + { + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(context.Background(), pod) + }).Should(Succeed()) + + // Check if correct applied + Expect(pod.Spec.PriorityClassName).To(Equal(class.GetName())) + Expect(pod.Spec.Priority).To(Equal(&class.Value)) + Expect(pod.Spec.PreemptionPolicy).To(Equal(class.PreemptionPolicy)) }) It("should mutate to default tenant PriorityClass although the cluster global one is allowed", Label("skip-on-openshift"), func() { @@ -631,31 +772,11 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { global.SetResourceVersion("") Expect(k8sClient.Create(context.TODO(), global)).Should(Succeed()) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefaults.GetName(), + }) NamespaceCreation(ns, tntWithDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefaults, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-default-allowed", - Namespace: ns.GetName(), - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", - }, - }, - }, - } - - EventuallyCreation(func() error { - return k8sClient.Create(context.Background(), pod) - }).Should(Succeed()) - // Check if correctly applied - Expect(pod.Spec.PriorityClassName).To(Equal(class.GetName())) - Expect(*pod.Spec.Priority).To(Equal(class.Value)) + NamespaceIsPartOfTenant(tntWithDefaults, ns).Should(Succeed()) By("verify Status (Creation)", func() { Eventually(func() ([]string, error) { @@ -672,5 +793,30 @@ var _ = Describe("enforcing a Priority Class", Label("pod", "classes"), func() { }, defaultTimeoutInterval, defaultPollInterval). Should(ConsistOf(global.GetName(), tenantDefault.GetName(), customerBronze.GetName())) }) + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-default-allowed", + Namespace: ns.GetName(), + }, + Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + Containers: []corev1.Container{ + { + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(context.Background(), pod) + }).Should(Succeed()) + // Check if correctly applied + Expect(pod.Spec.PriorityClassName).To(Equal(class.GetName())) + Expect(*pod.Spec.Priority).To(Equal(class.Value)) }) }) diff --git a/e2e/pod_runtime_class_test.go b/e2e/pod_runtime_class_test.go index a6c8a83f..abf11256 100644 --- a/e2e/pod_runtime_class_test.go +++ b/e2e/pod_runtime_class_test.go @@ -19,21 +19,26 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" ) -var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { +var _ = Describe("enforcing a Runtime Class", Ordered, Label("pod", "classes", "runtimeclass"), func() { tntWithDefault := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-runtime-selection", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "george", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-runtimeclass-1", Kind: "User", }, }, @@ -48,7 +53,7 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { }, LabelSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ - "env": "customers", + "environment": "customers", }, }, }, @@ -59,13 +64,16 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { tntNoRestrictions := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-runtime-no-restrictions", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: []api.OwnerSpec{ + Owners: []rbac.OwnerSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "e2e-gateway-no-restrictions", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-runtimeclass-2", Kind: "User", }, }, @@ -78,8 +86,9 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "default-runtime", Labels: map[string]string{ - "name": "default-runtime", - "env": "customers", + "name": "default-runtime", + "environment": "customers", + "env": "e2e", }, }, Handler: "custom-handler", @@ -89,7 +98,8 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "legacy", Labels: map[string]string{ - "env": "e2e", + "environment": "disallowed", + "env": "e2e", }, }, Handler: "custom-handler", @@ -99,7 +109,8 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "disallowed", Labels: map[string]string{ - "env": "e2e", + "environment": "disallowed", + "env": "e2e", }, }, Handler: "custom-handler", @@ -109,8 +120,9 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "customer-containerd", Labels: map[string]string{ - "name": "customer-containerd", - "env": "customers", + "name": "customer-containerd", + "environment": "customers", + "env": "e2e", }, }, Handler: "custom-handler", @@ -120,8 +132,9 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "customer-virt", Labels: map[string]string{ - "name": "customer-virt", - "env": "customers", + "name": "customer-virt", + "environment": "customers", + "env": "e2e", }, }, Handler: "custom-handler", @@ -144,7 +157,10 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) } + for _, crd := range []*nodev1.RuntimeClass{legacy, disallowed, customerUni, customerKubevirt, customerContainerd} { Eventually(func() error { crd.ResourceVersion = "" @@ -155,25 +171,35 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { JustAfterEach(func() { for _, tnt := range []*capsulev1beta2.Tenant{tntWithDefault, tntNoRestrictions} { - EventuallyCreation(func() error { - return ignoreNotFound(k8sClient.Delete(context.TODO(), tnt)) - }).Should(Succeed()) + EventuallyDeletion(tnt) } - Eventually(func() (err error) { - req, _ := labels.NewRequirement("env", selection.Exists, nil) + req, err := labels.NewRequirement("env", selection.Equals, []string{"e2e"}) + Expect(err).NotTo(HaveOccurred()) - return k8sClient.DeleteAllOf(context.TODO(), &nodev1.RuntimeClass{}, &client.DeleteAllOfOptions{ - ListOptions: client.ListOptions{ - LabelSelector: labels.NewSelector().Add(*req), - }, - }) - }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + var list nodev1.RuntimeClassList + Expect(k8sClient.List( + context.TODO(), + &list, + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*req), + }, + )).Should(Succeed()) + + for i := range list.Items { + EventuallyDeletion(&list.Items[i]) + } }) It("should allow all classes", Label("skip-on-openshift"), func() { all := []string{customerUni.GetName(), customerKubevirt.GetName(), customerContainerd.GetName(), legacy.GetName(), disallowed.GetName()} + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoRestrictions.GetName(), + }) + NamespaceCreation(ns, tntNoRestrictions.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tntNoRestrictions, ns).Should(Succeed()) + By("Verify Status (Creation)", func() { Eventually(func() ([]string, error) { t := &capsulev1beta2.Tenant{} @@ -190,10 +216,6 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { Should(ConsistOf(all)) }) - ns := NewNamespace("") - NamespaceCreation(ns, tntNoRestrictions.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoRestrictions, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - By("providing any runtimeclass", func() { for _, class := range all { Eventually(func() (err error) { @@ -204,13 +226,16 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { Namespace: ns.GetName(), }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + RuntimeClassName: &class, Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, - RuntimeClassName: &class, }, } @@ -221,8 +246,9 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { By("Verify Status (Deletion)", func() { for _, crd := range []*nodev1.RuntimeClass{customerKubevirt} { - Expect(ignoreNotFound(k8sClient.Delete(context.TODO(), crd))).To(Succeed()) + EventuallyDeletion(crd) } + Eventually(func() ([]string, error) { t := &capsulev1beta2.Tenant{} if err := k8sClient.Get( @@ -241,21 +267,43 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { It("should block non allowed Runtime Class", func() { By("blocked disallowed runtime", func() { - ns := NewNamespace("rt-disallow") + ns := NewNamespace("rt-disallow", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) + + By("verify status", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntWithDefault.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.RuntimeClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ConsistOf(customerContainerd.GetName(), customerKubevirt.GetName(), legacy.GetName())) + }) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + RuntimeClassName: &disallowed.Name, Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, - RuntimeClassName: &disallowed.Name, }, } @@ -265,6 +313,14 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { return err }).ShouldNot(Succeed()) }) + }) + + It("should allow exact match", func() { + ns := NewNamespace("rt-exact-match", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) + NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) By("verify status", func() { Eventually(func() ([]string, error) { @@ -281,23 +337,22 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { }, defaultTimeoutInterval, defaultPollInterval). Should(ConsistOf(customerContainerd.GetName(), customerKubevirt.GetName(), legacy.GetName())) }) - }) - It("should allow exact match", func() { - ns := NewNamespace("rt-exact-match") - NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + RuntimeClassName: &legacy.Name, Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, - RuntimeClassName: &legacy.Name, }, } @@ -309,33 +364,73 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { }) It("should allow regex match", func() { - ns := NewNamespace("rc-regex-match") - + ns := NewNamespace("rc-regex-match", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) + + By("verify status", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntWithDefault.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.RuntimeClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ConsistOf(customerContainerd.GetName(), customerKubevirt.GetName(), legacy.GetName())) + }) for i, rt := range []string{"hardened-crio", "hardened-containerd", "hardened-dockerd"} { runtimeName := strings.Join([]string{rt, "-", strconv.Itoa(i)}, "") runtime := &nodev1.RuntimeClass{ ObjectMeta: metav1.ObjectMeta{ Name: runtimeName, + Labels: map[string]string{ + "env": "e2e", + }, }, Handler: "custom-handler", } Expect(k8sClient.Create(context.TODO(), runtime)).Should(Succeed()) + By("verify status", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntWithDefault.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.RuntimeClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ContainElement(runtime.GetName())) + }) + pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: rt, }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + RuntimeClassName: &runtimeName, Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, - RuntimeClassName: &runtimeName, }, } @@ -346,14 +441,16 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { return err }).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), runtime)).Should(Succeed()) + EventuallyDeletion(runtime) } }) It("should allow selector match", func() { - ns := NewNamespace("rc-selector-match") - + ns := NewNamespace("rc-selector-match", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) for i, rt := range []string{"customer-containerd", "customer-crio", "customer-dockerd"} { runtimeName := strings.Join([]string{rt, "-", strconv.Itoa(i)}, "") @@ -361,8 +458,9 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { ObjectMeta: metav1.ObjectMeta{ Name: runtimeName, Labels: map[string]string{ - "name": runtimeName, - "env": "customers", + "name": runtimeName, + "env": "e2e", + "environment": "customers", }, }, Handler: "custom-handler", @@ -370,18 +468,37 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { Expect(k8sClient.Create(context.TODO(), runtime)).Should(Succeed()) + By("verify status", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntWithDefault.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.RuntimeClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ContainElement(runtime.GetName())) + }) + pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: rt, }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + RuntimeClassName: &runtimeName, Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, - RuntimeClassName: &runtimeName, }, } @@ -411,22 +528,43 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { }) It("should auto assign the default", func() { - ns := NewNamespace("rc-default") - + ns := NewNamespace("rc-default", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) Expect(k8sClient.Create(context.TODO(), defaultRuntime)).Should(Succeed()) + By("verify status", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntWithDefault.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.RuntimeClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ConsistOf(defaultRuntime.GetName(), customerContainerd.GetName(), customerKubevirt.GetName(), legacy.GetName())) + }) + pod := corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "rc-default", Namespace: ns.Name, }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -445,21 +583,5 @@ var _ = Describe("enforcing a Runtime Class", Label("pod", "classes"), func() { Expect(createdPod.Spec.RuntimeClassName).NotTo(BeNil()) _, err := Equal(createdPod.Spec.RuntimeClassName).Match(tntWithDefault.Spec.RuntimeClasses.Default) Expect(err).NotTo(HaveOccurred()) - - By("verify status", func() { - Eventually(func() ([]string, error) { - t := &capsulev1beta2.Tenant{} - if err := k8sClient.Get( - context.TODO(), - types.NamespacedName{Name: tntWithDefault.GetName()}, - t, - ); err != nil { - return nil, err - } - - return t.Status.Classes.RuntimeClasses, nil - }, defaultTimeoutInterval, defaultPollInterval). - Should(ConsistOf(defaultRuntime.GetName(), customerContainerd.GetName(), customerKubevirt.GetName(), legacy.GetName())) - }) }) }) diff --git a/e2e/resourcepool_test.go b/e2e/pool_resourcepool_test.go similarity index 56% rename from e2e/resourcepool_test.go rename to e2e/pool_resourcepool_test.go index 917d66c6..15fa4074 100644 --- a/e2e/resourcepool_test.go +++ b/e2e/pool_resourcepool_test.go @@ -5,12 +5,15 @@ package e2e import ( "context" + "fmt" + "reflect" "slices" "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -24,7 +27,7 @@ import ( "github.com/projectcapsule/capsule/pkg/utils" ) -var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { +var _ = Describe("ResourcePool Tests", Ordered, Label("resourcepool", "pool"), func() { JustAfterEach(func() { Eventually(func() error { poolList := &capsulev1beta2.TenantList{} @@ -89,14 +92,14 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "defaults-pool", + "e2e.capsule.dev/test-suite": "defaults-pool", }, }, }, { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "defaults-pool", + "e2e.capsule.dev/test-suite": "defaults-pool", }, }, }, @@ -115,8 +118,11 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { namespaces := []string{"ns-1-default-pool", "ns-2-default-pool", "ns-3-default-pool"} By("Create the ResourcePool", func() { - err := k8sClient.Create(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to create ResourcePool %s", pool) + EventuallyCreation(func() error { + pool.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), pool) + }).Should(Succeed(), "Failed to create ResourcePool %s", pool) }) By("Get Applied revision", func() { @@ -133,19 +139,20 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }) By("Verify Status was correctly initialized", func() { - expected := &capsulev1beta2.ResourcePoolQuotaStatus{ - Hard: pool.Spec.Quota.Hard, - Claimed: corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("0"), - corev1.ResourceLimitsMemory: resource.MustParse("0"), - corev1.ResourceRequestsCPU: resource.MustParse("0"), - corev1.ResourceRequestsMemory: resource.MustParse("0"), - }, - Available: pool.Spec.Quota.Hard, - } + Eventually(func(g Gomega) { + expected := &capsulev1beta2.ResourcePoolQuotaStatus{ + Hard: pool.Spec.Quota.Hard, + Claimed: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("0"), + corev1.ResourceLimitsMemory: resource.MustParse("0"), + corev1.ResourceRequestsCPU: resource.MustParse("0"), + corev1.ResourceRequestsMemory: resource.MustParse("0"), + }, + Available: pool.Spec.Quota.Hard, + } - ok, msg := DeepCompare(*expected, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for expected status allocation: %s", msg) + ExpectPoolAllocation(pool.Name, *expected) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("Create Namespaces, which are selected by the pool", func() { @@ -153,8 +160,8 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "ns-1-default-pool", Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "defaults-pool", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "defaults-pool", }, }, } @@ -166,8 +173,8 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "ns-2-default-pool", Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "defaults-pool", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "defaults-pool", }, }, } @@ -179,8 +186,8 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "ns-3-default-pool", Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "defaults-pool", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "defaults-pool", }, }, } @@ -190,43 +197,21 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }) By("Verify Namespaces are shown as allowed targets", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) - Expect(err).Should(Succeed()) + Eventually(func(g Gomega) { + stat := &capsulev1beta2.ResourcePool{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, stat) + g.Expect(err).Should(Succeed()) - ok, msg := DeepCompare(namespaces, pool.Status.Namespaces) - Expect(ok).To(BeTrue(), "Mismatch for expected namespaces: %s", msg) - - Expect(pool.Status.NamespaceSize).To(Equal(uint(3))) + g.Expect(stat.Status.Namespaces).To(ConsistOf(namespaces)) + g.Expect(stat.Status.NamespaceSize).To(Equal(uint(len(namespaces)))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("Verify ResourceQuotas for namespaces", func() { - quotaLabel, err := utils.GetTypeLabel(&capsulev1beta2.ResourcePool{}) - Expect(err).Should(Succeed()) - for _, ns := range namespaces { - rq := &corev1.ResourceQuota{} - - err := k8sClient.Get(context.TODO(), client.ObjectKey{ - Name: pool.GetQuotaName(), - Namespace: ns}, - rq) - Expect(err).Should(Succeed()) - - Expect(rq.ObjectMeta.Labels[quotaLabel]).To(Equal(pool.Name), "Expected "+quotaLabel+" to be set to "+pool.Name) - - Expect(rq.Spec.Hard).To(BeNil()) - - found := false - for _, ref := range rq.OwnerReferences { - if ref.Kind == "ResourcePool" && ref.UID == pool.UID { - found = true - break - } - } - Expect(found).To(BeTrue(), "Expected ResourcePool to be owner of ResourceQuota in namespace %s", ns) + ExpectResourceQuotaEventually(ns, pool.GetQuotaName(), nil, pool.Name, pool.UID) } }) - By("Add Claims for namespaces", func() { claim1 := &capsulev1beta2.ResourcePoolClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -240,8 +225,11 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - err := k8sClient.Create(context.TODO(), claim1) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim1) + Eventually(func() error { + claim1.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), claim1) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed(), "Failed to create Claim %s", claim1) isSuccessfullyBoundAndUnsedToPool(pool, claim1) @@ -257,8 +245,11 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - err = k8sClient.Create(context.TODO(), claim2) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim2) + Eventually(func() error { + claim2.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), claim2) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed(), "Failed to create Claim %s", claim2) isSuccessfullyBoundAndUnsedToPool(pool, claim2) @@ -274,17 +265,97 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - err = k8sClient.Create(context.TODO(), claim3) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim3) + Eventually(func() error { + claim3.ResourceVersion = "" - Expect(isBoundToPool(pool, claim3)).To(BeFalse()) + return k8sClient.Create(context.TODO(), claim3) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed(), "Failed to create Claim %s", claim3) + + Eventually(func(g Gomega) { + fetchedPool := &capsulev1beta2.ResourcePool{} + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, fetchedPool)).To(Succeed()) + + fetchedClaim := &capsulev1beta2.ResourcePoolClaim{} + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKey{ + Name: claim3.Name, + Namespace: claim3.Namespace, + }, fetchedClaim)).To(Succeed()) + + g.Expect(isNotBoundToPool(fetchedPool, fetchedClaim)).To(BeTrue()) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("Verify Status was correctly initialized", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) - Expect(err).Should(Succeed()) + Eventually(func(g Gomega) { + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) + g.Expect(err).Should(Succeed()) - expected := &capsulev1beta2.ResourcePoolQuotaStatus{ + expected := &capsulev1beta2.ResourcePoolQuotaStatus{ + Hard: pool.Spec.Quota.Hard, + Claimed: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("0"), + corev1.ResourceRequestsMemory: resource.MustParse("0"), + corev1.ResourceRequestsCPU: resource.MustParse("0"), + corev1.ResourceLimitsMemory: resource.MustParse("640Mi"), + }, + Available: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("2"), + corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("2"), + corev1.ResourceLimitsMemory: resource.MustParse("1408Mi"), + }, + } + + ExpectPoolAllocation(pool.Name, *expected) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + }) + + By("Pool Has Finalizer", func() { + ExpectResourcePoolFinalizerEventually(pool.Name, true) + }) + + By("Verify ResourceQuotas for namespaces", func() { + status := map[string]corev1.ResourceList{ + "ns-1-default-pool": corev1.ResourceList{ + corev1.ResourceLimitsMemory: resource.MustParse("128Mi"), + }, + "ns-2-default-pool": corev1.ResourceList{ + corev1.ResourceLimitsMemory: resource.MustParse("512Mi"), + }, + "ns-3-default-pool": nil, + } + + for ns, expected := range status { + ExpectResourceQuotaEventually(ns, pool.GetQuotaName(), expected, pool.Name, pool.UID) + } + }) + + By("Update the ResourcePool", func() { + Eventually(func() error { + current := &capsulev1beta2.ResourcePool{} + if err := k8sClient.Get( + context.TODO(), + client.ObjectKey{Name: pool.Name}, + current, + ); err != nil { + return err + } + + current.Spec.Defaults = corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("1"), + corev1.ResourceLimitsMemory: resource.MustParse("1Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("1"), + corev1.ResourceRequestsMemory: resource.MustParse("1Gi"), + corev1.ResourceRequestsStorage: resource.MustParse("5Gi"), + } + + return k8sClient.Update(context.TODO(), current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + By("Wait for ResourcePool allocation after defaults update", func() { + expected := capsulev1beta2.ResourcePoolQuotaStatus{ Hard: pool.Spec.Quota.Hard, Claimed: corev1.ResourceList{ corev1.ResourceLimitsCPU: resource.MustParse("0"), @@ -300,64 +371,7 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - ok, msg := DeepCompare(*expected, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for expected status allocation: %s", msg) - }) - - By("Pool Has Finalizer", func() { - Expect(controllerutil.ContainsFinalizer(pool, meta.ControllerFinalizer)).To(BeTrue()) - }) - - By("Verify ResourceQuotas for namespaces", func() { - status := map[string]corev1.ResourceList{ - "ns-1-default-pool": corev1.ResourceList{ - corev1.ResourceLimitsMemory: resource.MustParse("128Mi"), - }, - "ns-2-default-pool": corev1.ResourceList{ - corev1.ResourceLimitsMemory: resource.MustParse("512Mi"), - }, - "ns-3-default-pool": nil, - } - - quotaLabel, err := utils.GetTypeLabel(&capsulev1beta2.ResourcePool{}) - Expect(err).Should(Succeed()) - - for _, ns := range namespaces { - rq := &corev1.ResourceQuota{} - - err := k8sClient.Get(context.TODO(), client.ObjectKey{ - Name: pool.GetQuotaName(), - Namespace: ns}, - rq) - Expect(err).Should(Succeed()) - - Expect(rq.ObjectMeta.Labels[quotaLabel]).To(Equal(pool.Name), "Expected "+quotaLabel+" to be set to "+pool.Name) - - ok, msg := DeepCompare(status[ns], rq.Spec.Hard) - Expect(ok).To(BeTrue(), "Mismatch for resources for resourcequota: %s", msg) - - found := false - for _, ref := range rq.OwnerReferences { - if ref.Kind == "ResourcePool" && ref.UID == pool.UID { - found = true - break - } - } - Expect(found).To(BeTrue(), "Expected ResourcePool to be owner of ResourceQuota in namespace %s", ns) - } - }) - - By("Update the ResourcePool", func() { - pool.Spec.Defaults = corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("1"), - corev1.ResourceLimitsMemory: resource.MustParse("1Gi"), - corev1.ResourceRequestsCPU: resource.MustParse("1"), - corev1.ResourceRequestsMemory: resource.MustParse("1Gi"), - corev1.ResourceRequestsStorage: resource.MustParse("5Gi"), - } - - err := k8sClient.Update(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to update ResourcePool %s", pool) + ExpectResourcePoolAllocationEventually(pool.Name, expected) }) By("Verify ResourceQuotas for namespaces", func() { @@ -386,20 +400,8 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) - Expect(err).Should(Succeed()) - - for _, ns := range namespaces { - rq := &corev1.ResourceQuota{} - - err := k8sClient.Get(context.TODO(), client.ObjectKey{ - Name: pool.GetQuotaName(), - Namespace: ns}, - rq) - Expect(err).Should(Succeed()) - - ok, msg := DeepCompare(status[ns], rq.Spec.Hard) - Expect(ok).To(BeTrue(), "Mismatch for resources for resourcequota: %s", msg) + for ns, expected := range status { + ExpectResourceQuotaEventually(ns, pool.GetQuotaName(), expected, pool.Name, pool.UID) } }) @@ -410,44 +412,40 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.Name}, ns) - Expect(err).Should(Succeed()) + Eventually(func(g Gomega) { + stat := &corev1.Namespace{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.Name}, stat) + g.Expect(err).Should(Succeed()) - ns.ObjectMeta.Labels = map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "do-not-select", - } + stat.ObjectMeta.Labels = map[string]string{ + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "do-not-select", + } - err = k8sClient.Update(context.TODO(), ns) - Expect(err).Should(Succeed()) + err = k8sClient.Update(context.TODO(), stat) + g.Expect(err).Should(Succeed()) + }).Should(Succeed()) }) - By("Verify Namespaces was removed as allowed targets", func() { + By("Verify Namespaces were removed as allowed targets", func() { expected := []string{"ns-1-default-pool", "ns-3-default-pool"} - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) - Expect(err).Should(Succeed()) + Eventually(func(g Gomega) { + stat := &capsulev1beta2.ResourcePool{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, stat) + g.Expect(err).Should(Succeed()) - ok, msg := DeepCompare(expected, pool.Status.Namespaces) - Expect(ok).To(BeTrue(), "Mismatch for expected namespaces: %s", msg) - - Expect(pool.Status.NamespaceSize).To(Equal(uint(2))) - Expect(pool.Status.ClaimSize).To(Equal(uint(1))) + g.Expect(stat.Status.Namespaces).To(ConsistOf(expected)) + g.Expect(stat.Status.NamespaceSize).To(Equal(uint(len(expected)))) + g.Expect(stat.Status.ClaimSize).To(Equal(uint(1))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("Verify ResourceQuota was cleaned up", func() { - rq := &corev1.ResourceQuota{} - Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKey{ - Name: pool.GetQuotaName(), - Namespace: "ns-2-default-pool", - }, rq) - }, "30s", "1s").ShouldNot(Succeed(), "Expected ResourceQuota to be deleted from namespace %s", "ns-2-default-pool") + ExpectResourceQuotaDeletedEventually("ns-2-default-pool", pool.GetQuotaName()) }) By("Verify Status was correctly initialized", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) - Expect(err).Should(Succeed()) expected := &capsulev1beta2.ResourcePoolQuotaStatus{ Hard: pool.Spec.Quota.Hard, @@ -465,11 +463,18 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) - Expect(err).Should(Succeed()) + Eventually(func(g Gomega) { + stat := &capsulev1beta2.ResourcePool{} - ok, msg := DeepCompare(*expected, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for expected status allocation: %s", msg) + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, stat) + g.Expect(err).Should(Succeed()) + + err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, stat) + g.Expect(err).Should(Succeed()) + + ok, msg := DeepCompare(*expected, stat.Status.Allocation) + g.Expect(ok).To(BeTrue(), "Mismatch for expected status allocation: %s", msg) + }).Should(Succeed()) }) By("Remove namespace from being selected (Delete Namespace)", func() { @@ -479,8 +484,7 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - err := k8sClient.Delete(context.TODO(), ns) - Expect(err).Should(Succeed()) + EventuallyDeletion(ns) }) By("Get Applied revision", func() { @@ -491,26 +495,25 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { By("Verify Namespaces was removed as allowed targets", func() { expected := []string{"ns-1-default-pool"} - ok, msg := DeepCompare(expected, pool.Status.Namespaces) - Expect(ok).To(BeTrue(), "Mismatch for expected namespaces: %s", msg) + Eventually(func(g Gomega) { + current := &capsulev1beta2.ResourcePool{} - Expect(pool.Status.NamespaceSize).To(Equal(uint(1))) + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, current) + g.Expect(err).Should(Succeed()) + + g.Expect(current.Status.Namespaces).To(ConsistOf(expected)) + g.Expect(current.Status.NamespaceSize).To(Equal(uint(len(expected)))) + g.Expect(current.Status.ClaimSize).To(Equal(uint(1))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("Delete Resourcepool", func() { - err := k8sClient.Delete(context.TODO(), pool) - Expect(err).Should(Succeed()) + EventuallyDeletion(pool) }) By("Ensure ResourceQuotas are cleaned up", func() { for _, ns := range namespaces { - rq := &corev1.ResourceQuota{} - Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKey{ - Name: pool.GetQuotaName(), - Namespace: ns, - }, rq) - }, "30s", "1s").ShouldNot(Succeed(), "Expected ResourceQuota to be deleted from namespace %s", ns) + ExpectResourceQuotaDeletedEventually(ns, pool.GetQuotaName()) } }) }) @@ -528,14 +531,14 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "no-defaults", + "e2e.capsule.dev/test-suite": "no-defaults", }, }, }, { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "no-defaults", + "e2e.capsule.dev/test-suite": "no-defaults", }, }, }, @@ -557,8 +560,11 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { namespaces := []string{"ns-1-zero-pool", "ns-2-zero-pool"} By("Create the ResourcePool", func() { - err := k8sClient.Create(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to create ResourcePool %s", pool) + EventuallyCreation(func() error { + pool.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), pool) + }).Should(Succeed(), "Failed to create ResourcePool %s", pool) }) By("Get Applied revision", func() { @@ -589,8 +595,7 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { Available: pool.Spec.Quota.Hard, } - ok, msg := DeepCompare(*expected, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for expected status allocation: %s", msg) + ExpectPoolAllocation(pool.Name, *expected) }) By("Create Namespaces, which are selected by the pool", func() { @@ -598,8 +603,8 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "ns-1-zero-pool", Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "no-defaults", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "no-defaults", }, }, } @@ -611,8 +616,8 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "ns-2-zero-pool", Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "no-defaults", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "no-defaults", }, }, } @@ -622,13 +627,14 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }) By("Verify Namespaces are shown as allowed targets", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) - Expect(err).Should(Succeed()) + Eventually(func(g Gomega) { + stat := &capsulev1beta2.ResourcePool{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, stat) + g.Expect(err).Should(Succeed()) - ok, msg := DeepCompare(namespaces, pool.Status.Namespaces) - Expect(ok).To(BeTrue(), "Mismatch for expected namespaces: %s", msg) - - Expect(pool.Status.NamespaceSize).To(Equal(uint(2))) + g.Expect(stat.Status.Namespaces).To(ConsistOf(namespaces)) + g.Expect(stat.Status.NamespaceSize).To(Equal(uint(2))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("Verify ResourceQuotas for namespaces", func() { @@ -639,49 +645,18 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { corev1.ResourceRequestsMemory: resource.MustParse("0"), } - quotaLabel, err := utils.GetTypeLabel(&capsulev1beta2.ResourcePool{}) - Expect(err).Should(Succeed()) - for _, ns := range namespaces { - rq := &corev1.ResourceQuota{} - - err := k8sClient.Get(context.TODO(), client.ObjectKey{ - Name: pool.GetQuotaName(), - Namespace: ns}, - rq) - Expect(err).Should(Succeed()) - - Expect(rq.ObjectMeta.Labels[quotaLabel]).To(Equal(pool.Name), "Expected "+quotaLabel+" to be set to "+pool.Name) - - ok, msg := DeepCompare(resources, rq.Spec.Hard) - Expect(ok).To(BeTrue(), "Mismatch for resources for resourcequota: %s", msg) - - found := false - for _, ref := range rq.OwnerReferences { - if ref.Kind == "ResourcePool" && ref.UID == pool.UID { - found = true - break - } - } - Expect(found).To(BeTrue(), "Expected ResourcePool to be owner of ResourceQuota in namespace %s", ns) - + ExpectResourceQuotaEventually(ns, pool.GetQuotaName(), resources, pool.Name, pool.UID) } }) By("Delete Resourcepool", func() { - err := k8sClient.Delete(context.TODO(), pool) - Expect(err).Should(Succeed()) + EventuallyDeletion(pool) }) By("Ensure ResourceQuotas are cleaned up", func() { for _, ns := range namespaces { - rq := &corev1.ResourceQuota{} - Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKey{ - Name: pool.GetQuotaName(), - Namespace: ns, - }, rq) - }, "30s", "1s").ShouldNot(Succeed(), "Expected ResourceQuota to be deleted from namespace %s", ns) + ExpectResourceQuotaDeletedEventually(ns, pool.GetQuotaName()) } }) @@ -700,7 +675,7 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "unordered-scheduling", + "e2e.capsule.dev/test-suite": "unordered-scheduling", }, }, }, @@ -720,8 +695,11 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { } By("Create the ResourcePool", func() { - err := k8sClient.Create(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to create ResourcePool %s", pool) + EventuallyCreation(func() error { + pool.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), pool) + }).Should(Succeed(), "Failed to create ResourcePool %s", pool) }) By("Create source namespaces", func() { @@ -729,8 +707,8 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "ns-1-pool-unordered", Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "unordered-scheduling", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "unordered-scheduling", }, }, } @@ -742,8 +720,8 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "ns-2-pool-unordered", Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "unordered-scheduling", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "unordered-scheduling", }, }, } @@ -792,8 +770,7 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - ok, msg := DeepCompare(*expected, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for expected status allocation: %s", msg) + ExpectPoolAllocation(pool.Name, *expected) }) By("Verify ResourceQuota", func() { @@ -801,16 +778,7 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { corev1.ResourceLimitsMemory: resource.MustParse("128Mi"), } - rq := &corev1.ResourceQuota{} - - err := k8sClient.Get(context.TODO(), client.ObjectKey{ - Name: pool.GetQuotaName(), - Namespace: "ns-1-pool-unordered"}, - rq) - Expect(err).Should(Succeed()) - - ok, msg := DeepCompare(rqHardResources, rq.Spec.Hard) - Expect(ok).To(BeTrue(), "Mismatch for resources for resourcequota: %s", msg) + ExpectResourceQuotaEventually("ns-1-pool-unordered", pool.GetQuotaName(), rqHardResources, pool.Name, pool.UID) }) By("Create claim exhausting requests.cpu", func() { @@ -829,21 +797,10 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { err := k8sClient.Create(context.TODO(), claim) Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) - Expect(isBoundToPool(pool, claim)).To(BeFalse()) - - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - expected := []string{ + assertClaimExhausted(pool, claim, meta.PoolExhaustedReason, []string{ "requested.requests.cpu=4", "available.requests.cpu=2", - } - - exhausted := claim.Status.Conditions.GetConditionByType(meta.ExhaustedCondition) - Expect(containsAll(extractResourcePoolMessage(exhausted.Message), expected)).To(BeTrue(), "Actual message"+exhausted.Message) - Expect(exhausted.Reason).To(Equal(meta.PoolExhaustedReason)) - Expect(exhausted.Status).To(Equal(metav1.ConditionTrue)) - Expect(exhausted.Type).To(Equal(meta.ExhaustedCondition)) + }) }) By("Create claim for request.memory", func() { @@ -885,8 +842,7 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - ok, msg := DeepCompare(*expected, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for expected status allocation: %s", msg) + ExpectPoolAllocation(pool.Name, *expected) }) By("Create claim for requests.cpu (skip exhausting one)", func() { @@ -928,29 +884,43 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - ok, msg := DeepCompare(*expected, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for expected status allocation: %s", msg) + ExpectPoolAllocation(pool.Name, *expected) }) By("Reverify claim exhausting requests.cpu", func() { - claim := &capsulev1beta2.ResourcePoolClaim{} - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: "simple-2", Namespace: "ns-1-pool-unordered"}, claim) - Expect(err).Should(Succeed()) - - Expect(isBoundToPool(pool, claim)).To(BeFalse()) - - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - exhausted := claim.Status.Conditions.GetConditionByType(meta.ExhaustedCondition) expected := []string{ "requested.requests.cpu=4", "available.requests.cpu=0", } - Expect(containsAll(extractResourcePoolMessage(exhausted.Message), expected)).To(BeTrue(), "Actual message"+claim.Status.Condition.Message) - Expect(exhausted.Reason).To(Equal(meta.PoolExhaustedReason)) - Expect(exhausted.Status).To(Equal(metav1.ConditionTrue)) + Eventually(func(g Gomega) { + fetchedPool := &capsulev1beta2.ResourcePool{} + g.Expect(k8sClient.Get( + context.TODO(), + client.ObjectKey{Name: pool.Name}, + fetchedPool, + )).To(Succeed()) + + claim := &capsulev1beta2.ResourcePoolClaim{} + g.Expect(k8sClient.Get( + context.TODO(), + client.ObjectKey{Name: "simple-2", Namespace: "ns-1-pool-unordered"}, + claim, + )).To(Succeed()) + + g.Expect(fetchedPool.GetClaimFromStatus(claim)).To(BeNil()) + + exhausted := claim.Status.Conditions.GetConditionByType(meta.ExhaustedCondition) + g.Expect(exhausted).NotTo(BeNil(), "Exhausted condition should be present") + + g.Expect(containsAll( + extractResourcePoolMessage(exhausted.Message), + expected, + )).To(BeTrue(), "Actual message: %s", exhausted.Message) + + g.Expect(exhausted.Reason).To(Equal(meta.PoolExhaustedReason)) + g.Expect(exhausted.Status).To(Equal(metav1.ConditionTrue)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) }) @@ -967,7 +937,7 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "ordered-scheduling", + "e2e.capsule.dev/test-suite": "ordered-scheduling", }, }, }, @@ -987,8 +957,11 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { } By("Create the ResourcePool", func() { - err := k8sClient.Create(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to create ResourcePool %s", pool) + EventuallyCreation(func() error { + pool.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), pool) + }).Should(Succeed(), "Failed to create ResourcePool %s", pool) }) By("Create source namespaces", func() { @@ -996,8 +969,8 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "ns-1-pool-ordered", Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "ordered-scheduling", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "ordered-scheduling", }, }, } @@ -1009,8 +982,8 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "ns-2-pool-ordered", Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "ordered-scheduling", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "ordered-scheduling", }, }, } @@ -1059,9 +1032,6 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }) By("Verify Status was correctly initialized", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) - Expect(err).Should(Succeed()) - expected := &capsulev1beta2.ResourcePoolQuotaStatus{ Hard: pool.Spec.Quota.Hard, Claimed: corev1.ResourceList{ @@ -1078,8 +1048,13 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - ok, msg := DeepCompare(*expected, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for expected status allocation: %s", msg) + Eventually(func(g Gomega) { + current := &capsulev1beta2.ResourcePool{} + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, current)).To(Succeed()) + + ok, msg := DeepCompare(*expected, current.Status.Allocation) + g.Expect(ok).To(BeTrue(), "Mismatch for expected status allocation: %s", msg) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("Create claim exhausting requests.cpu", func() { @@ -1095,23 +1070,15 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - err := k8sClient.Create(context.TODO(), claim) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) + Eventually(func() error { + claim.ResourceVersion = "" + return k8sClient.Create(context.TODO(), claim) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed(), "Failed to create Claim %s", claim) - Expect(isBoundToPool(pool, claim)).To(BeFalse()) - - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - exhausted := claim.Status.Conditions.GetConditionByType(meta.ExhaustedCondition) - expected := []string{ + assertClaimExhausted(pool, claim, meta.PoolExhaustedReason, []string{ "requested.requests.cpu=4", "available.requests.cpu=2", - } - - Expect(containsAll(extractResourcePoolMessage(exhausted.Message), expected)).To(BeTrue(), "Actual message"+exhausted.Message) - Expect(exhausted.Reason).To(Equal(meta.PoolExhaustedReason)) - Expect(exhausted.Status).To(Equal(metav1.ConditionTrue)) + }) }) By("Create claim exhausting limits.cpu", func() { @@ -1127,23 +1094,15 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - err := k8sClient.Create(context.TODO(), claim) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) + Eventually(func() error { + claim.ResourceVersion = "" + return k8sClient.Create(context.TODO(), claim) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed(), "Failed to create Claim %s", claim) - Expect(isBoundToPool(pool, claim)).To(BeFalse()) - - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - exhausted := claim.Status.Conditions.GetConditionByType(meta.ExhaustedCondition) - expected := []string{ + assertClaimExhausted(pool, claim, meta.PoolExhaustedReason, []string{ "requested.limits.cpu=4", "available.limits.cpu=2", - } - - Expect(containsAll(extractResourcePoolMessage(exhausted.Message), expected)).To(BeTrue(), "Actual message"+exhausted.Message) - Expect(exhausted.Reason).To(Equal(meta.PoolExhaustedReason)) - Expect(exhausted.Status).To(Equal(metav1.ConditionTrue)) + }) }) By("Create claim for requests.cpu (attempt to skip exhausting one)", func() { @@ -1160,75 +1119,73 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - err := k8sClient.Create(context.TODO(), claim) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) + Eventually(func() error { + claim.ResourceVersion = "" + return k8sClient.Create(context.TODO(), claim) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed(), "Failed to create Claim %s", claim) - Expect(isBoundToPool(pool, claim)).To(BeFalse()) - - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - exhausted := claim.Status.Conditions.GetConditionByType(meta.ExhaustedCondition) - expected := []string{ + assertClaimExhausted(pool, claim, meta.QueueExhaustedReason, []string{ "requested.limits.cpu=2", "queued.limits.cpu=4", "requested.requests.cpu=2", "queued.requests.cpu=4", - } - - Expect(containsAll(extractResourcePoolMessage(exhausted.Message), expected)).To(BeTrue(), "Actual message"+exhausted.Message) - Expect(exhausted.Reason).To(Equal(meta.QueueExhaustedReason)) - Expect(exhausted.Status).To(Equal(metav1.ConditionTrue)) + }) }) By("Verify ResourceQuotas for namespaces", func() { - namespaces := []string{"ns-1-pool-ordered", "ns-2-pool-ordered"} - status := map[string]corev1.ResourceList{ - - "ns-1-pool-ordered": corev1.ResourceList{ + "ns-1-pool-ordered": { corev1.ResourceLimitsMemory: resource.MustParse("512Mi"), }, - "ns-2-pool-ordered": corev1.ResourceList{ + "ns-2-pool-ordered": { corev1.ResourceRequestsMemory: resource.MustParse("750Mi"), }, } - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) - Expect(err).Should(Succeed()) + Eventually(func(g Gomega) { + for ns, expected := range status { + rq := &corev1.ResourceQuota{} - for _, ns := range namespaces { - rq := &corev1.ResourceQuota{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{ + Name: pool.GetQuotaName(), + Namespace: ns, + }, rq) + g.Expect(err).Should(Succeed()) - err := k8sClient.Get(context.TODO(), client.ObjectKey{ - Name: pool.GetQuotaName(), - Namespace: ns}, - rq) - Expect(err).Should(Succeed()) - - ok, msg := DeepCompare(status[ns], rq.Spec.Hard) - Expect(ok).To(BeTrue(), "Mismatch for resources for resourcequota: %s", msg) - } + ok, msg := DeepCompare(expected, rq.Spec.Hard) + g.Expect(ok).To(BeTrue(), "Mismatch for resources for resourcequota: %s", msg) + } + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("Allocate more resources to Resourcepool (requests.cpu)", func() { - pool.Spec.Quota.Hard = corev1.ResourceList{ + Eventually(func() error { + current := &capsulev1beta2.ResourcePool{} + if err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, current); err != nil { + return err + } + + current.Spec.Quota.Hard = corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("4"), + corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("4"), + corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), + } + + return k8sClient.Update(context.TODO(), current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + By("Verify Status was correctly initialized", func() { + expectedHard := corev1.ResourceList{ corev1.ResourceLimitsCPU: resource.MustParse("4"), corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), corev1.ResourceRequestsCPU: resource.MustParse("4"), corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), } - err := k8sClient.Update(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to allocate Resourcepool %s", pool) - }) - - By("Verify Status was correctly initialized", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) - Expect(err).Should(Succeed()) - expected := &capsulev1beta2.ResourcePoolQuotaStatus{ - Hard: pool.Spec.Quota.Hard, + Hard: expectedHard, Claimed: corev1.ResourceList{ corev1.ResourceLimitsCPU: resource.MustParse("4"), corev1.ResourceRequestsMemory: resource.MustParse("750Mi"), @@ -1243,15 +1200,25 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - ok, msg := DeepCompare(*expected, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for expected status allocation: %s", msg) + Eventually(func(g Gomega) { + current := &capsulev1beta2.ResourcePool{} + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, current)).To(Succeed()) + + ok, msg := DeepCompare(*expected, current.Status.Allocation) + g.Expect(ok).To(BeTrue(), "Mismatch for expected status allocation: %s", msg) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("Verify queued claim can be allocated", func() { claim := &capsulev1beta2.ResourcePoolClaim{} - err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: "simple-2", Namespace: "ns-2-pool-ordered"}, claim) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) + Eventually(func() error { + return k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: "simple-2", Namespace: "ns-2-pool-ordered"}, + claim, + ) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) isSuccessfullyBoundAndUnsedToPool(pool, claim) }) @@ -1259,54 +1226,46 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { By("Verify queued claim can be allocated", func() { claim := &capsulev1beta2.ResourcePoolClaim{} - err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: "simple-3", Namespace: "ns-1-pool-ordered"}, claim) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) + Eventually(func() error { + return k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: "simple-3", Namespace: "ns-1-pool-ordered"}, + claim, + ) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) isSuccessfullyBoundAndUnsedToPool(pool, claim) }) By("Verify ResourceQuotas for namespaces", func() { - namespaces := []string{"ns-1-pool-ordered", "ns-2-pool-ordered"} status := map[string]corev1.ResourceList{ - "ns-1-pool-ordered": corev1.ResourceList{ + "ns-1-pool-ordered": { corev1.ResourceLimitsMemory: resource.MustParse("512Mi"), corev1.ResourceLimitsCPU: resource.MustParse("4"), }, - "ns-2-pool-ordered": corev1.ResourceList{ + "ns-2-pool-ordered": { corev1.ResourceRequestsMemory: resource.MustParse("750Mi"), corev1.ResourceRequestsCPU: resource.MustParse("4"), }, } - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) - Expect(err).Should(Succeed()) + Eventually(func(g Gomega) { + for ns, expected := range status { + rq := &corev1.ResourceQuota{} - for _, ns := range namespaces { - rq := &corev1.ResourceQuota{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{ + Name: pool.GetQuotaName(), + Namespace: ns, + }, rq) + g.Expect(err).Should(Succeed()) - err := k8sClient.Get(context.TODO(), client.ObjectKey{ - Name: pool.GetQuotaName(), - Namespace: ns}, - rq) - Expect(err).Should(Succeed()) - - ok, msg := DeepCompare(status[ns], rq.Spec.Hard) - Expect(ok).To(BeTrue(), "Mismatch for resources for resourcequota: %s", msg) - } + ok, msg := DeepCompare(expected, rq.Spec.Hard) + g.Expect(ok).To(BeTrue(), "Mismatch for resources for resourcequota: %s", msg) + } + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("Verify moved up in queue", func() { - claim := &capsulev1beta2.ResourcePoolClaim{} - - err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: "simple-4", Namespace: "ns-2-pool-ordered"}, claim) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) - - Expect(isBoundToPool(pool, claim)).To(BeFalse()) - - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - exhausted := claim.Status.Conditions.GetConditionByType(meta.ExhaustedCondition) expected := []string{ "requested.limits.cpu=2", "available.limits.cpu=0", @@ -1314,9 +1273,30 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { "available.requests.cpu=0", } - Expect(containsAll(extractResourcePoolMessage(exhausted.Message), expected)).To(BeTrue(), "Actual message"+exhausted.Message) - Expect(exhausted.Reason).To(Equal(meta.PoolExhaustedReason)) - Expect(exhausted.Status).To(Equal(metav1.ConditionTrue)) + Eventually(func(g Gomega) { + currentPool := &capsulev1beta2.ResourcePool{} + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, currentPool)).To(Succeed()) + + claim := &capsulev1beta2.ResourcePoolClaim{} + g.Expect(k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: "simple-4", Namespace: "ns-2-pool-ordered"}, + claim, + )).To(Succeed()) + + g.Expect(currentPool.GetClaimFromStatus(claim)).To(BeNil()) + + exhausted := claim.Status.Conditions.GetConditionByType(meta.ExhaustedCondition) + g.Expect(exhausted).NotTo(BeNil(), "Exhausted condition should be present") + + g.Expect(containsAll( + extractResourcePoolMessage(exhausted.Message), + expected, + )).To(BeTrue(), "Actual message: %s", exhausted.Message) + + g.Expect(exhausted.Reason).To(Equal(meta.PoolExhaustedReason)) + g.Expect(exhausted.Status).To(Equal(metav1.ConditionTrue)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) }) @@ -1333,7 +1313,7 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "bind-namespaces", + "e2e.capsule.dev/test-suite": "bind-namespaces", }, }, }, @@ -1350,8 +1330,10 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { } By("Create the ResourcePool", func() { - err := k8sClient.Create(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to create ResourcePool %s", pool) + EventuallyCreation(func() error { + pool.ResourceVersion = "" + return k8sClient.Create(context.TODO(), pool) + }).Should(Succeed(), "Failed to create ResourcePool %s", pool) }) By("Create source namespaces", func() { @@ -1359,39 +1341,44 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "ns-1-pool-bind", Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "bind-namespaces", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "bind-namespaces", }, }, } - err := k8sClient.Create(context.TODO(), ns1) - Expect(err).Should(Succeed(), "Failed to create Namespace %s", ns1) + EventuallyCreation(func() error { + ns1.ResourceVersion = "" + return k8sClient.Create(context.TODO(), ns1) + }).Should(Succeed(), "Failed to create Namespace %s", ns1) ns2 := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "ns-2-pool-bind", Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "bind-namespaces-no", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "bind-namespaces-no", }, }, } - err = k8sClient.Create(context.TODO(), ns2) - Expect(err).Should(Succeed(), "Failed to create Namespace %s", ns2) + EventuallyCreation(func() error { + ns2.ResourceVersion = "" + return k8sClient.Create(context.TODO(), ns2) + }).Should(Succeed(), "Failed to create Namespace %s", ns2) }) By("Verify only matching namespaces", func() { expected := []string{"ns-1-pool-bind"} - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) - Expect(err).Should(Succeed()) + Eventually(func(g Gomega) { + stat := &capsulev1beta2.ResourcePool{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, stat) + g.Expect(err).Should(Succeed()) - ok, msg := DeepCompare(expected, pool.Status.Namespaces) - Expect(ok).To(BeTrue(), "Mismatch for expected namespaces: %s", msg) - - Expect(pool.Status.NamespaceSize).To(Equal(uint(1))) + g.Expect(stat.Status.Namespaces).To(ConsistOf(expected)) + g.Expect(stat.Status.NamespaceSize).To(Equal(uint(1))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("Create claim in matching namespace", func() { @@ -1407,8 +1394,10 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - err := k8sClient.Create(context.TODO(), claim) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) + EventuallyCreation(func() error { + claim.ResourceVersion = "" + return k8sClient.Create(context.TODO(), claim) + }).Should(Succeed(), "Failed to create Claim %s", claim) isSuccessfullyBoundAndUnsedToPool(pool, claim) }) @@ -1427,50 +1416,62 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { }, } - err := k8sClient.Create(context.TODO(), claim) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) + EventuallyCreation(func() error { + claim.ResourceVersion = "" + return k8sClient.Create(context.TODO(), claim) + }).Should(Succeed(), "Failed to create Claim %s", claim) - Expect(isBoundToPool(pool, claim)).To(BeFalse()) + Eventually(func(g Gomega) { + currentPool := &capsulev1beta2.ResourcePool{} + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, currentPool)).To(Succeed()) - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) + currentClaim := &capsulev1beta2.ResourcePoolClaim{} + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKey{ + Name: claim.Name, + Namespace: claim.Namespace, + }, currentClaim)).To(Succeed()) - assigned := claim.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(currentPool.GetClaimFromStatus(currentClaim)).To(BeNil()) - Expect(assigned.Reason).To(Equal(meta.FailedReason)) - Expect(assigned.Status).To(Equal(metav1.ConditionFalse)) - Expect(assigned.Type).To(Equal(meta.ReadyCondition)) + assigned := currentClaim.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(assigned).NotTo(BeNil(), "Ready condition should be present") + + g.Expect(assigned.Reason).To(Equal(meta.FailedReason)) + g.Expect(assigned.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(assigned.Type).To(Equal(meta.ReadyCondition)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("Update Namespace Labels to become matching", func() { - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ns-2-pool-bind", - Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "bind-namespaces", - }, - }, - } + Eventually(func() error { + ns := &corev1.Namespace{} + if err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: "ns-2-pool-bind"}, ns); err != nil { + return err + } - err := k8sClient.Update(context.TODO(), ns) - Expect(err).Should(Succeed(), "Failed to update namespace %s", ns) + if ns.Labels == nil { + ns.Labels = map[string]string{} + } + + ns.Labels["e2e-resourcepool"] = "test" + ns.Labels["e2e.capsule.dev/test-suite"] = "bind-namespaces" + + return k8sClient.Update(context.TODO(), ns) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("Reverify claim in namespace", func() { - claim := &capsulev1beta2.ResourcePoolClaim{ - ObjectMeta: metav1.ObjectMeta{ + claim := &capsulev1beta2.ResourcePoolClaim{} + + Eventually(func() error { + return k8sClient.Get(context.TODO(), client.ObjectKey{ Name: "simple-1", Namespace: "ns-2-pool-bind", - }, - } - - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) + }, claim) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) isSuccessfullyBoundAndUnsedToPool(pool, claim) }) - }) It("ResourcePool Deletion - Not Cascading", func() { @@ -1486,7 +1487,7 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "delete-bound-resources", + "e2e.capsule.dev/test-suite": "delete-bound-resources", }, }, }, @@ -1503,8 +1504,11 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { } By("Create the ResourcePool", func() { - err := k8sClient.Create(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to create ResourcePool %s", pool) + EventuallyCreation(func() error { + pool.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), pool) + }).Should(Succeed(), "Failed to create ResourcePool %s", pool) }) By("Unbinding From Pool", func() { @@ -1536,8 +1540,8 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: claim1.Namespace, Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "delete-bound-resources", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "delete-bound-resources", }, }, } @@ -1546,38 +1550,68 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: claim2.Namespace, Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "delete-bound-resources", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "delete-bound-resources", }, }, } - err := k8sClient.Create(context.TODO(), ns1) - Expect(err).Should(Succeed()) - err = k8sClient.Create(context.TODO(), ns2) - Expect(err).Should(Succeed()) - err = k8sClient.Create(context.TODO(), claim1) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim1) - err = k8sClient.Create(context.TODO(), claim2) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim1) + EventuallyCreation(func() error { + ns1.ResourceVersion = "" + return k8sClient.Create(context.TODO(), ns1) + }).Should(Succeed(), "Failed to create Namespace %s", ns1.Name) - isBoundToPool(pool, claim1) - isBoundToPool(pool, claim2) + EventuallyCreation(func() error { + ns2.ResourceVersion = "" + return k8sClient.Create(context.TODO(), ns2) + }).Should(Succeed(), "Failed to create Namespace %s", ns2.Name) - err = k8sClient.Delete(context.TODO(), pool) + ExpectResourcePoolNamespacesEventually(pool.Name, []string{ + ns1.Name, + ns2.Name, + }) + + EventuallyCreation(func() error { + claim1.ResourceVersion = "" + return k8sClient.Create(context.TODO(), claim1) + }).Should(Succeed(), "Failed to create Claim %s/%s", claim1.Namespace, claim1.Name) + + EventuallyCreation(func() error { + claim2.ResourceVersion = "" + return k8sClient.Create(context.TODO(), claim2) + }).Should(Succeed(), "Failed to create Claim %s/%s", claim2.Namespace, claim2.Name) + + isSuccessfullyBoundAndUnsedToPool(pool, claim1) + isSuccessfullyBoundAndUnsedToPool(pool, claim2) + + err := k8sClient.Delete(context.TODO(), pool) Expect(err).Should(Succeed(), "Failed to delete Pool %s", claim1) Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(claim1), &capsulev1beta2.ResourcePoolClaim{}) - }).Should(Succeed(), "Expected claim1 to be gone") + return k8sClient.Get( + context.TODO(), + client.ObjectKey{Name: claim1.Name, Namespace: claim1.Namespace}, + &capsulev1beta2.ResourcePoolClaim{}, + ) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed(), "Expected claim1 to be present") Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(claim2), &capsulev1beta2.ResourcePoolClaim{}) - }).Should(Succeed(), "Expected claim2 to be present") + return k8sClient.Get( + context.TODO(), + client.ObjectKey{Name: claim2.Name, Namespace: claim2.Namespace}, + &capsulev1beta2.ResourcePoolClaim{}, + ) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed(), "Expected claim2 to be present") - Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(pool), &capsulev1beta2.ResourcePoolClaim{}) - }).ShouldNot(Succeed(), "Expected pool to be gone") + Eventually(func() bool { + err := k8sClient.Get( + context.TODO(), + client.ObjectKey{Name: pool.Name}, + &capsulev1beta2.ResourcePool{}, + ) + + return apierrors.IsNotFound(err) + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue(), "Expected pool to be gone") }) }) @@ -1594,7 +1628,7 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "delete-bound-resources", + "e2e.capsule.dev/test-suite": "delete-bound-resources", }, }, }, @@ -1611,8 +1645,11 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { } By("Create the ResourcePool", func() { - err := k8sClient.Create(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to create ResourcePool %s", pool) + EventuallyCreation(func() error { + pool.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), pool) + }).Should(Succeed(), "Failed to create ResourcePool %s", pool) }) By("Cascading Deletion", func() { @@ -1644,8 +1681,8 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: claim1.Namespace, Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "delete-bound-resources", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "delete-bound-resources", }, }, } @@ -1654,38 +1691,54 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: claim2.Namespace, Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "delete-bound-resources", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "delete-bound-resources", }, }, } - err := k8sClient.Create(context.TODO(), ns1) - Expect(err).Should(Succeed()) - err = k8sClient.Create(context.TODO(), ns2) - Expect(err).Should(Succeed()) - err = k8sClient.Create(context.TODO(), claim1) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim1) - err = k8sClient.Create(context.TODO(), claim2) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim1) + EventuallyCreation(func() error { + ns1.ResourceVersion = "" + return k8sClient.Create(context.TODO(), ns1) + }).Should(Succeed(), "Failed to create Namespace %s", ns1.Name) - isBoundToPool(pool, claim1) - isBoundToPool(pool, claim2) + EventuallyCreation(func() error { + ns2.ResourceVersion = "" + return k8sClient.Create(context.TODO(), ns2) + }).Should(Succeed(), "Failed to create Namespace %s", ns2.Name) - err = k8sClient.Delete(context.TODO(), pool) + ExpectResourcePoolNamespacesEventually(pool.Name, []string{ + ns1.Name, + ns2.Name, + }) + + EventuallyCreation(func() error { + claim1.ResourceVersion = "" + return k8sClient.Create(context.TODO(), claim1) + }).Should(Succeed(), "Failed to create Claim %s/%s", claim1.Namespace, claim1.Name) + + EventuallyCreation(func() error { + claim2.ResourceVersion = "" + return k8sClient.Create(context.TODO(), claim2) + }).Should(Succeed(), "Failed to create Claim %s/%s", claim2.Namespace, claim2.Name) + + isSuccessfullyBoundAndUnsedToPool(pool, claim1) + isSuccessfullyBoundAndUnsedToPool(pool, claim2) + + err := k8sClient.Delete(context.TODO(), pool) Expect(err).Should(Succeed(), "Failed to delete Pool %s", claim1) - Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(claim1), &capsulev1beta2.ResourcePoolClaim{}) - }).ShouldNot(Succeed(), "Expected claim1 to be gone") + Eventually(func() bool { + err := k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(claim1), &capsulev1beta2.ResourcePoolClaim{}) + return apierrors.IsNotFound(err) + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue(), "Expected claim1 to be gone") - Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(claim2), &capsulev1beta2.ResourcePoolClaim{}) - }).ShouldNot(Succeed(), "Expected claim2 to be gone") + Eventually(func() bool { + err := k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(claim2), &capsulev1beta2.ResourcePoolClaim{}) + return apierrors.IsNotFound(err) + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue(), "Expected claim2 to be gone") - Eventually(func() error { - return k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(pool), &capsulev1beta2.ResourcePoolClaim{}) - }).ShouldNot(Succeed(), "Expected pool to be gone") + ExpectResourcePoolDeletedEventually(pool.Name) }) }) @@ -1702,7 +1755,7 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { { LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "admission", + "e2e.capsule.dev/test-suite": "admission", }, }, }, @@ -1722,8 +1775,11 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { } By("Create the ResourcePool", func() { - err := k8sClient.Create(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to create ResourcePool %s", pool) + EventuallyCreation(func() error { + pool.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), pool) + }).Should(Succeed(), "Failed to create ResourcePool %s", pool) }) By("Create Namespaces, which are selected by the pool", func() { @@ -1731,8 +1787,8 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "ns-1-admission-pool", Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "admission", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "admission", }, }, } @@ -1744,14 +1800,20 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { ObjectMeta: metav1.ObjectMeta{ Name: "ns-2-admission-pool", Labels: map[string]string{ - "e2e-resourcepool": "test", - "capsule.clastix.io/tenant": "admission", + "e2e-resourcepool": "test", + "e2e.capsule.dev/test-suite": "admission", }, }, } err = k8sClient.Create(context.TODO(), ns2) Expect(err).Should(Succeed()) + + ExpectResourcePoolNamespacesEventually(pool.Name, []string{ + ns1.Name, + ns2.Name, + }) + }) By("Create claims", func() { @@ -1817,20 +1879,16 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) Expect(err).Should(Succeed()) - ok, msg := DeepCompare(expectedAllocation, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for resource allocation: %s", msg) + ExpectPoolAllocation(pool.Name, expectedAllocation) }) By("Allow increasing the size of the pool", func() { - pool.Spec.Quota.Hard = corev1.ResourceList{ + UpdatePoolEventually(pool.Name, corev1.ResourceList{ corev1.ResourceLimitsCPU: resource.MustParse("4"), corev1.ResourceLimitsMemory: resource.MustParse("4Gi"), corev1.ResourceRequestsCPU: resource.MustParse("2"), corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), - } - - err := k8sClient.Update(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to update ResourcePool %s", pool) + }) }) By("Verify ResourcePool Status Allocation", func() { @@ -1858,20 +1916,16 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) Expect(err).Should(Succeed()) - ok, msg := DeepCompare(expectedAllocation, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for resource allocation: %s", msg) + ExpectPoolAllocation(pool.Name, expectedAllocation) }) By("Allow Decreasing", func() { - pool.Spec.Quota.Hard = corev1.ResourceList{ + UpdatePoolEventually(pool.Name, corev1.ResourceList{ corev1.ResourceLimitsCPU: resource.MustParse("2"), corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), corev1.ResourceRequestsCPU: resource.MustParse("2"), corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), - } - - err := k8sClient.Update(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to update ResourcePool %s", pool) + }) }) By("Verify ResourcePool Status Allocation", func() { @@ -1899,30 +1953,23 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) Expect(err).Should(Succeed()) - ok, msg := DeepCompare(expectedAllocation, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for resource allocation: %s", msg) + ExpectPoolAllocation(pool.Name, expectedAllocation) }) By("Don't allow Decreasing under claimed usage", func() { - pool.Spec.Quota.Hard = corev1.ResourceList{ + UpdatePoolShouldFail(pool.Name, corev1.ResourceList{ corev1.ResourceLimitsCPU: resource.MustParse("2"), corev1.ResourceLimitsMemory: resource.MustParse("10Mi"), corev1.ResourceRequestsCPU: resource.MustParse("0.5"), corev1.ResourceRequestsMemory: resource.MustParse("128Mi"), - } - - err := k8sClient.Update(context.TODO(), pool) - Expect(err).ShouldNot(Succeed(), "Update to ResourcePool %s should be blocked", pool) + }) }) By("May Remove unused resources", func() { - pool.Spec.Quota.Hard = corev1.ResourceList{ + UpdatePoolEventually(pool.Name, corev1.ResourceList{ corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), - } - - err := k8sClient.Update(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to update ResourcePool %s", pool) + }) }) By("Verify ResourcePool Status Allocation", func() { @@ -1944,18 +1991,14 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) Expect(err).Should(Succeed()) - ok, msg := DeepCompare(expectedAllocation, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for resource allocation: %s", msg) + ExpectPoolAllocation(pool.Name, expectedAllocation) }) By("May Decrase to actual usage", func() { - pool.Spec.Quota.Hard = corev1.ResourceList{ + UpdatePoolEventually(pool.Name, corev1.ResourceList{ corev1.ResourceLimitsMemory: resource.MustParse("1Gi"), corev1.ResourceRequestsMemory: resource.MustParse("512Mi"), - } - - err := k8sClient.Update(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to update ResourcePool %s", pool) + }) }) By("Verify ResourcePool Status Allocation", func() { @@ -1977,93 +2020,160 @@ var _ = Describe("ResourcePool Tests", Label("resourcepool"), func() { err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) Expect(err).Should(Succeed()) - ok, msg := DeepCompare(expectedAllocation, pool.Status.Allocation) - Expect(ok).To(BeTrue(), "Mismatch for resource allocation: %s", msg) + ExpectPoolAllocation(pool.Name, expectedAllocation) }) By("May not set 0 on usage", func() { - pool.Spec.Quota.Hard = corev1.ResourceList{ + UpdatePoolShouldFail(pool.Name, corev1.ResourceList{ corev1.ResourceLimitsMemory: resource.MustParse("0"), corev1.ResourceRequestsMemory: resource.MustParse("0"), - } - - err := k8sClient.Update(context.TODO(), pool) - Expect(err).ShouldNot(Succeed(), "Update to ResourcePool %s should be blocked", pool) + }) }) By("May not remove resource in use", func() { - pool.Spec.Quota.Hard = corev1.ResourceList{ + UpdatePoolShouldFail(pool.Name, corev1.ResourceList{ corev1.ResourceRequestsCPU: resource.MustParse("1"), - } - - err := k8sClient.Update(context.TODO(), pool) - Expect(err).ShouldNot(Succeed(), "Update to ResourcePool %s should be blocked", pool) + }) }) - }) }) func isSuccessfullyBoundAndUnsedToPool(pool *capsulev1beta2.ResourcePool, claim *capsulev1beta2.ResourcePoolClaim) { - fetchedPool := &capsulev1beta2.ResourcePool{} - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, fetchedPool) - Expect(err).Should(Succeed()) + Eventually(func(g Gomega) { + fetchedPool := &capsulev1beta2.ResourcePool{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, fetchedPool) + g.Expect(err).Should(Succeed()) - fetchedClaim := &capsulev1beta2.ResourcePoolClaim{} - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, fetchedClaim) - Expect(err).Should(Succeed()) + fetchedClaim := &capsulev1beta2.ResourcePoolClaim{} + err = k8sClient.Get(context.TODO(), client.ObjectKey{ + Name: claim.Name, + Namespace: claim.Namespace, + }, fetchedClaim) + g.Expect(err).Should(Succeed()) - isBoundToPool(fetchedPool, fetchedClaim) + isBoundToPool(fetchedPool, fetchedClaim) - Expect(fetchedClaim.Status.Pool.Name.String()).To(Equal(fetchedPool.Name)) - Expect(fetchedClaim.Status.Pool.UID).To(Equal(fetchedPool.GetUID())) + g.Expect(fetchedClaim.Status.Pool.Name.String()).To(Equal(fetchedPool.Name)) + g.Expect(fetchedClaim.Status.Pool.UID).To(Equal(fetchedPool.GetUID())) - bound := fetchedClaim.Status.Conditions.GetConditionByType(meta.BoundCondition) + bound := fetchedClaim.Status.Conditions.GetConditionByType(meta.BoundCondition) + g.Expect(bound).NotTo(BeNil(), "Bound condition should be present") - Expect(bound.Type).To(Equal(meta.BoundCondition)) - Expect(bound.Status).To(Equal(metav1.ConditionFalse)) - Expect(bound.Reason).To(Equal(meta.UnusedReason)) + g.Expect(bound.Type).To(Equal(meta.BoundCondition)) + g.Expect(bound.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(bound.Reason).To(Equal(meta.UnusedReason)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } func isSuccessfullyBoundAndUsedToPool(pool *capsulev1beta2.ResourcePool, claim *capsulev1beta2.ResourcePoolClaim) { - fetchedPool := &capsulev1beta2.ResourcePool{} - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, fetchedPool) - Expect(err).Should(Succeed()) + Eventually(func(g Gomega) { + fetchedPool := &capsulev1beta2.ResourcePool{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, fetchedPool) + g.Expect(err).Should(Succeed()) - fetchedClaim := &capsulev1beta2.ResourcePoolClaim{} - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, fetchedClaim) - Expect(err).Should(Succeed()) + fetchedClaim := &capsulev1beta2.ResourcePoolClaim{} + err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, fetchedClaim) + g.Expect(err).Should(Succeed()) - isBoundToPool(fetchedPool, fetchedClaim) + g.Expect(assertBoundToPool(fetchedPool, fetchedClaim)).Should(Succeed()) - Expect(fetchedClaim.Status.Pool.Name.String()).To(Equal(fetchedPool.Name)) - Expect(fetchedClaim.Status.Pool.UID).To(Equal(fetchedPool.GetUID())) + g.Expect(fetchedClaim.Status.Pool.Name.String()).To(Equal(fetchedPool.Name)) + g.Expect(fetchedClaim.Status.Pool.UID).To(Equal(fetchedPool.GetUID())) - bound := fetchedClaim.Status.Conditions.GetConditionByType(meta.BoundCondition) + bound := fetchedClaim.Status.Conditions.GetConditionByType(meta.BoundCondition) + g.Expect(bound).NotTo(BeNil(), "Bound condition should be present") - Expect(bound.Type).To(Equal(meta.BoundCondition)) - Expect(bound.Status).To(Equal(metav1.ConditionTrue)) - Expect(bound.Reason).To(Equal(meta.InUseReason)) + g.Expect(bound.Type).To(Equal(meta.BoundCondition)) + g.Expect(bound.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(bound.Reason).To(Equal(meta.InUseReason)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } -func isBoundToPool(pool *capsulev1beta2.ResourcePool, claim *capsulev1beta2.ResourcePoolClaim) bool { - fetchedPool := &capsulev1beta2.ResourcePool{} - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, fetchedPool) - Expect(err).Should(Succeed()) - - fetchedClaim := &capsulev1beta2.ResourcePoolClaim{} - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, fetchedClaim) - Expect(err).Should(Succeed()) - - status := fetchedPool.GetClaimFromStatus(fetchedClaim) +func assertBoundToPool(pool *capsulev1beta2.ResourcePool, claim *capsulev1beta2.ResourcePoolClaim) error { + status := pool.GetClaimFromStatus(claim) if status == nil { - return false + return fmt.Errorf("claim %s/%s not found in pool %s status", claim.Namespace, claim.Name, pool.Name) } for name, cl := range status.Claims { - Expect(cl).To(Equal(fetchedClaim.Spec.ResourceClaims[name])) + expected, ok := claim.Spec.ResourceClaims[name] + if !ok { + return fmt.Errorf("pool status contains unexpected claim key %q", name) + } + + if !reflect.DeepEqual(cl, expected) { + return fmt.Errorf("claim %q differs from spec: got %#v, want %#v", name, cl, expected) + } } - return true + return nil +} + +func isNotBoundToPool(pool *capsulev1beta2.ResourcePool, claim *capsulev1beta2.ResourcePoolClaim) bool { + status := pool.GetClaimFromStatus(claim) + return status == nil +} + +func isBoundToPool(pool *capsulev1beta2.ResourcePool, claim *capsulev1beta2.ResourcePoolClaim) error { + status := pool.GetClaimFromStatus(claim) + if status == nil { + return fmt.Errorf("claim %s/%s not found in pool %s status", claim.Namespace, claim.Name, pool.Name) + } + + for name, cl := range status.Claims { + expected, ok := claim.Spec.ResourceClaims[name] + if !ok { + return fmt.Errorf("pool status contains unexpected claim key %q", name) + } + + if !reflect.DeepEqual(cl, expected) { + return fmt.Errorf("claim %q differs from spec: got %#v, want %#v", name, cl, expected) + } + } + + return nil +} + +func ExpectResourceQuotaEventually(namespace, name string, expected corev1.ResourceList, poolName string, poolUID types.UID) { + quotaLabel, err := utils.GetTypeLabel(&capsulev1beta2.ResourcePool{}) + Expect(err).To(Succeed()) + + Eventually(func(g Gomega) { + rq := &corev1.ResourceQuota{} + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKey{Name: name, Namespace: namespace}, rq)).To(Succeed()) + + g.Expect(rq.Labels).To(HaveKeyWithValue(quotaLabel, poolName), "Expected %s to be set to %s", quotaLabel, poolName) + + ok, msg := DeepCompare(expected, rq.Spec.Hard) + g.Expect(ok).To(BeTrue(), "Mismatch for resourcequota %s/%s: %s", namespace, name, msg) + + g.Expect(rq.OwnerReferences).To(ContainElement(SatisfyAll( + WithTransform(func(ref metav1.OwnerReference) string { return ref.Kind }, Equal("ResourcePool")), + WithTransform(func(ref metav1.OwnerReference) types.UID { return ref.UID }, Equal(poolUID)), + )), "Expected ResourcePool to be owner of ResourceQuota in namespace %s", namespace) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func ExpectResourceQuotaDeletedEventually(namespace, name string) { + Eventually(func() bool { + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: name, Namespace: namespace}, &corev1.ResourceQuota{}) + return apierrors.IsNotFound(err) + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue(), "Expected ResourceQuota %s/%s to be deleted", namespace, name) +} + +func ExpectResourcePoolDeletedEventually(name string) { + Eventually(func() bool { + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: name}, &capsulev1beta2.ResourcePool{}) + return apierrors.IsNotFound(err) + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue(), "Expected ResourcePool %s to be deleted", name) +} + +func ExpectResourcePoolFinalizerEventually(name string, present bool) { + Eventually(func(g Gomega) { + current := &capsulev1beta2.ResourcePool{} + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKey{Name: name}, current)).To(Succeed()) + g.Expect(controllerutil.ContainsFinalizer(current, meta.ControllerFinalizer)).To(Equal(present)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } func containsAll[T comparable](haystack []T, needles []T) bool { @@ -2097,3 +2207,114 @@ func extractResourcePoolMessage(msg string) []string { } return out } + +func assertClaimExhausted(pool *capsulev1beta2.ResourcePool, claim *capsulev1beta2.ResourcePoolClaim, reason string, expected []string) { + Eventually(func(g Gomega) { + fetchedPool := &capsulev1beta2.ResourcePool{} + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, fetchedPool)).To(Succeed()) + + fetchedClaim := &capsulev1beta2.ResourcePoolClaim{} + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKey{ + Name: claim.Name, + Namespace: claim.Namespace, + }, fetchedClaim)).To(Succeed()) + + g.Expect(fetchedPool.GetClaimFromStatus(fetchedClaim)).To(BeNil()) + + exhausted := fetchedClaim.Status.Conditions.GetConditionByType(meta.ExhaustedCondition) + g.Expect(exhausted).NotTo(BeNil(), "Exhausted condition should be present") + + g.Expect(containsAll( + extractResourcePoolMessage(exhausted.Message), + expected, + )).To(BeTrue(), "Actual message: %s", exhausted.Message) + + g.Expect(exhausted.Reason).To(Equal(reason)) + g.Expect(exhausted.Status).To(Equal(metav1.ConditionTrue)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func UpdatePoolEventually(name string, hard corev1.ResourceList) { + Eventually(func() error { + current := &capsulev1beta2.ResourcePool{} + if err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: name}, current); err != nil { + return err + } + + current.Spec.Quota.Hard = hard + + return k8sClient.Update(context.TODO(), current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func UpdatePoolShouldFail(name string, hard corev1.ResourceList) { + current := &capsulev1beta2.ResourcePool{} + Expect(k8sClient.Get(context.TODO(), client.ObjectKey{Name: name}, current)).To(Succeed()) + + current.Spec.Quota.Hard = hard + + Expect(k8sClient.Update(context.TODO(), current)).ShouldNot(Succeed()) +} + +func ExpectPoolAllocation(name string, expected capsulev1beta2.ResourcePoolQuotaStatus) { + Eventually(func(g Gomega) { + current := &capsulev1beta2.ResourcePool{} + g.Expect(k8sClient.Get(context.TODO(), client.ObjectKey{Name: name}, current)).To(Succeed()) + + ok, msg := DeepCompare(expected, current.Status.Allocation) + g.Expect(ok).To(BeTrue(), "Mismatch for resource allocation: %s", msg) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func ExpectNamespaceInResourcePoolEventually(poolName string, namespace string) { + Eventually(func(g Gomega) { + current := &capsulev1beta2.ResourcePool{} + + g.Expect(k8sClient.Get( + context.TODO(), + client.ObjectKey{Name: poolName}, + current, + )).To(Succeed()) + + g.Expect(current.Status.Namespaces).To(ContainElement(namespace)) + g.Expect(current.Status.NamespaceSize).To(BeNumerically(">", 0)) + + condition := current.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil(), "ResourcePool Ready condition should exist") + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(condition.Reason).To(Equal(meta.SucceededReason)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func ExpectResourcePoolAllocationEventually( + poolName string, + expected capsulev1beta2.ResourcePoolQuotaStatus, +) { + Eventually(func(g Gomega) { + current := &capsulev1beta2.ResourcePool{} + + g.Expect(k8sClient.Get( + context.TODO(), + client.ObjectKey{Name: poolName}, + current, + )).To(Succeed()) + + ok, msg := DeepCompare(expected, current.Status.Allocation) + g.Expect(ok).To(BeTrue(), "Mismatch for expected status allocation: %s", msg) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func ExpectResourcePoolNamespacesEventually(poolName string, expected []string) { + Eventually(func(g Gomega) { + current := &capsulev1beta2.ResourcePool{} + + g.Expect(k8sClient.Get( + context.TODO(), + client.ObjectKey{Name: poolName}, + current, + )).To(Succeed()) + + g.Expect(current.Status.Namespaces).To(ConsistOf(expected)) + g.Expect(current.Status.NamespaceSize).To(Equal(uint(len(expected)))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} diff --git a/e2e/pool_resourcepoolclaim_test.go b/e2e/pool_resourcepoolclaim_test.go new file mode 100644 index 00000000..0497c631 --- /dev/null +++ b/e2e/pool_resourcepoolclaim_test.go @@ -0,0 +1,877 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +var _ = Describe("ResourcePoolClaim Tests", Ordered, Label("resourcepool", "claim"), func() { + _ = &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-test-claims-1", + Labels: map[string]string{ + "env": "e2e", + }, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{ + { + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "wind-user", + Kind: "User", + }, + }, + }, + }, + }, + } + + JustAfterEach(func() { + Eventually(func() error { + poolList := &capsulev1beta2.TenantList{} + labelSelector := client.MatchingLabels{"e2e-resourcepoolclaims": "test"} + if err := k8sClient.List(context.TODO(), poolList, labelSelector); err != nil { + return err + } + + for _, pool := range poolList.Items { + EventuallyDeletion(&pool) + } + + return nil + }, "30s", "5s").Should(Succeed()) + + Eventually(func() error { + poolList := &capsulev1beta2.ResourcePoolList{} + labelSelector := client.MatchingLabels{"e2e-resourcepoolclaims": "test"} + if err := k8sClient.List(context.TODO(), poolList, labelSelector); err != nil { + return err + } + + for _, pool := range poolList.Items { + EventuallyDeletion(&pool) + } + + return nil + }, "30s", "5s").Should(Succeed()) + + Eventually(func() error { + poolList := &corev1.NamespaceList{} + labelSelector := client.MatchingLabels{"e2e-resourcepoolclaims": "test"} + if err := k8sClient.List(context.TODO(), poolList, labelSelector); err != nil { + return err + } + + for _, pool := range poolList.Items { + EventuallyDeletion(&pool) + } + + return nil + }, "30s", "5s").Should(Succeed()) + + }) + + It("Claim to Pool Assignment", func() { + pool1 := &capsulev1beta2.ResourcePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding-claims", + Labels: map[string]string{ + "e2e-resourcepoolclaims": "test", + }, + }, + Spec: capsulev1beta2.ResourcePoolSpec{ + Selectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "e2e.capsule.dev/test-suite": "claims-bindings", + }, + }, + }, + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "e2e.capsule.dev/test-suite": "claims-bindings-2", + }, + }, + }, + }, + Quota: corev1.ResourceQuotaSpec{ + Hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("2"), + corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("2"), + corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), + }, + }, + }, + } + + claim1 := &capsulev1beta2.ResourcePoolClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "assign-pool-claim-1", + Namespace: "ns-1-pool-assign", + }, + Spec: capsulev1beta2.ResourcePoolClaimSpec{ + Pool: "test-binding-claims", + ResourceClaims: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("0"), + corev1.ResourceLimitsMemory: resource.MustParse("0"), + corev1.ResourceRequestsCPU: resource.MustParse("0"), + corev1.ResourceRequestsMemory: resource.MustParse("0"), + }, + }, + } + + claim2 := &capsulev1beta2.ResourcePoolClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "assign-pool-claim-2", + Namespace: "ns-2-pool-assign", + }, + Spec: capsulev1beta2.ResourcePoolClaimSpec{ + Pool: "test-binding-claims", + ResourceClaims: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("0"), + corev1.ResourceLimitsMemory: resource.MustParse("0"), + corev1.ResourceRequestsCPU: resource.MustParse("0"), + corev1.ResourceRequestsMemory: resource.MustParse("0"), + }, + }, + } + + By("Create the ResourcePool", func() { + EventuallyCreation(func() error { + pool1.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), pool1) + }).Should(Succeed(), "Failed to create ResourcePool %s", pool1) + }) + + By("Get Applied revision", func() { + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool1.Name}, pool1) + Expect(err).Should(Succeed()) + }) + + By("Create Namespaces, which are selected by the pool", func() { + ns1 := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ns-1-pool-assign", + Labels: map[string]string{ + "e2e-resourcepoolclaims": "test", + "e2e.capsule.dev/test-suite": "claims-bindings", + }, + }, + } + + err := k8sClient.Create(context.TODO(), ns1) + Expect(err).Should(Succeed()) + + ns2 := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ns-2-pool-assign", + Labels: map[string]string{ + "e2e-resourcepoolclaims": "test", + "e2e.capsule.dev/test-suite": "claims-bindings-2", + }, + }, + } + + err = k8sClient.Create(context.TODO(), ns2) + Expect(err).Should(Succeed()) + + ns3 := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ns-3-pool-assign", + Labels: map[string]string{ + "e2e-resourcepoolclaims": "test", + "e2e.capsule.dev/test-suite": "something-else", + }, + }, + } + + err = k8sClient.Create(context.TODO(), ns3) + Expect(err).Should(Succeed()) + }) + + By("Verify Namespaces are shown as allowed targets", func() { + expectedNamespaces := []string{"ns-1-pool-assign", "ns-2-pool-assign"} + + Eventually(func(g Gomega) { + stat := &capsulev1beta2.ResourcePool{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool1.Name}, stat) + g.Expect(err).Should(Succeed()) + + g.Expect(stat.Status.Namespaces).To(Equal(expectedNamespaces)) + g.Expect(stat.Status.NamespaceSize).To(Equal(uint(2))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + By("Create a first claim and verify binding", func() { + err := k8sClient.Create(context.TODO(), claim1) + Expect(err).Should(Succeed(), "Failed to create Claim %s", claim1) + + isSuccessfullyBoundAndUnsedToPool(pool1, claim1) + }) + + By("Create a second claim and verify binding", func() { + err := k8sClient.Create(context.TODO(), claim2) + Expect(err).Should(Succeed(), "Failed to create Claim %s", claim2) + + isSuccessfullyBoundAndUnsedToPool(pool1, claim2) + }) + + By("Create a third claim and verify error", func() { + claim := &capsulev1beta2.ResourcePoolClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "assign-pool-claim-3", + Namespace: "ns-3-pool-assign", + }, + Spec: capsulev1beta2.ResourcePoolClaimSpec{ + Pool: "test-binding-claims", + ResourceClaims: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("0"), + corev1.ResourceLimitsMemory: resource.MustParse("0"), + corev1.ResourceRequestsCPU: resource.MustParse("0"), + corev1.ResourceRequestsMemory: resource.MustParse("0"), + }, + }, + } + + err := k8sClient.Create(context.TODO(), claim) + Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) + + Eventually(func(g Gomega) { + stat := &capsulev1beta2.ResourcePoolClaim{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + g.Expect(err).Should(Succeed()) + + expectedPool := meta.LocalRFC1123ObjectReferenceWithUID{} + g.Expect(stat.Status.Pool).To(Equal(expectedPool), "expected pool name to be empty") + + g.Expect(len(stat.Status.Conditions)).To(Equal(1), "expected single condition") + g.Expect(len(stat.OwnerReferences)).To(Equal(0), "expected no ownerreferences") + assigned := stat.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(assigned.Status).To(Equal(metav1.ConditionFalse), "failed to verify condition status") + g.Expect(assigned.Type).To(Equal(meta.ReadyCondition), "failed to verify condition type") + g.Expect(assigned.Reason).To(Equal(meta.FailedReason), "failed to verify condition reason") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + }) + + It("Admission (Validation) - Patch Guard", Label("skip-on-openshift"), func() { + pool := &capsulev1beta2.ResourcePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-admission-claims", + Labels: map[string]string{ + "e2e-resourcepoolclaims": "test", + }, + }, + Spec: capsulev1beta2.ResourcePoolSpec{ + Config: capsulev1beta2.ResourcePoolSpecConfiguration{ + DeleteBoundResources: ptr.To(false), + }, + Selectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "e2e.capsule.dev/test-suite": "admission-guards", + }, + }, + }, + }, + Quota: corev1.ResourceQuotaSpec{ + Hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("2"), + corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("2"), + corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), + }, + }, + }, + } + + claim := &capsulev1beta2.ResourcePoolClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "admission-pool-claim-1", + Namespace: "ns-1-pool-admission", + }, + Spec: capsulev1beta2.ResourcePoolClaimSpec{ + Pool: pool.GetName(), + ResourceClaims: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("1"), + corev1.ResourceLimitsMemory: resource.MustParse("1Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("1"), + corev1.ResourceRequestsMemory: resource.MustParse("1Gi"), + }, + }, + } + + By("Create the Claim", func() { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: claim.Namespace, + Labels: map[string]string{ + "e2e-resourcepoolclaims": "test", + "e2e.capsule.dev/test-suite": "admission-guards", + }, + }, + } + + EventuallyCreation(func() error { + ns.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), ns) + }).Should(Succeed(), "Failed to create %s", ns) + + EventuallyCreation(func() error { + claim.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), claim) + }).Should(Succeed(), "Failed to create %s", claim) + + }) + + By("Create the ResourcePool", func() { + EventuallyCreation(func() error { + pool.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), pool) + }).Should(Succeed(), "Failed to create %s", pool) + }) + + By("Get Applied revision", func() { + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) + Expect(err).Should(Succeed()) + }) + + By("Bind a claim", func() { + Eventually(func(g Gomega) { + stat := &capsulev1beta2.ResourcePoolClaim{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + g.Expect(err).Should(Succeed()) + + expectedPool := meta.LocalRFC1123ObjectReferenceWithUID{ + Name: meta.RFC1123Name(pool.Name), + UID: pool.GetUID(), + } + + g.Expect(stat.Status.Pool).To(Equal(expectedPool), "expected pool name to match") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + isBoundAndUnusedCondition(claim) + }) + + By("Create a pod with resource requests/limits", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "claim-pod", + Namespace: claim.Namespace, + Labels: map[string]string{ + "e2e": "claim-pod", + }, + }, + Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + // optional: helps schedule quickly, avoid restarts + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Name: "pause", + Image: "registry.k8s.io/pause:3.9", + SecurityContext: restrictedContainerSecurityContext(), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("20m"), + corev1.ResourceMemory: resource.MustParse("32Mi"), + }, + }, + }, + }, + }, + } + + Expect(k8sClient.Create(context.TODO(), pod)).To(Succeed()) + }) + + By("Verify the claim is used and cant be deleted", func() { + Eventually(func() error { + stat := &capsulev1beta2.ResourcePoolClaim{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + Expect(err).Should(Succeed()) + + isBoundAndUsedCondition(stat) + + return k8sClient.Delete(context.TODO(), stat) + }, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed()) + }) + + By("Error on patching resources for claim (Increase)", func() { + Eventually(func() error { + stat := &capsulev1beta2.ResourcePoolClaim{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + Expect(err).Should(Succeed()) + + stat.Spec.ResourceClaims = corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("2"), + corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("2"), + corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), + } + + return k8sClient.Update(context.TODO(), stat) + }, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed()) + }) + + By("Error on patching resources for claim (Decrease)", func() { + Eventually(func() error { + stat := &capsulev1beta2.ResourcePoolClaim{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + Expect(err).Should(Succeed()) + + stat.Spec.ResourceClaims = corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("0"), + corev1.ResourceLimitsMemory: resource.MustParse("0Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("0"), + corev1.ResourceRequestsMemory: resource.MustParse("0Gi"), + } + + return k8sClient.Update(context.TODO(), stat) + }, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed()) + }) + + By("Error on patching pool name", func() { + Eventually(func() error { + stat := &capsulev1beta2.ResourcePoolClaim{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + Expect(err).Should(Succeed()) + + stat.Spec.Pool = "some-random-pool" + + return k8sClient.Update(context.TODO(), stat) + }, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed()) + }) + + By("Make the claim unused", func() { + key := client.ObjectKey{Name: "claim-pod", Namespace: claim.Namespace} + + pod := &corev1.Pod{} + err := k8sClient.Get(context.TODO(), key, pod) + Expect(err).To(Succeed(), "pod must exist before deleting") + + Expect(k8sClient.Delete(context.TODO(), pod, &client.DeleteOptions{ + GracePeriodSeconds: ptr.To(int64(0)), + })).To(Succeed()) + + Eventually(func() bool { + p := &corev1.Pod{} + err := k8sClient.Get( + context.TODO(), + key, + p, + ) + return apierrors.IsNotFound(err) + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + + }) + + By("Bind a claim", func() { + Eventually(func(g Gomega) { + stat := &capsulev1beta2.ResourcePoolClaim{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + g.Expect(err).Should(Succeed()) + + expectedPool := meta.LocalRFC1123ObjectReferenceWithUID{ + Name: meta.RFC1123Name(pool.Name), + UID: pool.GetUID(), + } + + isBoundAndUnusedCondition(stat) + g.Expect(stat.Status.Pool).To(Equal(expectedPool), "expected pool name to match") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + By("Allow on patching resources for claim (Increase)", func() { + Eventually(func() error { + stat := &capsulev1beta2.ResourcePoolClaim{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + Expect(err).Should(Succeed()) + + stat.Spec.ResourceClaims = corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("2"), + corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("2"), + corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), + } + + return k8sClient.Update(context.TODO(), stat) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + By("Allow on patching resources for claim (Decrease)", func() { + Eventually(func() error { + stat := &capsulev1beta2.ResourcePoolClaim{} + if err := k8sClient.Get( + context.TODO(), + client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, + stat, + ); err != nil { + return err + } + + stat.Spec.ResourceClaims = corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("0"), + corev1.ResourceLimitsMemory: resource.MustParse("0Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("0"), + corev1.ResourceRequestsMemory: resource.MustParse("0Gi"), + } + + return k8sClient.Update(context.TODO(), stat) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + By("Allow on patching pool name", func() { + Eventually(func() error { + stat := &capsulev1beta2.ResourcePoolClaim{} + + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + Expect(err).Should(Succeed()) + + stat.Spec.Pool = "some-random-pool" + + return k8sClient.Update(context.TODO(), stat) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + By("Delete Pool", func() { + EventuallyDeletion(pool) + }) + + By("Verify claim is no longer bound", func() { + isUnassignedCondition(claim) + }) + + By("Allow patching resources for claim (Increase)", func() { + Eventually(func() error { + stat := &capsulev1beta2.ResourcePoolClaim{} + + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + Expect(err).Should(Succeed()) + + stat.Spec.ResourceClaims = corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("2"), + corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("2"), + corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), + } + + return k8sClient.Update(context.TODO(), stat) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + By("Allow patching resources for claim (Decrease)", func() { + Eventually(func() error { + stat := &capsulev1beta2.ResourcePoolClaim{} + + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + Expect(err).Should(Succeed()) + + stat.Spec.ResourceClaims = corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("0"), + corev1.ResourceLimitsMemory: resource.MustParse("0Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("0"), + corev1.ResourceRequestsMemory: resource.MustParse("0Gi"), + } + + return k8sClient.Update(context.TODO(), stat) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + By("Allow patching pool name", func() { + Eventually(func() error { + stat := &capsulev1beta2.ResourcePoolClaim{} + + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + Expect(err).Should(Succeed()) + + stat.Spec.Pool = "some-random-pool" + + return k8sClient.Update(context.TODO(), stat) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + }) + + It("Admission (Mutation) - Auto Pool Assign", Label("skip-on-openshift"), func() { + pool1 := &capsulev1beta2.ResourcePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-auto-assign-1", + Labels: map[string]string{ + "e2e-resourcepoolclaims": "test", + }, + }, + Spec: capsulev1beta2.ResourcePoolSpec{ + Config: capsulev1beta2.ResourcePoolSpecConfiguration{ + DeleteBoundResources: ptr.To(false), + }, + Selectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "e2e.capsule.dev/test-suite": "admission-auto-assign", + }, + }, + }, + }, + Quota: corev1.ResourceQuotaSpec{ + Hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("2"), + corev1.ResourceRequestsCPU: resource.MustParse("2"), + }, + }, + }, + } + + pool2 := &capsulev1beta2.ResourcePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-auto-assign-2", + Labels: map[string]string{ + "e2e-resourcepoolclaims": "test", + }, + }, + Spec: capsulev1beta2.ResourcePoolSpec{ + Config: capsulev1beta2.ResourcePoolSpecConfiguration{ + DeleteBoundResources: ptr.To(false), + }, + Selectors: []selectors.NamespaceSelector{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "e2e.capsule.dev/test-suite": "admission-auto-assign", + }, + }, + }, + }, + Quota: corev1.ResourceQuotaSpec{ + Hard: corev1.ResourceList{ + corev1.ResourceLimitsMemory: resource.MustParse("2"), + corev1.ResourceRequestsMemory: resource.MustParse("2"), + }, + }, + }, + } + + By("Create the ResourcePools", func() { + EventuallyCreation(func() error { + pool1.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), pool1) + }).Should(Succeed(), "Failed to create %s", pool1) + + EventuallyCreation(func() error { + pool2.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), pool2) + }).Should(Succeed(), "Failed to create %s", pool2) + }) + + By("Auto Assign Claim (CPU)", func() { + claim := &capsulev1beta2.ResourcePoolClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "auto-assign-1", + Namespace: "ns-1-pool-assign", + }, + Spec: capsulev1beta2.ResourcePoolClaimSpec{ + ResourceClaims: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("1"), + corev1.ResourceRequestsCPU: resource.MustParse("1"), + }, + }, + } + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: claim.Namespace, + Labels: map[string]string{ + "e2e-resourcepoolclaims": "test", + "e2e.capsule.dev/test-suite": "admission-auto-assign", + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(context.TODO(), ns) + }).Should(Succeed()) + + ExpectNamespaceInResourcePoolEventually(pool1.Name, ns.Name) + + EventuallyCreation(func() error { + claim.ResourceVersion = "" + return k8sClient.Create(context.TODO(), claim) + }).Should(Succeed(), "Failed to create Claim %s/%s", claim.Namespace, claim.Name) + + Eventually(func(g Gomega) { + stat := &capsulev1beta2.ResourcePoolClaim{} + + g.Expect(k8sClient.Get( + context.TODO(), + client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, + stat, + )).To(Succeed()) + + g.Expect(stat.Spec.Pool).To(Equal(pool1.Name), "expected pool name to match") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + By("Auto Assign Claim (Memory)", func() { + claim := &capsulev1beta2.ResourcePoolClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "auto-assign-1", + Namespace: "ns-2-pool-assign", + }, + Spec: capsulev1beta2.ResourcePoolClaimSpec{ + ResourceClaims: corev1.ResourceList{ + corev1.ResourceLimitsMemory: resource.MustParse("1"), + corev1.ResourceRequestsMemory: resource.MustParse("1"), + }, + }, + } + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: claim.Namespace, + Labels: map[string]string{ + "e2e-resourcepoolclaims": "test", + "e2e.capsule.dev/test-suite": "admission-auto-assign", + }, + }, + } + + err := k8sClient.Create(context.TODO(), ns) + Expect(err).Should(Succeed()) + + ExpectNamespaceInResourcePoolEventually(pool2.Name, ns.Name) + + err = k8sClient.Create(context.TODO(), claim) + Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) + + Eventually(func(g Gomega) { + stat := &capsulev1beta2.ResourcePoolClaim{} + err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + g.Expect(err).Should(Succeed()) + + g.Expect(stat.Spec.Pool).To(Equal(pool2.Name), "expected pool name to match") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + By("No Default available (Storage)", func() { + claim := &capsulev1beta2.ResourcePoolClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "auto-assign-3", + Namespace: "ns-3-pool-assign", + }, + Spec: capsulev1beta2.ResourcePoolClaimSpec{ + ResourceClaims: corev1.ResourceList{ + corev1.ResourceRequestsStorage: resource.MustParse("1"), + }, + }, + } + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: claim.Namespace, + Labels: map[string]string{ + "e2e-resourcepoolclaims": "test", + "e2e.capsule.dev/test-suite": "admission-auto-assign", + }, + }, + } + + EventuallyCreation(func() error { + ns.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), ns) + }).Should(Succeed(), "Failed to create %s", ns) + + ExpectNamespaceInResourcePoolEventually(pool1.Name, ns.Name) + + EventuallyCreation(func() error { + claim.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), claim) + }).Should(Succeed(), "Failed to create %s", claim) + + Eventually(func(g Gomega) { + stat := &capsulev1beta2.ResourcePoolClaim{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, stat) + g.Expect(err).Should(Succeed()) + + g.Expect(stat.Spec.Pool).To(Equal(""), "expected pool name to match") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + }) +}) + +func isUnassignedCondition(claim *capsulev1beta2.ResourcePoolClaim) { + Eventually(func(g Gomega) { + cl := &capsulev1beta2.ResourcePoolClaim{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, cl) + g.Expect(err).Should(Succeed()) + + assigned := cl.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(assigned).NotTo(BeNil(), "Ready condition should be present") + + g.Expect(assigned.Status).To(Equal(metav1.ConditionFalse), "failed to verify condition status") + g.Expect(assigned.Type).To(Equal(meta.ReadyCondition), "failed to verify condition type") + g.Expect(assigned.Reason).To(Equal(meta.FailedReason), "failed to verify condition reason") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func isBoundAndUnusedCondition(claim *capsulev1beta2.ResourcePoolClaim) { + Eventually(func(g Gomega) { + cl := &capsulev1beta2.ResourcePoolClaim{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, cl) + g.Expect(err).Should(Succeed()) + + bound := cl.Status.Conditions.GetConditionByType(meta.BoundCondition) + g.Expect(bound).NotTo(BeNil(), "Bound condition should be present") + + g.Expect(bound.Type).To(Equal(meta.BoundCondition), "failed to verify condition type") + g.Expect(bound.Reason).To(Equal(meta.UnusedReason), "failed to verify condition reason") + g.Expect(bound.Status).To(Equal(metav1.ConditionFalse), "failed to verify condition status") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func isBoundAndUsedCondition(claim *capsulev1beta2.ResourcePoolClaim) { + Eventually(func(g Gomega) { + cl := &capsulev1beta2.ResourcePoolClaim{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, cl) + g.Expect(err).Should(Succeed()) + + bound := cl.Status.Conditions.GetConditionByType(meta.BoundCondition) + g.Expect(bound).NotTo(BeNil(), "Bound condition should be present") + + g.Expect(bound.Status).To(Equal(metav1.ConditionTrue), "failed to verify condition status") + g.Expect(bound.Type).To(Equal(meta.BoundCondition), "failed to verify condition type") + g.Expect(bound.Reason).To(Equal(meta.InUseReason), "failed to verify condition reason") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} diff --git a/e2e/preventing_pv_cross_tenant_mount_test.go b/e2e/preventing_pv_cross_tenant_mount_test.go deleted file mode 100644 index 03229983..00000000 --- a/e2e/preventing_pv_cross_tenant_mount_test.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2020-2023 Project Capsule Authors. -// SPDX-License-Identifier: Apache-2.0 - -package e2e - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" -) - -var _ = Describe("preventing PersistentVolume cross-tenant mount", Label("tenant", "storage"), func() { - tnt1 := &capsulev1beta2.Tenant{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pv-one", - }, - Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ - { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "jessica", - Kind: "User", - }, - }, - }, - }, - }, - } - - tnt2 := &capsulev1beta2.Tenant{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pv-two", - }, - Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ - { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "leto", - Kind: "User", - }, - }, - }, - }, - }, - } - - JustBeforeEach(func() { - for _, tnt := range []*capsulev1beta2.Tenant{tnt1, tnt2} { - EventuallyCreation(func() error { - tnt.ResourceVersion = "" - - return k8sClient.Create(context.TODO(), tnt) - }).Should(Succeed()) - } - }) - - JustAfterEach(func() { - for _, tnt := range []*capsulev1beta2.Tenant{tnt1, tnt2} { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) - } - }) - - It("should add labels to PersistentVolume and prevent cross-Tenant mount", Label("skip-on-openshift"), func() { - ns := NewNamespace("") - NamespaceCreation(ns, tnt1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt1, defaultTimeoutInterval).Should(ContainElement(ns.Name)) - - pvc := corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "arrakis", - Namespace: ns.Name, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - StorageClassName: ptr.To("standard"), - }, - } - - EventuallyCreation(func() error { - return k8sClient.Create(context.Background(), &pvc) - }).Should(Succeed()) - - pod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "arrakis-pod", - Namespace: ns.Name, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "container", - Image: "gcr.io/google_containers/pause-amd64:3.0", - ImagePullPolicy: corev1.PullAlways, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "data", - MountPath: "/tmp", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "data", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc.Name, - }, - }, - }, - }, - }, - } - - EventuallyCreation(func() error { - return k8sClient.Create(context.Background(), &pod) - }).Should(Succeed()) - - Eventually(func() int { - nsName := types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace} - - if err := k8sClient.Get(context.Background(), nsName, &pvc); err != nil { - return 0 - } - - return len(pvc.Spec.VolumeName) - }, defaultTimeoutInterval, defaultPollInterval).Should(BeNumerically(">", 0)) - - pv := corev1.PersistentVolume{} - defer func() { - _ = k8sClient.Delete(context.Background(), &pv) - }() - - Eventually(func() string { - if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: pvc.Spec.VolumeName}, &pv); err != nil { - return "not-found" - } - - if pv.GetLabels() == nil { - return "no-labels" - } - - return pv.GetLabels()["capsule.clastix.io/tenant"] - }, defaultTimeoutInterval, defaultPollInterval).Should(Equal(tnt1.Name)) - - Eventually(func() error { - nsName := types.NamespacedName{Name: pv.Name} - - if err := k8sClient.Get(context.Background(), nsName, &pv); err != nil { - return err - } - - pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRecycle - - return k8sClient.Update(context.Background(), &pv) - }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) - - Expect(k8sClient.Delete(context.Background(), &pod, &client.DeleteOptions{GracePeriodSeconds: ptr.To(int64(0))})).ToNot(HaveOccurred()) - - ns2 := NewNamespace("") - NamespaceCreation(ns2, tnt2.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt2, defaultTimeoutInterval).Should(ContainElement(ns2.Name)) - - Consistently(func() error { - pvc := corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "caladan", - Namespace: ns2.Name, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - StorageClassName: ptr.To("standard"), - VolumeName: pv.Name, - }, - } - - return k8sClient.Create(context.Background(), &pvc) - }, defaultTimeoutInterval, defaultPollInterval).Should(HaveOccurred()) - }) -}) diff --git a/e2e/replications_globaltenantresource_test.go b/e2e/replications_globaltenantresource_test.go new file mode 100644 index 00000000..292fbdc6 --- /dev/null +++ b/e2e/replications_globaltenantresource_test.go @@ -0,0 +1,1292 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + apimeta "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" + "github.com/projectcapsule/capsule/pkg/runtime/gvk" + "github.com/projectcapsule/capsule/pkg/template" +) + +const ( + managedByLabel = "projectcapsule.dev/managed-by" + createdByLabel = "projectcapsule.dev/created-by" + resourcesLabel = "resources" +) + +var _ = Describe("GlobalTenantResource", Ordered, Label("replications", "global", "globaltenantresource"), Ordered, func() { + var ( + ctx context.Context + originConfig *capsulev1beta2.CapsuleConfiguration + + tenantA *capsulev1beta2.Tenant + tenantB *capsulev1beta2.Tenant + + tenantAOwner rbac.UserSpec + tenantBOwner rbac.UserSpec + + tenantANamespaces []string + tenantBNamespaces []string + allNamespaces []string + ) + + BeforeEach(func() { + ctx = context.Background() + originConfig = &capsulev1beta2.CapsuleConfiguration{} + + tenantAOwner = rbac.UserSpec{Name: "e2e-gtr-tenant-a", Kind: rbac.OwnerKind("User")} + tenantBOwner = rbac.UserSpec{Name: "e2e-gtr-tenant-b", Kind: rbac.OwnerKind("User")} + + tenantANamespaces = []string{"e2e-gtr-tenant-a-one", "e2e-gtr-tenant-a-two", "e2e-gtr-tenant-a-system"} + tenantBNamespaces = []string{"e2e-gtr-tenant-b-one", "e2e-gtr-tenant-b-two"} + allNamespaces = append(append([]string{}, tenantANamespaces...), tenantBNamespaces...) + + tenantA = &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-gtr-tenant-a", + Labels: map[string]string{ + "energy": "solar", + "group": "alpha", + "env": "e2e", + }, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{{ + CoreOwnerSpec: rbac.CoreOwnerSpec{UserSpec: tenantAOwner}, + }}, + AdditionalRoleBindings: []rbac.AdditionalRoleBindingsSpec{{ + ClusterRoleName: "admin", + Subjects: []rbacv1.Subject{{ + Kind: "User", + Name: "bob", + }}, + }}, + }, + } + + tenantB = &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-gtr-tenant-b", + Labels: map[string]string{ + "energy": "lunar", + "group": "beta", + "env": "e2e", + }, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{{ + CoreOwnerSpec: rbac.CoreOwnerSpec{UserSpec: tenantBOwner}, + }}, + }, + } + + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: defaultConfigurationName}, originConfig)).To(Succeed()) + + EventuallyCreation(func() error { + tenantA.ResourceVersion = "" + return k8sClient.Create(ctx, tenantA) + }).Should(Succeed()) + TenantReady(tenantA, metav1.ConditionTrue, defaultTimeoutInterval) + + EventuallyCreation(func() error { + tenantB.ResourceVersion = "" + return k8sClient.Create(ctx, tenantB) + }).Should(Succeed()) + TenantReady(tenantB, metav1.ConditionTrue, defaultTimeoutInterval) + + for _, ns := range tenantANamespaces { + namespace := NewNamespace(ns, map[string]string{ + apimeta.TenantLabel: tenantA.GetName(), + }) + NamespaceCreation(namespace, tenantAOwner, defaultTimeoutInterval).Should(Succeed()) + } + + for _, ns := range tenantBNamespaces { + namespace := NewNamespace(ns, map[string]string{ + apimeta.TenantLabel: tenantB.GetName(), + }) + NamespaceCreation(namespace, tenantBOwner, defaultTimeoutInterval).Should(Succeed()) + } + }) + + AfterEach(func() { + for _, ns := range allNamespaces { + ForceDeleteNamespace(ctx, ns) + } + + EventuallyDeletion(tenantA) + EventuallyDeletion(tenantB) + + Eventually(func() error { + poolList := &capsulev1beta2.GlobalTenantResourceList{} + labelSelector := client.MatchingLabels{"e2e.capsule.dev/test-suite": "true"} + if err := k8sClient.List(context.TODO(), poolList, labelSelector); err != nil { + return err + } + + for _, pool := range poolList.Items { + if err := k8sClient.Delete(context.TODO(), &pool); err != nil { + return err + } + } + + return nil + }, "30s", "5s").Should(Succeed()) + + Eventually(func() error { + cfg := &capsulev1beta2.CapsuleConfiguration{} + if err := k8sClient.Get(ctx, client.ObjectKey{Name: originConfig.Name}, cfg); err != nil { + return err + } + cfg.Spec = originConfig.Spec + return k8sClient.Update(ctx, cfg) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + }) + + It("fails to replicate namespacedItems when the impersonated service account cannot read source resources", func() { + saName := "gtr-no-namespaceditem-read" + ensureServiceAccount("capsule-system", saName) + + sourceNs := "gtr-source-items" + sourceNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: sourceNs}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, sourceNamespace) }).Should(Succeed()) + defer ForceDeleteNamespace(ctx, sourceNs) + + sourceSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-secret", + Namespace: sourceNs, + Labels: map[string]string{ + "replicate": "true", + }, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{"token": "abc"}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, sourceSecret) }).Should(Succeed()) + + for _, ns := range tenantANamespaces { + bindServiceAccountToSecretWriter("capsule-system", saName, ns) + } + + gtr := &capsulev1beta2.GlobalTenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gtr-sa-no-namespaceditem-read", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "true", + }, + }, + Spec: capsulev1beta2.GlobalTenantResourceSpec{ + Scope: api.ResourceScopeNamespace, + ServiceAccount: &apimeta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: apimeta.RFC1123Name(saName), + Namespace: apimeta.RFC1123SubdomainName("capsule-system"), + }, + TenantSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + }, + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + ResyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + PruningOnDelete: ptr.To(true), + Resources: []capsulev1beta2.ResourceSpec{{ + NamespacedItems: []template.ResourceReference{{ + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Secret", + }, + Namespace: sourceNs, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "replicate": "true", + }, + }, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + expectGlobalTenantResourceFailed("gtr-sa-no-namespaceditem-read", "forbidden") + + for _, ns := range tenantANamespaces { + expectSecretAbsent(ns, "source-secret") + } + }) + + Context("scope handling", func() { + It("reconciles with scope Namespace", func() { + gtr := newRawConfigMapGlobalTenantResourceWithScope( + "gtr-scope-namespace", + api.ResourceScopeNamespace, + map[string]string{"mode": "namespace"}, + ) + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + renameFirstRawConfigMap(gtr, "gtr-scope-namespace") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + for _, ns := range tenantANamespaces { + expectConfigMapData(ns, "gtr-scope-namespace", map[string]string{"mode": "namespace"}) + } + for _, ns := range tenantBNamespaces { + expectConfigMapAbsent(ns, "gtr-scope-namespace") + } + }) + + It("accepts scope Tenant", func() { + gtr := newRawConfigMapGlobalTenantResourceWithScope( + "gtr-scope-tenant", + api.ResourceScopeTenant, + map[string]string{"mode": "tenant"}, + ) + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + renameFirstRawConfigMap(gtr, "gtr-scope-tenant") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + Eventually(func(g Gomega) { + current := &capsulev1beta2.GlobalTenantResource{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: gtr.Name}, current)).To(Succeed()) + rdy := current.Status.Conditions.GetConditionByType(apimeta.ReadyCondition) + g.Expect(rdy).ToNot(BeNil()) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + It("accepts scope None", func() { + gtr := newRawConfigMapGlobalTenantResourceWithScope( + "gtr-scope-none", + api.ResourceScopeNone, + map[string]string{"mode": "none"}, + ) + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + renameFirstRawConfigMap(gtr, "gtr-scope-none") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + Eventually(func(g Gomega) { + current := &capsulev1beta2.GlobalTenantResource{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: gtr.Name}, current)).To(Succeed()) + rdy := current.Status.Conditions.GetConditionByType(apimeta.ReadyCondition) + g.Expect(rdy).ToNot(BeNil()) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + }) + + Context("multiple GlobalTenantResources", func() { + It("fails when multiple GlobalTenantResources target the same preexisting object without adoption", func() { + for _, ns := range tenantANamespaces { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gtr-shared-preexisting", + Namespace: ns, + }, + Data: map[string]string{ + "existing": "true", + }, + } + EventuallyCreation(func() error { + cm.ResourceVersion = "" + return k8sClient.Create(ctx, cm) + }).Should(Succeed()) + } + + gtrA := newRawConfigMapGlobalTenantResource("gtr-collision-preexisting-a", map[string]string{ + "from": "a", + }) + gtrA.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + gtrA.Spec.Settings.Adopt = ptr.To(false) + renameFirstRawConfigMap(gtrA, "gtr-shared-preexisting") + + gtrB := newRawConfigMapGlobalTenantResource("gtr-collision-preexisting-b", map[string]string{ + "from": "b", + }) + gtrB.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + gtrB.Spec.Settings.Adopt = ptr.To(false) + renameFirstRawConfigMap(gtrB, "gtr-shared-preexisting") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtrA) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtrB) }).Should(Succeed()) + + expectGlobalTenantResourceFailed("gtr-collision-preexisting-a", "applying of") + expectGlobalTenantResourceFailed("gtr-collision-preexisting-b", "applying of") + + for _, ns := range tenantANamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: "gtr-shared-preexisting", + Namespace: ns, + }, cm)).To(Succeed()) + g.Expect(cm.Data).To(Equal(map[string]string{ + "existing": "true", + })) + g.Expect(cm.Labels).ToNot(HaveKey(managedByLabel)) + g.Expect(cm.Labels).ToNot(HaveKey(createdByLabel)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + + It("fails when a second GlobalTenantResource targets an object already managed by another GlobalTenantResource without adoption", func() { + gtrA := newRawConfigMapGlobalTenantResource("gtr-collision-managed-a", map[string]string{ + "owner": "first", + }) + gtrA.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + gtrA.Spec.Settings.Adopt = ptr.To(false) + renameFirstRawConfigMap(gtrA, "gtr-shared-managed") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtrA) }).Should(Succeed()) + expectGlobalTenantResourceReady("gtr-collision-managed-a") + + for _, ns := range tenantANamespaces { + expectConfigMapData(ns, "gtr-shared-managed", map[string]string{"owner": "first"}) + } + + gtrB := newRawConfigMapGlobalTenantResource("gtr-collision-managed-b", map[string]string{ + "owner": "second", + }) + gtrB.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + gtrB.Spec.Settings.Adopt = ptr.To(false) + renameFirstRawConfigMap(gtrB, "gtr-shared-managed") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtrB) }).Should(Succeed()) + expectGlobalTenantResourceFailed("gtr-collision-managed-b", "applying of") + + for _, ns := range tenantANamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: "gtr-shared-managed", + Namespace: ns, + }, cm)).To(Succeed()) + g.Expect(cm.Data).To(HaveKeyWithValue("owner", "first")) + g.Expect(cm.Labels).To(HaveKeyWithValue(managedByLabel, meta.ValueControllerReplications)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + }) + + Context("impersonation", func() { + It("fails to apply raw items when the impersonated service account cannot create target resources", func() { + saName := "gtr-no-create" + ensureServiceAccount("capsule-system", saName) + + gtr := newRawConfigMapGlobalTenantResource("gtr-sa-no-create", map[string]string{ + "mode": "blocked", + }) + gtr.Spec.ServiceAccount = &apimeta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: apimeta.RFC1123Name(saName), + Namespace: apimeta.RFC1123SubdomainName("capsule-system"), + } + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + renameFirstRawConfigMap(gtr, "gtr-blocked-create") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + expectGlobalTenantResourceFailed("gtr-sa-no-create", "applying of") + + for _, ns := range tenantANamespaces { + expectConfigMapAbsent(ns, "gtr-blocked-create") + } + }) + + It("fails to render generators when the impersonated service account cannot read context resources", func() { + saName := "gtr-no-context-read" + ensureServiceAccount("capsule-system", saName) + + sourceNs := "gtr-context-source" + sourceNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: sourceNs}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, sourceNamespace) }).Should(Succeed()) + defer ForceDeleteNamespace(ctx, sourceNs) + + sourceSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ctx-secret", + Namespace: sourceNs, + Labels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{".dockerconfigjson": "e30="}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, sourceSecret) }).Should(Succeed()) + + for _, ns := range tenantANamespaces { + bindServiceAccountToConfigMapWriter("capsule-system", saName, ns) + } + + gtr := &capsulev1beta2.GlobalTenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gtr-sa-no-context-read", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "true", + }, + }, + Spec: capsulev1beta2.GlobalTenantResourceSpec{ + Scope: api.ResourceScopeNamespace, + ServiceAccount: &apimeta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: apimeta.RFC1123Name(saName), + Namespace: apimeta.RFC1123SubdomainName("capsule-system"), + }, + TenantSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + }, + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + ResyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + PruningOnDelete: ptr.To(true), + Resources: []capsulev1beta2.ResourceSpec{{ + Context: &template.TemplateContext{ + Resources: []*template.TemplateResourceReference{{ + Index: "secrets", + ResourceReference: template.ResourceReference{ + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Secret", + }, + Namespace: sourceNs, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + }, + }}, + }, + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "error", + Template: `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gtr-blocked-context +data: + count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}' +`, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + expectGlobalTenantResourceFailed("gtr-sa-no-context-read", "forbidden") + + for _, ns := range tenantANamespaces { + expectConfigMapAbsent(ns, "gtr-blocked-context") + } + }) + + It("fails to replicate namespacedItems when the impersonated service account cannot read source resources", func() { + saName := "gtr-no-namespaceditem-read" + ensureServiceAccount("capsule-system", saName) + + sourceNs := "gtr-source-items" + sourceNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: sourceNs}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, sourceNamespace) }).Should(Succeed()) + defer ForceDeleteNamespace(ctx, sourceNs) + + sourceSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-secret", + Namespace: sourceNs, + Labels: map[string]string{ + "replicate": "true", + }, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{"token": "abc"}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, sourceSecret) }).Should(Succeed()) + + for _, ns := range tenantANamespaces { + bindServiceAccountToSecretWriter("capsule-system", saName, ns) + } + + gtr := &capsulev1beta2.GlobalTenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gtr-sa-no-namespaceditem-read", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "true", + }, + }, + Spec: capsulev1beta2.GlobalTenantResourceSpec{ + Scope: api.ResourceScopeNamespace, + ServiceAccount: &apimeta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: apimeta.RFC1123Name(saName), + Namespace: apimeta.RFC1123SubdomainName("capsule-system"), + }, + TenantSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + }, + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + ResyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + PruningOnDelete: ptr.To(true), + Resources: []capsulev1beta2.ResourceSpec{{ + NamespacedItems: []template.ResourceReference{{ + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Secret", + }, + Namespace: sourceNs, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "replicate": "true", + }, + }, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + expectGlobalTenantResourceFailed("gtr-sa-no-namespaceditem-read", "forbidden") + + for _, ns := range tenantANamespaces { + expectSecretAbsent(ns, "source-secret") + } + }) + + It("fails to prune replicated resources when the impersonated service account cannot delete them", func() { + saCreate := "gtr-creator-ok" + saNoDelete := "gtr-creator-no-delete" + + ensureServiceAccount("capsule-system", saCreate) + ensureServiceAccount("capsule-system", saNoDelete) + + for _, ns := range tenantANamespaces { + bindServiceAccountToConfigMapWriter("capsule-system", saCreate, ns) + bindServiceAccountToConfigMapWriter("capsule-system", saNoDelete, ns) + } + + gtr := newRawConfigMapGlobalTenantResource("gtr-sa-no-prune", map[string]string{ + "mode": "created", + }) + gtr.Spec.ServiceAccount = &apimeta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: apimeta.RFC1123Name(saCreate), + Namespace: apimeta.RFC1123SubdomainName("capsule-system"), + } + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + gtr.Spec.PruningOnDelete = ptr.To(true) + renameFirstRawConfigMap(gtr, "gtr-prune-protected") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + expectGlobalTenantResourceReady("gtr-sa-no-prune") + + for _, ns := range tenantANamespaces { + expectConfigMapData(ns, "gtr-prune-protected", map[string]string{"mode": "created"}) + } + + Eventually(func() error { + current := &capsulev1beta2.GlobalTenantResource{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: gtr.Name}, current); err != nil { + return err + } + current.Spec.ServiceAccount = &apimeta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: apimeta.RFC1123Name(saNoDelete), + Namespace: apimeta.RFC1123SubdomainName("capsule-system"), + } + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Eventually(func(g Gomega) { + current := &capsulev1beta2.GlobalTenantResource{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: gtr.Name}, current)).To(Succeed()) + g.Expect(current.Status.ServiceAccount).ToNot(BeNil()) + g.Expect(current.Status.ServiceAccount.Name).To(Equal(apimeta.RFC1123Name(saNoDelete))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Expect(k8sClient.Delete(ctx, gtr)).To(Succeed()) + + for _, ns := range tenantANamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "gtr-prune-protected", Namespace: ns}, cm)).To(Succeed()) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + }) + + Context("adoption", func() { + It("fails on preexisting objects when adoption is disabled", func() { + for _, ns := range tenantANamespaces { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gtr-adopt-me", + Namespace: ns, + }, + Data: map[string]string{ + "existing": "true", + }, + } + EventuallyCreation(func() error { + cm.ResourceVersion = "" + return k8sClient.Create(ctx, cm) + }).Should(Succeed()) + } + + gtr := newRawConfigMapGlobalTenantResource("gtr-adoption-disabled", map[string]string{ + "mode": "new", + }) + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + gtr.Spec.Settings.Adopt = ptr.To(false) + renameFirstRawConfigMap(gtr, "gtr-adopt-me") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + expectGlobalTenantResourceFailed("gtr-adoption-disabled", "applying of") + + for _, ns := range tenantANamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "gtr-adopt-me", Namespace: ns}, cm)).To(Succeed()) + g.Expect(cm.Data).To(Equal(map[string]string{"existing": "true"})) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + + It("adopts preexisting objects when adoption is enabled", func() { + for _, ns := range tenantANamespaces { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gtr-adopt-me-enabled", + Namespace: ns, + }, + Data: map[string]string{ + "existing": "true", + }, + } + EventuallyCreation(func() error { + cm.ResourceVersion = "" + return k8sClient.Create(ctx, cm) + }).Should(Succeed()) + } + + gtr := newRawConfigMapGlobalTenantResource("gtr-adoption-enabled", map[string]string{ + "mode": "adopted", + "foo": "bar", + }) + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + gtr.Spec.Settings.Adopt = ptr.To(true) + renameFirstRawConfigMap(gtr, "gtr-adopt-me-enabled") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + expectGlobalTenantResourceReady("gtr-adoption-enabled") + + for _, ns := range tenantANamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "gtr-adopt-me-enabled", Namespace: ns}, cm)).To(Succeed()) + g.Expect(cm.Data).To(HaveKeyWithValue("mode", "adopted")) + g.Expect(cm.Data).To(HaveKeyWithValue("foo", "bar")) + g.Expect(cm.Labels).To(HaveKeyWithValue(managedByLabel, meta.ValueControllerReplications)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + }) + + Context("prune on selector drift", func() { + It("prunes objects from tenants that stop matching tenantSelector", func() { + gtr := newRawConfigMapGlobalTenantResource("gtr-tenant-selector-prune", map[string]string{ + "mode": "both", + }) + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: "energy", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"solar", "lunar"}, + }}, + } + renameFirstRawConfigMap(gtr, "gtr-tenant-selector-prune") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + for _, ns := range append(tenantANamespaces, tenantBNamespaces...) { + expectConfigMapData(ns, "gtr-tenant-selector-prune", map[string]string{"mode": "both"}) + } + + Eventually(func() error { + current := &capsulev1beta2.GlobalTenantResource{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: gtr.Name}, current); err != nil { + return err + } + current.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + for _, ns := range tenantANamespaces { + expectConfigMapData(ns, "gtr-tenant-selector-prune", map[string]string{"mode": "both"}) + } + for _, ns := range tenantBNamespaces { + expectConfigMapDeleted(ns, "gtr-tenant-selector-prune") + } + }) + + It("prunes objects from namespaces that stop matching namespaceSelector", func() { + for _, ns := range []string{"e2e-gtr-tenant-a-one", "e2e-gtr-tenant-a-two"} { + Eventually(func() error { + n := &corev1.Namespace{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: ns}, n); err != nil { + return err + } + lbls := n.GetLabels() + lbls["replicate"] = "true" + n.SetLabels(lbls) + return k8sClient.Update(ctx, n) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + gtr := newRawConfigMapGlobalTenantResource("gtr-namespace-selector-prune", map[string]string{ + "mode": "selected", + }) + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + gtr.Spec.Resources[0].NamespaceSelector = &metav1.LabelSelector{ + MatchLabels: map[string]string{"replicate": "true"}, + } + renameFirstRawConfigMap(gtr, "gtr-namespace-selector-prune") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + expectConfigMapData("e2e-gtr-tenant-a-one", "gtr-namespace-selector-prune", map[string]string{"mode": "selected"}) + expectConfigMapData("e2e-gtr-tenant-a-two", "gtr-namespace-selector-prune", map[string]string{"mode": "selected"}) + + Eventually(func() error { + n := &corev1.Namespace{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: "e2e-gtr-tenant-a-two"}, n); err != nil { + return err + } + lbls := n.GetLabels() + delete(lbls, "replicate") + n.SetLabels(lbls) + return k8sClient.Update(ctx, n) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + expectConfigMapData("e2e-gtr-tenant-a-one", "gtr-namespace-selector-prune", map[string]string{"mode": "selected"}) + expectConfigMapDeleted("e2e-gtr-tenant-a-two", "gtr-namespace-selector-prune") + }) + }) + + Context("service account resolution", func() { + It("reflects the resolved service account in status", func() { + gtr := newRawConfigMapGlobalTenantResource("gtr-sa-resolution", map[string]string{"mode": "default"}) + gtr.Spec.ServiceAccount = nil + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + Eventually(func(g Gomega) { + current := &capsulev1beta2.GlobalTenantResource{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: gtr.Name}, current)).To(Succeed()) + g.Expect(current.Status.ServiceAccount).ToNot(BeNil()) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.Impersonation.GlobalDefaultServiceAccount = "default" + configuration.Spec.Impersonation.GlobalDefaultServiceAccountNamespace = "capsule-system" + }) + + Eventually(func(g Gomega) { + current := &capsulev1beta2.GlobalTenantResource{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: gtr.Name}, current)).To(Succeed()) + g.Expect(current.Status.ServiceAccount).ToNot(BeNil()) + g.Expect(current.Status.ServiceAccount.Name).To(Equal(apimeta.RFC1123Name("default"))) + g.Expect(current.Status.ServiceAccount.Namespace).To(Equal(apimeta.RFC1123SubdomainName("capsule-system"))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + }) + + Context("selection and fan-out", func() { + It("applies raw items to all namespaces of the selected tenants", func() { + gtr := newRawConfigMapGlobalTenantResource("gtr-apply-selected-tenants", map[string]string{ + "mode": "solar", + }) + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + renameFirstRawConfigMap(gtr, "gtr-shared-config") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + for _, ns := range tenantANamespaces { + expectConfigMapData(ns, "gtr-shared-config", map[string]string{"mode": "solar"}) + } + for _, ns := range tenantBNamespaces { + expectConfigMapAbsent(ns, "gtr-shared-config") + } + }) + + It("applies only to namespaces matching namespaceSelector within the selected tenants", func() { + for _, ns := range []string{"e2e-gtr-tenant-a-one", "e2e-gtr-tenant-a-two"} { + Eventually(func() error { + n := &corev1.Namespace{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: ns}, n); err != nil { + return err + } + labels := n.GetLabels() + labels["replicate"] = "true" + n.SetLabels(labels) + return k8sClient.Update(ctx, n) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + gtr := newRawConfigMapGlobalTenantResource("gtr-tenant-and-namespace-selector", map[string]string{ + "mode": "filtered", + }) + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + gtr.Spec.Resources[0].NamespaceSelector = &metav1.LabelSelector{ + MatchLabels: map[string]string{"replicate": "true"}, + } + renameFirstRawConfigMap(gtr, "gtr-filtered-config") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + expectConfigMapData("e2e-gtr-tenant-a-one", "gtr-filtered-config", map[string]string{"mode": "filtered"}) + expectConfigMapData("e2e-gtr-tenant-a-two", "gtr-filtered-config", map[string]string{"mode": "filtered"}) + expectConfigMapAbsent("e2e-gtr-tenant-a-system", "gtr-filtered-config") + + for _, ns := range tenantBNamespaces { + expectConfigMapAbsent(ns, "gtr-filtered-config") + } + }) + }) + + Context("apply lifecycle", func() { + It("updates previously applied objects across all selected namespaces", func() { + gtr := newRawConfigMapGlobalTenantResource("gtr-update", map[string]string{"mode": "before"}) + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + renameFirstRawConfigMap(gtr, "gtr-update-config") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + for _, ns := range tenantANamespaces { + expectConfigMapData(ns, "gtr-update-config", map[string]string{"mode": "before"}) + } + + Eventually(func() error { + current := &capsulev1beta2.GlobalTenantResource{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: gtr.Name}, current); err != nil { + return err + } + + current.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "gtr-update-config", + }, + Data: map[string]string{ + "mode": "after", + "foo": "bar", + }, + }, + }, + } + + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + for _, ns := range tenantANamespaces { + expectConfigMapData(ns, "gtr-update-config", map[string]string{ + "mode": "after", + "foo": "bar", + }) + } + }) + + It("prunes applied objects on delete when pruningOnDelete is enabled", func() { + gtr := newRawConfigMapGlobalTenantResource("gtr-prune-enabled", map[string]string{"mode": "prune"}) + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + gtr.Spec.PruningOnDelete = ptr.To(true) + renameFirstRawConfigMap(gtr, "gtr-pruned") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + for _, ns := range tenantANamespaces { + expectConfigMapData(ns, "gtr-pruned", map[string]string{"mode": "prune"}) + } + + Expect(k8sClient.Delete(ctx, gtr)).To(Succeed()) + + for _, ns := range tenantANamespaces { + expectConfigMapDeleted(ns, "gtr-pruned") + } + }) + + It("keeps applied objects on delete when pruningOnDelete is disabled", func() { + gtr := newRawConfigMapGlobalTenantResource("gtr-prune-disabled", map[string]string{"mode": "keep"}) + gtr.Spec.TenantSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + } + gtr.Spec.PruningOnDelete = ptr.To(false) + renameFirstRawConfigMap(gtr, "gtr-kept") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + for _, ns := range tenantANamespaces { + expectConfigMapData(ns, "gtr-kept", map[string]string{"mode": "keep"}) + } + + Expect(k8sClient.Delete(ctx, gtr)).To(Succeed()) + + for _, ns := range tenantANamespaces { + expectConfigMapData(ns, "gtr-kept", map[string]string{"mode": "keep"}) + } + + }) + + }) + + Context("namespace target enforcement", func() { + It("forces raw items into the iterated namespace even if metadata.namespace is set elsewhere", func() { + gtr := &capsulev1beta2.GlobalTenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gtr-raw-target-namespace", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "true", + }, + }, + Spec: capsulev1beta2.GlobalTenantResourceSpec{ + Scope: api.ResourceScopeNamespace, + TenantSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + }, + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + ResyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + PruningOnDelete: ptr.To(true), + Resources: []capsulev1beta2.ResourceSpec{{ + RawItems: []capsulev1beta2.RawExtension{{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "gtr-raw-namespace-enforced", + Namespace: "kube-system", + }, + Data: map[string]string{ + "source": "raw", + }, + }, + }, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + for _, ns := range tenantANamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "gtr-raw-namespace-enforced", Namespace: ns}, cm)).To(Succeed()) + g.Expect(cm.Namespace).To(Equal(ns)) + g.Expect(cm.Data).To(HaveKeyWithValue("source", "raw")) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + expectConfigMapAbsent("kube-system", "gtr-raw-namespace-enforced") + }) + }) + + Context("raw and generator merge", func() { + It("merges raw items and generators when they target the same object", func() { + gtr := &capsulev1beta2.GlobalTenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gtr-raw-and-generator-same-object", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "true", + }, + }, + Spec: capsulev1beta2.GlobalTenantResourceSpec{ + Scope: api.ResourceScopeNamespace, + TenantSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + }, + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + ResyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + PruningOnDelete: ptr.To(true), + Resources: []capsulev1beta2.ResourceSpec{{ + RawItems: []capsulev1beta2.RawExtension{{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{Name: "gtr-shared-merge"}, + Data: map[string]string{ + "static": "raw", + }, + }, + }, + }}, + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "error", + Template: `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gtr-shared-merge +data: + generated-{{ .namespace.metadata.name }}: "true" +`, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + for _, ns := range tenantANamespaces { + expectConfigMapData(ns, "gtr-shared-merge", map[string]string{ + "static": "raw", + fmt.Sprintf("generated-%s", ns): "true", + }) + } + }) + + }) + + Context("context loading", func() { + It("allows context loading from another namespace", func() { + sourceNs := "gtr-shared-context" + sourceNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: sourceNs}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, sourceNamespace) }).Should(Succeed()) + defer ForceDeleteNamespace(ctx, sourceNs) + + for _, name := range []string{"ctx-1", "ctx-2"} { + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: sourceNs, + Labels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{".dockerconfigjson": "e30="}, + } + EventuallyCreation(func() error { + sec.ResourceVersion = "" + return k8sClient.Create(ctx, sec) + }).Should(Succeed()) + } + + gtr := &capsulev1beta2.GlobalTenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gtr-context-cross-namespace", + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "true", + }, + }, + Spec: capsulev1beta2.GlobalTenantResourceSpec{ + Scope: api.ResourceScopeNamespace, + TenantSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"energy": "solar"}, + }, + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + ResyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + PruningOnDelete: ptr.To(true), + Resources: []capsulev1beta2.ResourceSpec{{ + Context: &template.TemplateContext{ + Resources: []*template.TemplateResourceReference{{ + Index: "secrets", + ResourceReference: template.ResourceReference{ + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Secret", + }, + Namespace: sourceNs, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + }, + }}, + }, + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "error", + Template: `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gtr-context-count +data: + count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}' +`, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, gtr) }).Should(Succeed()) + + for _, ns := range tenantANamespaces { + expectConfigMapData(ns, "gtr-context-count", map[string]string{"count": "2"}) + } + }) + }) +}) + +func newRawConfigMapGlobalTenantResource(name string, data map[string]string) *capsulev1beta2.GlobalTenantResource { + return &capsulev1beta2.GlobalTenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "e2e.capsule.dev/test-suite": "true", + }, + }, + Spec: capsulev1beta2.GlobalTenantResourceSpec{ + Scope: api.ResourceScopeNamespace, + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + ResyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + PruningOnDelete: ptr.To(true), + Resources: []capsulev1beta2.ResourceSpec{{ + RawItems: []capsulev1beta2.RawExtension{{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "shared-config", + }, + Data: data, + }, + }, + }}, + AdditionalMetadata: &api.AdditionalMetadataSpec{ + Labels: map[string]string{ + "extra-label": "set-by-gtr", + }, + }, + }}, + }, + }, + } +} + +func renameFirstRawConfigMap(gtr *capsulev1beta2.GlobalTenantResource, name string) { + gtr.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Data: gtr.Spec.Resources[0].RawItems[0].RawExtension.Object.(*corev1.ConfigMap).Data, + }, + }, + } +} + +func newRawConfigMapGlobalTenantResourceWithScope(name string, scope api.ResourceScope, data map[string]string) *capsulev1beta2.GlobalTenantResource { + gtr := newRawConfigMapGlobalTenantResource(name, data) + gtr.Spec.Scope = scope + return gtr +} + +func expectGlobalTenantResourceReady(name string) { + Eventually(func(g Gomega) { + current := &capsulev1beta2.GlobalTenantResource{} + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name}, current)).To(Succeed()) + + rdy := current.Status.Conditions.GetConditionByType(apimeta.ReadyCondition) + g.Expect(rdy).ToNot(BeNil()) + g.Expect(rdy.Status).To(Equal(metav1.ConditionTrue)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func expectGlobalTenantResourceFailed(name, msgContains string) { + Eventually(func(g Gomega) { + current := &capsulev1beta2.GlobalTenantResource{} + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name}, current)).To(Succeed()) + + rdy := current.Status.Conditions.GetConditionByType(apimeta.ReadyCondition) + g.Expect(rdy).ToNot(BeNil()) + g.Expect(rdy.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(rdy.Message).To(ContainSubstring(msgContains)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} diff --git a/e2e/replications_tenantresource_test.go b/e2e/replications_tenantresource_test.go new file mode 100644 index 00000000..ec01acaf --- /dev/null +++ b/e2e/replications_tenantresource_test.go @@ -0,0 +1,1912 @@ +package e2e + +import ( + "context" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" + apimeta "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" + "github.com/projectcapsule/capsule/pkg/runtime/gvk" + "github.com/projectcapsule/capsule/pkg/template" +) + +var ( + resyncPeriod = metav1.Duration{Duration: 10 * time.Second} +) + +var _ = Describe("TenantResource SSA", Ordered, Label("replications", "namespace", "tenantresource"), Ordered, func() { + var ( + ctx context.Context + tnt *capsulev1beta2.Tenant + baseNamespace string + targetNamespaces []string + tenantOwner rbac.UserSpec + additionalBindingUser rbac.UserSpec + sharedSourceSecret *corev1.Secret + contextSecretOne *corev1.Secret + contextSecretTwo *corev1.Secret + ) + + originalConfig := &capsulev1beta2.CapsuleConfiguration{} + + BeforeEach(func() { + ctx = context.Background() + baseNamespace = "e2e-tenantresource-ssa-system" + targetNamespaces = []string{"e2e-tenantresource-ssa-one", "e2e-tenantresource-ssa-two", "e2e-tenantresource-ssa-three"} + tenantOwner = rbac.UserSpec{Name: "e2e-tr-owner", Kind: rbac.OwnerKind("User")} + additionalBindingUser = rbac.UserSpec{Name: "e2e-tr-additional", Kind: rbac.OwnerKind("User")} + + Expect(k8sClient.Get(context.Background(), client.ObjectKey{Name: defaultConfigurationName}, originalConfig)).To(Succeed()) + + tnt = &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-tenantresource-ssa", + Labels: map[string]string{ + "env": "e2e", + }, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{{ + CoreOwnerSpec: rbac.CoreOwnerSpec{UserSpec: tenantOwner}, + }}, + AdditionalRoleBindings: []rbac.AdditionalRoleBindingsSpec{{ + ClusterRoleName: "admin", + Subjects: []rbacv1.Subject{{Kind: "User", Name: additionalBindingUser.Name}}, + }}, + }, + } + + sharedSourceSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "seed-secret", + Namespace: baseNamespace, + Labels: map[string]string{ + "replicate": "true", + "source": "static", + }, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{"seed": "base"}, + } + + contextSecretOne = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pull-secret-one", + Namespace: "e2e-tenantresource-ssa-one", + Labels: map[string]string{"pullsecret.company.com": "true"}, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{".dockerconfigjson": "e30="}, + } + contextSecretTwo = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pull-secret-two", + Namespace: "e2e-tenantresource-ssa-one", + Labels: map[string]string{"pullsecret.company.com": "true"}, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{".dockerconfigjson": "e30="}, + } + + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: defaultConfigurationName}, originalConfig)).To(Succeed()) + + EventuallyCreation(func() error { + tnt.ResourceVersion = "" + return k8sClient.Create(ctx, tnt) + }).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) + + for _, ns := range append(append([]string{}, targetNamespaces...), baseNamespace) { + namespace := NewNamespace(ns, map[string]string{apimeta.TenantLabel: tnt.GetName()}) + NamespaceCreation(namespace, tenantOwner, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, namespace).Should(Succeed()) + } + }) + + AfterEach(func() { + ignoreNotFound(k8sClient.Delete(ctx, sharedSourceSecret)) + ignoreNotFound(k8sClient.Delete(ctx, contextSecretOne)) + ignoreNotFound(k8sClient.Delete(ctx, contextSecretTwo)) + + for _, ns := range append([]string{baseNamespace}, targetNamespaces...) { + ForceDeleteNamespace(ctx, ns) + } + + EventuallyDeletion(tnt) + + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec = originalConfig.Spec + }) + }) + + Context("generators and template context", func() { + + It("fails when a templated namespace resolves to a forbidden namespace", func() { + foreignSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "templated-foreign-secret", + Namespace: "kube-system", + Labels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{"token": "forbidden"}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, foreignSecret) }).Should(Succeed()) + defer ignoreNotFound(k8sClient.Delete(ctx, foreignSecret)) + + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "templated-forbidden-namespace", + Namespace: baseNamespace, + }, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + Context: &template.TemplateContext{ + Resources: []*template.TemplateResourceReference{{ + Index: "secrets", + ResourceReference: template.ResourceReference{ + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Secret", + }, + Namespace: "{{ forbiddenNamespace }}", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + }, + }}, + }, + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "zero", + Template: `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: templated-forbidden-context +data: + count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}' +`, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + + Eventually(func(g Gomega) { + current := &capsulev1beta2.TenantResource{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: tr.Namespace}, current)).To(Succeed()) + + rdy := current.Status.Conditions.GetConditionByType(apimeta.ReadyCondition) + g.Expect(rdy).ToNot(BeNil()) + g.Expect(rdy.Status).To(Equal(metav1.ConditionTrue)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + It("renders generator templates with tenant and namespace data", func() { + tr := newGeneratorConfigMapTenantResource(baseNamespace, "generator-template", `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: generated-{{ $.namespace.metadata.name }} +data: + tenant: "{{ $.tenant.metadata.name }}" + namespace: "{{ $.namespace.metadata.name }}" +`) + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + for _, ns := range targetNamespaces { + expectConfigMapData(ns, fmt.Sprintf("generated-%s", ns), map[string]string{ + "tenant": tnt.Name, + "namespace": ns, + }) + expectProcessedItemStatus(baseNamespace, tr.Name, configMapRID(tnt.Name, ns, fmt.Sprintf("generated-%s", ns), "0/generator-0-0"), metav1.ConditionTrue, true, "") + } + }) + + It("places generated objects into the current tenant namespace even when the template sets metadata.namespace to a foreign namespace", func() { + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "generator-enforce-target-namespace", + Namespace: baseNamespace, + }, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "error", + Template: `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: generated-namespace-locked + namespace: kube-system +data: + source: generator + renderedFor: "{{ $.namespace.metadata.name }}" +`, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + By("verifying the generated object is created in each tenant namespace, not in the foreign namespace") + for _, ns := range targetNamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: "generated-namespace-locked", + Namespace: ns, + }, cm)).To(Succeed()) + g.Expect(cm.Namespace).To(Equal(ns)) + g.Expect(cm.Data).To(HaveKeyWithValue("source", "generator")) + g.Expect(cm.Data).To(HaveKeyWithValue("renderedFor", ns)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + Consistently(func() error { + return k8sClient.Get(ctx, types.NamespacedName{ + Name: "generated-namespace-locked", + Namespace: "kube-system", + }, &corev1.ConfigMap{}) + }, 5*time.Second, defaultPollInterval).Should(HaveOccurred()) + }) + + It("loads context from the explicitly referenced namespace for every rendered namespace", func() { + EventuallyCreation(func() error { return k8sClient.Create(ctx, contextSecretOne) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, contextSecretTwo) }).Should(Succeed()) + + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{Name: "context-loading-fixed-namespace", Namespace: baseNamespace}, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + Context: &template.TemplateContext{ + Resources: []*template.TemplateResourceReference{{ + Index: "secrets", + ResourceReference: template.ResourceReference{ + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Secret", + }, + Namespace: "e2e-tenantresource-ssa-one", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + }, + }}, + }, + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "zero", + Template: `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: show-context +data: + count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}' +`, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + expectConfigMapData("e2e-tenantresource-ssa-one", "show-context", map[string]string{"count": "2"}) + expectConfigMapData("e2e-tenantresource-ssa-two", "show-context", map[string]string{"count": "2"}) + expectConfigMapData("e2e-tenantresource-ssa-three", "show-context", map[string]string{"count": "2"}) + }) + + It("loads context from the current iterating namespace", func() { + EventuallyCreation(func() error { return k8sClient.Create(ctx, contextSecretOne) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, contextSecretTwo) }).Should(Succeed()) + + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{Name: "context-loading-variable-namespace", Namespace: baseNamespace}, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + Context: &template.TemplateContext{ + Resources: []*template.TemplateResourceReference{{ + Index: "secrets", + ResourceReference: template.ResourceReference{ + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Secret", + }, + Namespace: "{{ namespace }}", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + }, + }}, + }, + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "zero", + Template: `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: show-context +data: + count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}' +`, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + expectConfigMapData("e2e-tenantresource-ssa-one", "show-context", map[string]string{"count": "2"}) + expectConfigMapData("e2e-tenantresource-ssa-two", "show-context", map[string]string{"count": "0"}) + expectConfigMapData("e2e-tenantresource-ssa-three", "show-context", map[string]string{"count": "0"}) + }) + + It("loads context per rendered namespace", func() { + EventuallyCreation(func() error { return k8sClient.Create(ctx, contextSecretOne) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, contextSecretTwo) }).Should(Succeed()) + + solarTwoSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pull-secret-e2e-tenantresource-ssa-two", + Namespace: "e2e-tenantresource-ssa-two", + Labels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{".dockerconfigjson": "e30="}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, solarTwoSecret) }).Should(Succeed()) + + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{Name: "context-variable-namespace", Namespace: baseNamespace}, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + Context: &template.TemplateContext{ + Resources: []*template.TemplateResourceReference{{ + Index: "secrets", + ResourceReference: template.ResourceReference{ + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Secret", + }, + Namespace: "{{.namespace}}", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + }, + }}, + }, + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "zero", + Template: `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: show-context +data: + count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}' +`, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + expectConfigMapData("e2e-tenantresource-ssa-one", "show-context", map[string]string{"count": "2"}) + expectConfigMapData("e2e-tenantresource-ssa-two", "show-context", map[string]string{"count": "1"}) + expectConfigMapData("e2e-tenantresource-ssa-three", "show-context", map[string]string{"count": "0"}) + }) + + It("fails when context tries to load from a namespace outside the tenant", func() { + foreignSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foreign-pull-secret", + Namespace: "kube-system", + Labels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{".dockerconfigjson": "e30="}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, foreignSecret) }).Should(Succeed()) + defer ignoreNotFound(k8sClient.Delete(ctx, foreignSecret)) + + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "context-forbidden-namespace", + Namespace: baseNamespace, + }, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + Context: &template.TemplateContext{ + Resources: []*template.TemplateResourceReference{{ + Index: "secrets", + ResourceReference: template.ResourceReference{ + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Secret", + }, + Namespace: "kube-system", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + }, + }}, + }, + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "zero", + Template: `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: forbidden-context +data: + count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}' +`, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + + Eventually(func(g Gomega) { + current := &capsulev1beta2.TenantResource{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: tr.Namespace}, current)).To(Succeed()) + + rdy := current.Status.Conditions.GetConditionByType(apimeta.ReadyCondition) + g.Expect(rdy).ToNot(BeNil()) + g.Expect(rdy.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(rdy.Message).To(ContainSubstring("cross-namespace selection is not allowed")) + g.Expect(rdy.Message).To(ContainSubstring("kube-system")) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + for _, ns := range targetNamespaces { + Consistently(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: "forbidden-context", Namespace: ns}, &corev1.ConfigMap{}) + }, 5*time.Second, defaultPollInterval).Should(HaveOccurred()) + } + }) + + It("fails when namespacedItems tries to load from a namespace outside the tenant", func() { + foreignSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foreign-source-secret", + Namespace: "kube-system", + Labels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{"token": "forbidden"}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, foreignSecret) }).Should(Succeed()) + defer ignoreNotFound(k8sClient.Delete(ctx, foreignSecret)) + + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "namespaceditems-forbidden-namespace", + Namespace: baseNamespace, + }, + Spec: capsulev1beta2.TenantResourceSpec{ + ServiceAccount: &apimeta.LocalRFC1123ObjectReference{Name: apimeta.RFC1123Name("replicator")}, + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + NamespacedItems: []template.ResourceReference{{ + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Secret", + }, + Namespace: "kube-system", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + }}, + }}, + }, + }, + } + + EnsureServiceAccount(ctx, k8sClient, tr.Spec.ServiceAccount.Name.String(), baseNamespace) + EnsureRoleAndBindingForNamespaces(ctx, k8sClient, tr.Spec.ServiceAccount.Name.String(), baseNamespace, append(targetNamespaces, baseNamespace)) + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + + Eventually(func(g Gomega) { + current := &capsulev1beta2.TenantResource{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: tr.Namespace}, current)).To(Succeed()) + + rdy := current.Status.Conditions.GetConditionByType(apimeta.ReadyCondition) + g.Expect(rdy).ToNot(BeNil()) + g.Expect(rdy.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(rdy.Message).To(ContainSubstring("cross-namespace selection is not allowed")) + g.Expect(rdy.Message).To(ContainSubstring("kube-system")) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + for _, ns := range targetNamespaces { + Consistently(func() error { + return k8sClient.Get(ctx, types.NamespacedName{Name: "foreign-source-secret", Namespace: ns}, &corev1.Secret{}) + }, 5*time.Second, defaultPollInterval).Should(HaveOccurred()) + } + }) + + It("places rawItems into the current tenant namespace even when metadata.namespace is set to a foreign namespace", func() { + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rawitems-enforce-target-namespace", + Namespace: baseNamespace, + }, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + RawItems: []capsulev1beta2.RawExtension{{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "raw-namespace-locked", + Namespace: "kube-system", + }, + Data: map[string]string{ + "source": "raw", + }, + }, + }, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + By("verifying the object is created in each tenant namespace, not in the foreign namespace") + for _, ns := range targetNamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: "raw-namespace-locked", + Namespace: ns, + }, cm)).To(Succeed()) + g.Expect(cm.Namespace).To(Equal(ns)) + g.Expect(cm.Data).To(HaveKeyWithValue("source", "raw")) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + Consistently(func() error { + return k8sClient.Get(ctx, types.NamespacedName{ + Name: "raw-namespace-locked", + Namespace: "kube-system", + }, &corev1.ConfigMap{}) + }, 5*time.Second, defaultPollInterval).Should(HaveOccurred()) + }) + + }) + + Context("multiple TenantResources targeting the same object", func() { + It("allows non-conflicting ownership without force", func() { + first := newGeneratorConfigMapTenantResource(baseNamespace, "same-object-no-force-a", `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: owned-together +data: + from-a: one +`) + second := newRawConfigMapTenantResource(baseNamespace, "same-object-no-force-b", map[string]string{"from-b": "two"}) + second.Spec.Resources[0].RawItems[0].RawExtension.Object.(*corev1.ConfigMap).Name = "owned-together" + + EventuallyCreation(func() error { return k8sClient.Create(ctx, first) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, second) }).Should(Succeed()) + + expectTenantResourceReady(baseNamespace, first.Name) + expectTenantResourceReady(baseNamespace, second.Name) + + for _, ns := range targetNamespaces { + expectConfigMapData(ns, "owned-together", map[string]string{"from-a": "one", "from-b": "two"}) + } + }) + + It("fails on conflicting ownership without force", func() { + first := newRawConfigMapTenantResource(baseNamespace, "same-object-conflict-a", map[string]string{"shared": "one"}) + first.Spec.Resources[0].RawItems[0].RawExtension.Object.(*corev1.ConfigMap).Name = "force-target" + + second := newRawConfigMapTenantResource(baseNamespace, "same-object-conflict-b", map[string]string{"shared": "two"}) + second.Spec.Resources[0].RawItems[0].RawExtension.Object.(*corev1.ConfigMap).Name = "force-target" + second.Spec.Settings.Force = ptr.To(false) + + EventuallyCreation(func() error { return k8sClient.Create(ctx, first) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, first.Name) + + EventuallyCreation(func() error { return k8sClient.Create(ctx, second) }).Should(Succeed()) + expectTenantResourceFailed(baseNamespace, second.Name, "applying of") + + for _, ns := range targetNamespaces { + expectConfigMapData(ns, "force-target", map[string]string{"shared": "one"}) + } + }) + + It("wins conflicting ownership with force", func() { + first := newRawConfigMapTenantResource(baseNamespace, "same-object-force-a", map[string]string{"shared": "one"}) + first.Spec.Resources[0].RawItems[0].RawExtension.Object.(*corev1.ConfigMap).Name = "forced-target" + + second := newRawConfigMapTenantResource(baseNamespace, "same-object-force-b", map[string]string{"shared": "two"}) + second.Spec.Resources[0].RawItems[0].RawExtension.Object.(*corev1.ConfigMap).Name = "forced-target" + second.Spec.Settings.Force = ptr.To(true) + + EventuallyCreation(func() error { return k8sClient.Create(ctx, first) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, first.Name) + + EventuallyCreation(func() error { return k8sClient.Create(ctx, second) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, second.Name) + + for _, ns := range targetNamespaces { + expectConfigMapData(ns, "forced-target", map[string]string{"shared": "two"}) + } + }) + }) + + Context("namespaced item replication", func() { + It("replicates source objects and strips selector labels to avoid loops", func() { + EventuallyCreation(func() error { return k8sClient.Create(ctx, sharedSourceSecret) }).Should(Succeed()) + + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{Name: "selector-replication", Namespace: baseNamespace}, + Spec: capsulev1beta2.TenantResourceSpec{ + ServiceAccount: &apimeta.LocalRFC1123ObjectReference{Name: apimeta.RFC1123Name("replicator")}, + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + NamespacedItems: []template.ResourceReference{{ + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Secret", + }, + Namespace: baseNamespace, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "replicate": "true", + }}, + }}, + }}, + }, + }, + } + + EnsureServiceAccount(ctx, k8sClient, tr.Spec.ServiceAccount.Name.String(), baseNamespace) + EnsureRoleAndBindingForNamespaces(ctx, k8sClient, tr.Spec.ServiceAccount.Name.String(), baseNamespace, append(targetNamespaces, baseNamespace)) + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + for _, ns := range targetNamespaces { + Eventually(func(g Gomega) { + sec := &corev1.Secret{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: sharedSourceSecret.Name, Namespace: ns}, sec)).To(Succeed()) + g.Expect(sec.Labels).ToNot(HaveKey("replicate")) + g.Expect(sec.Labels).To(HaveKeyWithValue("source", "static")) + g.Expect(sec.Labels).To(HaveKeyWithValue(apimeta.CreatedByCapsuleLabel, apimeta.ValueControllerReplications)) + g.Expect(sec.Labels).To(HaveKeyWithValue(apimeta.NewManagedByCapsuleLabel, apimeta.ValueControllerReplications)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + }) + + Context("apply lifecycle with prune enabled", func() { + It("applies, updates and prunes raw items", func() { + tr := newRawConfigMapTenantResource(baseNamespace, "raw-prune-enabled", map[string]string{ + "mode": "before", + "foo": "one", + }) + tr.Spec.PruningOnDelete = ptr.To(true) + + By("creating the TenantResource") + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + By("verifying the created ConfigMaps and status entries") + for _, ns := range targetNamespaces { + expectConfigMapData(ns, "shared-config", map[string]string{"mode": "before", "foo": "one"}) + expectManagedLabelsOnConfigMap(ns, "shared-config", true) + expectProcessedItemStatus(baseNamespace, tr.Name, configMapRID(tnt.Name, ns, "shared-config", "0/raw-0"), metav1.ConditionTrue, true, "") + } + + By("updating the applied data") + Eventually(func() error { + current := &capsulev1beta2.TenantResource{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: baseNamespace}, current); err != nil { + return err + } + + current.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "shared-config", + }, + Data: map[string]string{ + "mode": "after", + "foo": "two", + "bar": "three", + }, + }, + }, + } + + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + for _, ns := range targetNamespaces { + expectConfigMapData(ns, "shared-config", map[string]string{"mode": "after", "foo": "two", "bar": "three"}) + expectProcessedItemApplied(baseNamespace, tr.Name, configMapRID(tnt.Name, ns, "shared-config", "0/raw-0")) + } + + By("deleting the TenantResource and pruning created objects") + Expect(k8sClient.Delete(ctx, tr)).To(Succeed()) + for _, ns := range targetNamespaces { + expectConfigMapDeleted(ns, "shared-config") + } + }) + }) + + It("places generated objects into the current tenant namespace even when the template sets metadata.namespace to a foreign namespace", func() { + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "generator-enforce-target-namespace", + Namespace: baseNamespace, + }, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "error", + Template: `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: generated-namespace-locked + namespace: kube-system +data: + source: generator + renderedFor: "{{ $.namespace.metadata.name }}" +`, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + By("verifying the generated object is created in each tenant namespace, not in the foreign namespace") + for _, ns := range targetNamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: "generated-namespace-locked", + Namespace: ns, + }, cm)).To(Succeed()) + g.Expect(cm.Namespace).To(Equal(ns)) + g.Expect(cm.Data).To(HaveKeyWithValue("source", "generator")) + g.Expect(cm.Data).To(HaveKeyWithValue("renderedFor", ns)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + Consistently(func() error { + return k8sClient.Get(ctx, types.NamespacedName{ + Name: "generated-namespace-locked", + Namespace: "kube-system", + }, &corev1.ConfigMap{}) + }, 5*time.Second, defaultPollInterval).Should(HaveOccurred()) + }) + + It("merge rawItems and generators when they target the same object", func() { + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "raw-and-generator-same-object", + Namespace: baseNamespace, + }, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + RawItems: []capsulev1beta2.RawExtension{{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "raw-generated-shared", + }, + Data: map[string]string{ + "static": "raw", + }, + }, + }, + }}, + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "zero", + Template: `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: raw-generated-shared +data: + generated-{{ $.namespace.metadata.name }}: "true" +`, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + By("verifying the object contains both raw and generated data in every target namespace") + for _, ns := range targetNamespaces { + expectConfigMapData(ns, "raw-generated-shared", map[string]string{ + "static": "raw", + "generated-" + ns: "true", + }) + } + }) + + It("allows multiple TenantResources to adopt and co-manage the same preexisting object with non-conflicting fields", func() { + By("creating the preexisting object in all target namespaces") + for _, ns := range targetNamespaces { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "shared-adopted-config", + Namespace: ns, + }, + Data: map[string]string{ + "existing": "true", + }, + } + EventuallyCreation(func() error { + cm.ResourceVersion = "" + return k8sClient.Create(ctx, cm) + }).Should(Succeed()) + } + + trA := newRawConfigMapTenantResource(baseNamespace, "adopt-shared-a", map[string]string{ + "foo": "one", + }) + trA.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "shared-adopted-config", + }, + Data: map[string]string{ + "foo": "one", + }, + }, + }, + } + trA.Spec.Settings.Adopt = ptr.To(true) + + trB := newRawConfigMapTenantResource(baseNamespace, "adopt-shared-b", map[string]string{ + "bar": "two", + }) + trB.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "shared-adopted-config", + }, + Data: map[string]string{ + "bar": "two", + }, + }, + }, + } + trB.Spec.Settings.Adopt = ptr.To(true) + + By("creating both TenantResources") + EventuallyCreation(func() error { return k8sClient.Create(ctx, trA) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, trB) }).Should(Succeed()) + + expectTenantResourceReady(baseNamespace, trA.Name) + expectTenantResourceReady(baseNamespace, trB.Name) + + By("verifying the final object contains the merged fields and is adopted, not created") + for _, ns := range targetNamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "shared-adopted-config", Namespace: ns}, cm)).To(Succeed()) + g.Expect(cm.Data).To(HaveKeyWithValue("existing", "true")) + g.Expect(cm.Data).To(HaveKeyWithValue("foo", "one")) + g.Expect(cm.Data).To(HaveKeyWithValue("bar", "two")) + g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.NewManagedByCapsuleLabel, apimeta.ValueControllerReplications)) + g.Expect(cm.Labels).ToNot(HaveKey(apimeta.CreatedByCapsuleLabel)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + + It("aligns objects created by the legacy resource label implementation", func() { + By("creating legacy-labelled objects in all target namespaces") + for _, ns := range targetNamespaces { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "legacy-aligned-config", + Namespace: ns, + Labels: map[string]string{ + "capsule.clastix.io/resources": "0", + apimeta.TenantLabel: tnt.GetName(), + }, + }, + Data: map[string]string{ + "legacy": "true", + }, + } + EventuallyCreation(func() error { + cm.ResourceVersion = "" + return k8sClient.Create(ctx, cm) + }).Should(Succeed()) + } + + tr := newRawConfigMapTenantResource(baseNamespace, "legacy-alignment", map[string]string{ + "mode": "new-controller", + "foo": "bar", + }) + tr.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "legacy-aligned-config", + }, + Data: map[string]string{ + "mode": "new-controller", + "foo": "bar", + }, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + By("verifying the object was aligned to the new implementation") + for _, ns := range targetNamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: "legacy-aligned-config", + Namespace: ns, + }, cm)).To(Succeed()) + + g.Expect(cm.Data).To(HaveKeyWithValue("mode", "new-controller")) + g.Expect(cm.Data).To(HaveKeyWithValue("foo", "bar")) + + // legacy marker still present from the old object + g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.TenantLabel, tnt.GetName())) + + // new implementation metadata + g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.CreatedByCapsuleLabel, apimeta.ValueControllerReplications)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + + Context("impersonation", func() { + It("reflects the resolved service account in status", func() { + tr := newRawConfigMapTenantResource(baseNamespace, "sa-resolution", map[string]string{"mode": "default-controller"}) + tr.Spec.ServiceAccount = nil + + By("creating the TenantResource without an explicit ServiceAccount") + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + + By("defaulting to the controller service account") + expectResolvedServiceAccount(baseNamespace, tr.Name, "capsule", ControllerNamespace) + + By("configuring a tenant default service account") + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.Impersonation.TenantDefaultServiceAccount = "default" + }) + expectResolvedServiceAccount(baseNamespace, tr.Name, "default", baseNamespace) + + By("overriding with an explicit service account on the TenantResource") + Eventually(func() error { + current := &capsulev1beta2.TenantResource{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: baseNamespace}, current); err != nil { + return err + } + current.Spec.ServiceAccount = &apimeta.LocalRFC1123ObjectReference{Name: apimeta.RFC1123Name("custom-account")} + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + expectResolvedServiceAccount(baseNamespace, tr.Name, "custom-account", baseNamespace) + + By("removing the explicit override again") + Eventually(func() error { + current := &capsulev1beta2.TenantResource{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: baseNamespace}, current); err != nil { + return err + } + current.Spec.ServiceAccount = nil + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + expectResolvedServiceAccount(baseNamespace, tr.Name, "default", baseNamespace) + }) + + It("fails to apply raw items when the impersonated service account cannot create target resources", func() { + saName := "restricted-creator" + ensureServiceAccount(baseNamespace, saName) + + // Intentionally do not grant create/update/patch on configmaps in tenant namespaces. + + tr := newRawConfigMapTenantResource(baseNamespace, "sa-no-create", map[string]string{ + "mode": "blocked", + }) + tr.Spec.ServiceAccount = &apimeta.LocalRFC1123ObjectReference{ + Name: apimeta.RFC1123Name(saName), + } + renameFirstTenantResourceRawConfigMap(tr, "blocked-create-config") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + + expectResolvedServiceAccount(baseNamespace, tr.Name, saName, baseNamespace) + expectTenantResourceFailed(baseNamespace, tr.Name, "applying of") + + for _, ns := range targetNamespaces { + expectConfigMapAbsent(ns, "blocked-create-config") + } + }) + + It("fails to render generators when the impersonated service account cannot read context resources", func() { + saName := "restricted-context-reader" + ensureServiceAccount(baseNamespace, saName) + + // Create context source secret. + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ctx-secret", + Namespace: "e2e-tenantresource-ssa-one", + Labels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{".dockerconfigjson": "e30="}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, sec) }).Should(Succeed()) + + // Grant write on ConfigMaps in target namespaces if you want to isolate the failure to context loading. + for _, ns := range targetNamespaces { + bindServiceAccountToConfigMapWriter(baseNamespace, saName, ns) + } + // But do NOT grant get/list on secrets in e2e-tenantresource-ssa-one. + + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{Name: "sa-no-context-read", Namespace: baseNamespace}, + Spec: capsulev1beta2.TenantResourceSpec{ + ServiceAccount: &apimeta.LocalRFC1123ObjectReference{ + Name: apimeta.RFC1123Name(saName), + }, + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + ResyncPeriod: resyncPeriod, + PruningOnDelete: ptr.To(true), + Resources: []capsulev1beta2.ResourceSpec{{ + Context: &template.TemplateContext{ + Resources: []*template.TemplateResourceReference{{ + Index: "secrets", + ResourceReference: template.ResourceReference{ + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Secret", + }, + Namespace: "e2e-tenantresource-ssa-one", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "pullsecret.company.com": "true", + }, + }, + }, + }}, + }, + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "error", + Template: `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: blocked-context +data: + count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}' +`, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + + expectResolvedServiceAccount(baseNamespace, tr.Name, saName, baseNamespace) + expectTenantResourceFailed(baseNamespace, tr.Name, "forbidden") + + for _, ns := range targetNamespaces { + expectConfigMapAbsent(ns, "blocked-context") + } + }) + + It("fails to prune replicated resources when the impersonated service account cannot delete them", func() { + saCreate := "creator-ok" + saNoDelete := "creator-no-delete" + + ensureServiceAccount(baseNamespace, saCreate) + ensureServiceAccount(baseNamespace, saNoDelete) + + // Phase 1: creator can fully reconcile objects in every targeted namespace. + for _, ns := range append(targetNamespaces, baseNamespace) { + bindServiceAccountToConfigMapWriter(baseNamespace, saCreate, ns) + } + + // Phase 2 SA can still read/apply, but has no delete verb. + for _, ns := range append(targetNamespaces, baseNamespace) { + bindServiceAccountToConfigMapWriter(baseNamespace, saNoDelete, ns) + } + + tr := newRawConfigMapTenantResource(baseNamespace, "sa-no-prune", map[string]string{ + "mode": "created", + }) + tr.Spec.ServiceAccount = &apimeta.LocalRFC1123ObjectReference{ + Name: apimeta.RFC1123Name(saCreate), + } + tr.Spec.PruningOnDelete = ptr.To(true) + renameFirstTenantResourceRawConfigMap(tr, "prune-protected") + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + for _, ns := range append(targetNamespaces, baseNamespace) { + expectConfigMapData(ns, "prune-protected", map[string]string{"mode": "created"}) + } + + // Switch to the SA that cannot delete. + Eventually(func() error { + current := &capsulev1beta2.TenantResource{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: baseNamespace}, current); err != nil { + return err + } + current.Spec.ServiceAccount = &apimeta.LocalRFC1123ObjectReference{ + Name: apimeta.RFC1123Name(saNoDelete), + } + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Eventually(func(g Gomega) { + current := &capsulev1beta2.TenantResource{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: baseNamespace}, current)).To(Succeed()) + g.Expect(current.Status.ServiceAccount).ToNot(BeNil()) + g.Expect(current.Status.ServiceAccount.Name).To(Equal(apimeta.RFC1123Name(saNoDelete))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Expect(k8sClient.Delete(ctx, tr)).To(Succeed()) + + // Objects should remain because prune cannot delete them. + for _, ns := range append(targetNamespaces, baseNamespace) { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "prune-protected", Namespace: ns}, cm)).To(Succeed()) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + It("fails to replicate namespacedItems when the impersonated service account cannot read source resources", func() { + saName := "restricted-source-reader" + ensureServiceAccount(baseNamespace, saName) + + source := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-secret", + Namespace: baseNamespace, + Labels: map[string]string{ + "replicate": "true", + }, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{"token": "abc"}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, source) }).Should(Succeed()) + + // Allow write into targets only, but do not grant read on source namespace secrets. + for _, ns := range targetNamespaces { + bindServiceAccountToSecretWriter(baseNamespace, saName, ns) + } + + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{Name: "sa-no-namespaceditem-read", Namespace: baseNamespace}, + Spec: capsulev1beta2.TenantResourceSpec{ + ServiceAccount: &apimeta.LocalRFC1123ObjectReference{ + Name: apimeta.RFC1123Name(saName), + }, + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + ResyncPeriod: resyncPeriod, + PruningOnDelete: ptr.To(true), + Resources: []capsulev1beta2.ResourceSpec{{ + NamespacedItems: []template.ResourceReference{{ + VersionKind: gvk.VersionKind{ + APIVersion: "v1", + Kind: "Secret", + }, + Namespace: baseNamespace, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "replicate": "true", + }, + }, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + + expectResolvedServiceAccount(baseNamespace, tr.Name, saName, baseNamespace) + expectTenantResourceFailed(baseNamespace, tr.Name, "forbidden") + + for _, ns := range targetNamespaces { + expectSecretAbsent(ns, "source-secret") + } + }) + + }) + + Context("advanced TenantResource ownership and namespace behavior", func() { + It("fails when multiple TenantResources target the same preexisting object without adoption", func() { + By("creating the preexisting object in all target namespaces") + for _, ns := range targetNamespaces { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "shared-no-adopt-config", + Namespace: ns, + }, + Data: map[string]string{ + "existing": "true", + }, + } + EventuallyCreation(func() error { + cm.ResourceVersion = "" + return k8sClient.Create(ctx, cm) + }).Should(Succeed()) + } + + trA := newRawConfigMapTenantResource(baseNamespace, "no-adopt-shared-a", map[string]string{ + "foo": "one", + }) + trA.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "shared-no-adopt-config", + }, + Data: map[string]string{ + "foo": "one", + }, + }, + }, + } + trA.Spec.Settings.Adopt = ptr.To(false) + + trB := newRawConfigMapTenantResource(baseNamespace, "no-adopt-shared-b", map[string]string{ + "bar": "two", + }) + trB.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "shared-no-adopt-config", + }, + Data: map[string]string{ + "bar": "two", + }, + }, + }, + } + trB.Spec.Settings.Adopt = ptr.To(false) + + By("creating both TenantResources") + EventuallyCreation(func() error { return k8sClient.Create(ctx, trA) }).Should(Succeed()) + EventuallyCreation(func() error { return k8sClient.Create(ctx, trB) }).Should(Succeed()) + + expectTenantResourceFailed(baseNamespace, trA.Name, "applying of 3 resources failed") + expectTenantResourceFailed(baseNamespace, trB.Name, "applying of 3 resources failed") + + By("verifying the preexisting object remains unchanged") + for _, ns := range targetNamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "shared-no-adopt-config", Namespace: ns}, cm)).To(Succeed()) + g.Expect(cm.Data).To(Equal(map[string]string{ + "existing": "true", + })) + g.Expect(cm.Labels).ToNot(HaveKey(apimeta.NewManagedByCapsuleLabel)) + g.Expect(cm.Labels).ToNot(HaveKey(apimeta.CreatedByCapsuleLabel)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + + It("forces rawItems into the current iterating tenant namespace regardless of metadata.namespace", func() { + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "raw-target-namespace", + Namespace: baseNamespace, + }, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + RawItems: []capsulev1beta2.RawExtension{{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "raw-namespace-enforced", + Namespace: "kube-system", + }, + Data: map[string]string{ + "source": "raw", + }, + }, + }, + }}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + By("verifying the ConfigMap exists in each tenant namespace") + for _, ns := range targetNamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "raw-namespace-enforced", Namespace: ns}, cm)).To(Succeed()) + g.Expect(cm.Namespace).To(Equal(ns)) + g.Expect(cm.Data).To(HaveKeyWithValue("source", "raw")) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + By("verifying the ConfigMap was not created in the foreign namespace") + expectConfigMapAbsent("kube-system", "raw-namespace-enforced") + }) + }) + + Context("apply lifecycle with prune disabled", func() { + It("applies, updates and keeps objects while removing managed ownership", func() { + tr := newRawConfigMapTenantResource(baseNamespace, "raw-prune-disabled", map[string]string{"mode": "keep"}) + tr.Spec.PruningOnDelete = ptr.To(false) + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + for _, ns := range targetNamespaces { + expectConfigMapData(ns, "shared-config", map[string]string{"mode": "keep"}) + expectManagedLabelsOnConfigMap(ns, "shared-config", true) + } + + Expect(k8sClient.Delete(ctx, tr)).To(Succeed()) + + By("verifying the ConfigMaps remain but are no longer managed") + for _, ns := range targetNamespaces { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "shared-config", Namespace: ns}, cm)).To(Succeed()) + g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.CreatedByCapsuleLabel, apimeta.ValueControllerReplications)) + g.Expect(cm.Labels).ToNot(HaveKey(apimeta.NewManagedByCapsuleLabel)) + g.Expect(cm.Data).To(HaveKeyWithValue("mode", "keep")) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + }) + }) + + Context("adoption", func() { + It("fails without adopt and succeeds with adopt", func() { + preexisting := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "adopt-me", Namespace: "e2e-tenantresource-ssa-one"}, + Data: map[string]string{"existing": "true"}, + } + EventuallyCreation(func() error { return k8sClient.Create(ctx, preexisting) }).Should(Succeed()) + + withoutAdopt := newGeneratorConfigMapTenantResource(baseNamespace, "adopt-disabled", `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: adopt-me +data: + source: generator + namespace: "{{ $.namespace.metadata.name }}" +`) + withoutAdopt.Spec.PruningOnDelete = ptr.To(true) + withoutAdopt.Spec.Settings.Adopt = ptr.To(false) + + EventuallyCreation(func() error { return k8sClient.Create(ctx, withoutAdopt) }).Should(Succeed()) + expectTenantResourceFailed(baseNamespace, withoutAdopt.Name, "applying of") + expectProcessedItemStatus(baseNamespace, withoutAdopt.Name, configMapRID(tnt.Name, "e2e-tenantresource-ssa-one", "adopt-me", "0/generator-0-0"), metav1.ConditionFalse, false, "cannot be adopted") + + By("recreating a second resource with adoption enabled") + withAdopt := newGeneratorConfigMapTenantResource(baseNamespace, "adopt-enabled", `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: adopt-me +data: + source: generator + namespace: "{{ $.namespace.metadata.name }}" +`) + withAdopt.Spec.PruningOnDelete = ptr.To(true) + withAdopt.Spec.Settings.Adopt = ptr.To(true) + + EventuallyCreation(func() error { return k8sClient.Create(ctx, withAdopt) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, withAdopt.Name) + expectConfigMapData("e2e-tenantresource-ssa-one", "adopt-me", map[string]string{"source": "generator", "namespace": "e2e-tenantresource-ssa-one"}) + + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "adopt-me", Namespace: "e2e-tenantresource-ssa-one"}, cm)).To(Succeed()) + g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.NewManagedByCapsuleLabel, apimeta.ValueControllerReplications)) + g.Expect(cm.Labels).ToNot(HaveKey(apimeta.CreatedByCapsuleLabel)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + expectProcessedItemStatus(baseNamespace, withAdopt.Name, configMapRID(tnt.Name, "e2e-tenantresource-ssa-one", "adopt-me", "0/generator-0-0"), metav1.ConditionTrue, false, "") + }) + }) + + Context("same object within one TenantResource", func() { + It("merges non-conflicting fields from generator and raw item", func() { + tr := &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{Name: "same-object-merge", Namespace: baseNamespace}, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + PruningOnDelete: ptr.To(true), + ResyncPeriod: resyncPeriod, + Resources: []capsulev1beta2.ResourceSpec{{ + Generators: []capsulev1beta2.TemplateItemSpec{{ + MissingKey: "error", + Template: `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: common-config +data: + generated-{{ $.namespace.metadata.name }}: from-generator +`, + }}, + RawItems: []capsulev1beta2.RawExtension{{RawExtension: runtime.RawExtension{Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"}, + ObjectMeta: metav1.ObjectMeta{Name: "common-config"}, + Data: map[string]string{"additional-data": "raw"}, + }}}}, + }}, + }, + }, + } + + EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed()) + expectTenantResourceReady(baseNamespace, tr.Name) + + for _, ns := range targetNamespaces { + expectConfigMapData(ns, "common-config", map[string]string{ + fmt.Sprintf("generated-%s", ns): "from-generator", + "additional-data": "raw", + }) + } + }) + }) + +}) + +func newRawConfigMapTenantResource(namespace, name string, data map[string]string) *capsulev1beta2.TenantResource { + return &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + ResyncPeriod: resyncPeriod, + PruningOnDelete: ptr.To(true), + Resources: []capsulev1beta2.ResourceSpec{{ + RawItems: []capsulev1beta2.RawExtension{{RawExtension: runtime.RawExtension{Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"}, + ObjectMeta: metav1.ObjectMeta{Name: "shared-config"}, + Data: data, + }}}}, + AdditionalMetadata: &api.AdditionalMetadataSpec{Labels: map[string]string{"extra-label": "set-by-tr"}}, + }}, + }, + }, + } +} + +func newGeneratorConfigMapTenantResource(namespace, name, tpl string) *capsulev1beta2.TenantResource { + return &capsulev1beta2.TenantResource{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: capsulev1beta2.TenantResourceSpec{ + TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{ + ResyncPeriod: resyncPeriod, + PruningOnDelete: ptr.To(true), + Resources: []capsulev1beta2.ResourceSpec{{ + Generators: []capsulev1beta2.TemplateItemSpec{{MissingKey: "zero", Template: tpl}}, + }}, + }, + }, + } +} + +func getTenantResource(namespace, name string) *capsulev1beta2.TenantResource { + tr := &capsulev1beta2.TenantResource{} + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, tr)).To(Succeed()) + return tr +} + +func expectTenantResourceReady(namespace, name string) { + Eventually(func(g Gomega) { + tr := &capsulev1beta2.TenantResource{} + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, tr)).To(Succeed()) + rdy := tr.Status.Conditions.GetConditionByType(apimeta.ReadyCondition) + g.Expect(rdy).ToNot(BeNil()) + g.Expect(rdy.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(tr.Status.Size).To(BeNumerically(">", 0)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func expectTenantResourceFailed(namespace, name, contains string) { + Eventually(func(g Gomega) { + tr := &capsulev1beta2.TenantResource{} + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, tr)).To(Succeed()) + rdy := tr.Status.Conditions.GetConditionByType(apimeta.ReadyCondition) + g.Expect(rdy).ToNot(BeNil()) + g.Expect(rdy.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(strings.ToLower(rdy.Message)).To(ContainSubstring(strings.ToLower(contains))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func expectResolvedServiceAccount(namespace, name, saName, saNamespace string) { + Eventually(func(g Gomega) { + tr := &capsulev1beta2.TenantResource{} + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, tr)).To(Succeed()) + g.Expect(tr.Status.ServiceAccount).ToNot(BeNil()) + g.Expect(tr.Status.ServiceAccount.Name).To(Equal(apimeta.RFC1123Name(saName))) + g.Expect(tr.Status.ServiceAccount.Namespace).To(Equal(apimeta.RFC1123SubdomainName(saNamespace))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func expectConfigMapData(namespace, name string, expected map[string]string) { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, cm)).To(Succeed()) + for k, v := range expected { + g.Expect(cm.Data).To(HaveKeyWithValue(k, v)) + } + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func expectConfigMapDeleted(namespace, name string) { + Eventually(func() error { + cm := &corev1.ConfigMap{} + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, cm) + return client.IgnoreNotFound(err) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Consistently(func() bool { + cm := &corev1.ConfigMap{} + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, cm) + return client.IgnoreNotFound(err) == nil + }, 3*time.Second, defaultPollInterval).Should(BeTrue()) +} + +func expectManagedLabelsOnConfigMap(namespace, name string, created bool) { + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, cm)).To(Succeed()) + g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.NewManagedByCapsuleLabel, apimeta.ValueControllerReplications)) + if created { + g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.CreatedByCapsuleLabel, apimeta.ValueControllerReplications)) + } + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func expectProcessedItemApplied(namespace, trName string, rid gvk.ResourceID) { + Eventually(func(g Gomega) { + tr := getTenantResource(namespace, trName) + item := tr.Status.ProcessedItems.GetItem(rid) + g.Expect(item).ToNot(BeNil()) + g.Expect(item.LastApply).ToNot(BeNil()) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func expectProcessedItemStatus(namespace, trName string, rid gvk.ResourceID, cond metav1.ConditionStatus, created bool, msgContains string) { + Eventually(func(g Gomega) { + tr := getTenantResource(namespace, trName) + item := tr.Status.ProcessedItems.GetItem(rid) + g.Expect(item).ToNot(BeNil(), "processed item %+v not found", rid) + g.Expect(item.ObjectReferenceStatusCondition.Status).To(Equal(cond)) + g.Expect(item.ObjectReferenceStatusCondition.Type).To(Equal(apimeta.ReadyCondition)) + g.Expect(item.ObjectReferenceStatusCondition.Created).To(Equal(created)) + if msgContains != "" { + g.Expect(strings.ToLower(item.ObjectReferenceStatusCondition.Message)).To(ContainSubstring(strings.ToLower(msgContains))) + } + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func configMapRID(tenant, namespace, name, origin string) gvk.ResourceID { + return gvk.ResourceID{ + Version: "v1", + Kind: "ConfigMap", + Name: name, + Namespace: namespace, + TenantResourceIDWithOrigin: gvk.TenantResourceIDWithOrigin{ + Origin: origin, + TenantResourceID: gvk.TenantResourceID{Tenant: tenant}, + }, + } +} + +func expectConfigMapAbsent(namespace, name string) { + Consistently(func() error { + return k8sClient.Get(context.Background(), types.NamespacedName{ + Name: name, + Namespace: namespace, + }, &corev1.ConfigMap{}) + }, 5*time.Second, defaultPollInterval).Should(HaveOccurred()) +} + +func expectSecretAbsent(namespace, name string) { + Consistently(func() error { + return k8sClient.Get(context.Background(), types.NamespacedName{ + Name: name, + Namespace: namespace, + }, &corev1.Secret{}) + }, 5*time.Second, defaultPollInterval).Should(HaveOccurred()) +} + +func renameFirstTenantResourceRawConfigMap(tr *capsulev1beta2.TenantResource, name string) { + tr.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{ + RawExtension: runtime.RawExtension{ + Object: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Data: tr.Spec.Resources[0].RawItems[0].RawExtension.Object.(*corev1.ConfigMap).Data, + }, + }, + } +} + +func bindServiceAccountToNamespacedResource( + saNamespace, saName, targetNamespace string, + resources, verbs []string, +) { + ctx := context.Background() + + resourceKey := strings.Join(resources, "-") + roleName := fmt.Sprintf("sa-%s-%s-%s", saName, resourceKey, targetNamespace) + roleBindingName := fmt.Sprintf("sa-%s-%s-%s-binding", saName, resourceKey, targetNamespace) + + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleName, + Namespace: targetNamespace, + }, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: resources, + Verbs: verbs, + }}, + } + + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleBindingName, + Namespace: targetNamespace, + }, + Subjects: []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: saName, + Namespace: saNamespace, + }}, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "Role", + Name: roleName, + }, + } + + Eventually(func() error { + current := &rbacv1.Role{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: roleName, Namespace: targetNamespace}, current) + if apierrors.IsNotFound(err) { + return k8sClient.Create(ctx, role) + } + if err != nil { + return err + } + + current.Rules = role.Rules + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Eventually(func() error { + current := &rbacv1.RoleBinding{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: roleBindingName, Namespace: targetNamespace}, current) + if apierrors.IsNotFound(err) { + return k8sClient.Create(ctx, roleBinding) + } + if err != nil { + return err + } + + current.Subjects = roleBinding.Subjects + current.RoleRef = roleBinding.RoleRef + return k8sClient.Update(ctx, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func bindServiceAccountToSecretWriter(saNamespace, saName, targetNamespace string) { + bindServiceAccountToNamespacedResource( + saNamespace, + saName, + targetNamespace, + []string{"secrets"}, + []string{"get", "list", "watch", "create", "update", "patch"}, + ) +} + +func bindServiceAccountToSecretReader(saNamespace, saName, targetNamespace string) { + bindServiceAccountToNamespacedResource( + saNamespace, + saName, + targetNamespace, + []string{"secrets"}, + []string{"get", "list", "watch"}, + ) +} + +func bindServiceAccountToConfigMapWriter(saNamespace, saName, targetNamespace string) { + bindServiceAccountToNamespacedResource( + saNamespace, + saName, + targetNamespace, + []string{"configmaps"}, + []string{"get", "list", "watch", "create", "update", "patch"}, + ) +} + +func bindServiceAccountToConfigMapDeleter(saNamespace, saName, targetNamespace string) { + bindServiceAccountToNamespacedResource( + saNamespace, + saName, + targetNamespace, + []string{"configmaps"}, + []string{"get", "list", "watch", "delete"}, + ) +} + +func ensureServiceAccount(namespace, name string) { + ctx := context.Background() + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + + Eventually(func() error { + current := &corev1.ServiceAccount{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, current) + if apierrors.IsNotFound(err) { + return k8sClient.Create(ctx, sa) + } + return err + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} diff --git a/e2e/resourcepoolclaim_test.go b/e2e/resourcepoolclaim_test.go deleted file mode 100644 index d9fe7c55..00000000 --- a/e2e/resourcepoolclaim_test.go +++ /dev/null @@ -1,797 +0,0 @@ -// Copyright 2020-2023 Project Capsule Authors. -// SPDX-License-Identifier: Apache-2.0 - -package e2e - -import ( - "context" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" - "github.com/projectcapsule/capsule/pkg/api/meta" - "github.com/projectcapsule/capsule/pkg/runtime/selectors" -) - -var _ = Describe("ResourcePoolClaim Tests", Label("resourcepool"), func() { - _ = &capsulev1beta2.Tenant{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-claims-1", - Labels: map[string]string{ - "e2e-resourcepoolclaims": "test", - }, - }, - Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ - { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "wind-user", - Kind: "User", - }, - }, - }, - }, - }, - } - - JustAfterEach(func() { - Eventually(func() error { - poolList := &capsulev1beta2.TenantList{} - labelSelector := client.MatchingLabels{"e2e-resourcepoolclaims": "test"} - if err := k8sClient.List(context.TODO(), poolList, labelSelector); err != nil { - return err - } - - for _, pool := range poolList.Items { - if err := k8sClient.Delete(context.TODO(), &pool); err != nil { - return err - } - } - - return nil - }, "30s", "5s").Should(Succeed()) - - Eventually(func() error { - poolList := &capsulev1beta2.ResourcePoolList{} - labelSelector := client.MatchingLabels{"e2e-resourcepoolclaims": "test"} - if err := k8sClient.List(context.TODO(), poolList, labelSelector); err != nil { - return err - } - - for _, pool := range poolList.Items { - if err := k8sClient.Delete(context.TODO(), &pool); err != nil { - return err - } - } - - return nil - }, "30s", "5s").Should(Succeed()) - - Eventually(func() error { - poolList := &corev1.NamespaceList{} - labelSelector := client.MatchingLabels{"e2e-resourcepoolclaims": "test"} - if err := k8sClient.List(context.TODO(), poolList, labelSelector); err != nil { - return err - } - - for _, pool := range poolList.Items { - if err := k8sClient.Delete(context.TODO(), &pool); err != nil { - return err - } - } - - return nil - }, "30s", "5s").Should(Succeed()) - - }) - - It("Claim to Pool Assignment", func() { - pool1 := &capsulev1beta2.ResourcePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-binding-claims", - Labels: map[string]string{ - "e2e-resourcepoolclaims": "test", - }, - }, - Spec: capsulev1beta2.ResourcePoolSpec{ - Selectors: []selectors.NamespaceSelector{ - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "claims-bindings", - }, - }, - }, - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "claims-bindings-2", - }, - }, - }, - }, - Quota: corev1.ResourceQuotaSpec{ - Hard: corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("2"), - corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), - corev1.ResourceRequestsCPU: resource.MustParse("2"), - corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), - }, - }, - }, - } - - claim1 := &capsulev1beta2.ResourcePoolClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "assign-pool-claim-1", - Namespace: "ns-1-pool-assign", - }, - Spec: capsulev1beta2.ResourcePoolClaimSpec{ - Pool: "test-binding-claims", - ResourceClaims: corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("0"), - corev1.ResourceLimitsMemory: resource.MustParse("0"), - corev1.ResourceRequestsCPU: resource.MustParse("0"), - corev1.ResourceRequestsMemory: resource.MustParse("0"), - }, - }, - } - - claim2 := &capsulev1beta2.ResourcePoolClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "assign-pool-claim-2", - Namespace: "ns-2-pool-assign", - }, - Spec: capsulev1beta2.ResourcePoolClaimSpec{ - Pool: "test-binding-claims", - ResourceClaims: corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("0"), - corev1.ResourceLimitsMemory: resource.MustParse("0"), - corev1.ResourceRequestsCPU: resource.MustParse("0"), - corev1.ResourceRequestsMemory: resource.MustParse("0"), - }, - }, - } - - By("Create the ResourcePool", func() { - err := k8sClient.Create(context.TODO(), pool1) - Expect(err).Should(Succeed(), "Failed to create ResourcePool %s", pool1) - }) - - By("Get Applied revision", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool1.Name}, pool1) - Expect(err).Should(Succeed()) - }) - - By("Create Namespaces, which are selected by the pool", func() { - ns1 := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ns-1-pool-assign", - Labels: map[string]string{ - "e2e-resourcepoolclaims": "test", - "capsule.clastix.io/tenant": "claims-bindings", - }, - }, - } - - err := k8sClient.Create(context.TODO(), ns1) - Expect(err).Should(Succeed()) - - ns2 := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ns-2-pool-assign", - Labels: map[string]string{ - "e2e-resourcepoolclaims": "test", - "capsule.clastix.io/tenant": "claims-bindings-2", - }, - }, - } - - err = k8sClient.Create(context.TODO(), ns2) - Expect(err).Should(Succeed()) - - ns3 := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ns-3-pool-assign", - Labels: map[string]string{ - "e2e-resourcepoolclaims": "test", - "capsule.clastix.io/tenant": "something-else", - }, - }, - } - - err = k8sClient.Create(context.TODO(), ns3) - Expect(err).Should(Succeed()) - }) - - By("Verify Namespaces are shown as allowed targets", func() { - expectedNamespaces := []string{"ns-1-pool-assign", "ns-2-pool-assign"} - - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool1.Name}, pool1) - Expect(err).Should(Succeed()) - - Expect(pool1.Status.Namespaces).To(Equal(expectedNamespaces)) - Expect(pool1.Status.NamespaceSize).To(Equal(uint(2))) - }) - - By("Create a first claim and verify binding", func() { - - err := k8sClient.Create(context.TODO(), claim1) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim1) - - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim1.Name, Namespace: claim1.Namespace}, claim1) - Expect(err).Should(Succeed()) - - isSuccessfullyBoundAndUnsedToPool(pool1, claim1) - }) - - By("Create a second claim and verify binding", func() { - err := k8sClient.Create(context.TODO(), claim2) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim2) - - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim2.Name, Namespace: claim2.Namespace}, claim2) - Expect(err).Should(Succeed()) - - isSuccessfullyBoundAndUnsedToPool(pool1, claim2) - }) - - By("Create a third claim and verify error", func() { - claim := &capsulev1beta2.ResourcePoolClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "assign-pool-claim-3", - Namespace: "ns-3-pool-assign", - }, - Spec: capsulev1beta2.ResourcePoolClaimSpec{ - Pool: "test-binding-claims", - ResourceClaims: corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("0"), - corev1.ResourceLimitsMemory: resource.MustParse("0"), - corev1.ResourceRequestsCPU: resource.MustParse("0"), - corev1.ResourceRequestsMemory: resource.MustParse("0"), - }, - }, - } - - err := k8sClient.Create(context.TODO(), claim) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) - - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - expectedPool := meta.LocalRFC1123ObjectReferenceWithUID{} - Expect(claim.Status.Pool).To(Equal(expectedPool), "expected pool name to be empty") - - Expect(len(claim.Status.Conditions)).To(Equal(1), "expected single condition") - Expect(len(claim.OwnerReferences)).To(Equal(0), "expected no ownerreferences") - assigned := claim.Status.Conditions.GetConditionByType(meta.ReadyCondition) - Expect(assigned.Status).To(Equal(metav1.ConditionFalse), "failed to verify condition status") - Expect(assigned.Type).To(Equal(meta.ReadyCondition), "failed to verify condition type") - Expect(assigned.Reason).To(Equal(meta.FailedReason), "failed to verify condition reason") - }) - }) - - It("Admission (Validation) - Patch Guard", Label("skip-on-openshift"), func() { - pool := &capsulev1beta2.ResourcePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-admission-claims", - Labels: map[string]string{ - "e2e-resourcepoolclaims": "test", - }, - }, - Spec: capsulev1beta2.ResourcePoolSpec{ - Config: capsulev1beta2.ResourcePoolSpecConfiguration{ - DeleteBoundResources: ptr.To(false), - }, - Selectors: []selectors.NamespaceSelector{ - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "admission-guards", - }, - }, - }, - }, - Quota: corev1.ResourceQuotaSpec{ - Hard: corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("2"), - corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), - corev1.ResourceRequestsCPU: resource.MustParse("2"), - corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), - }, - }, - }, - } - - claim := &capsulev1beta2.ResourcePoolClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "admission-pool-claim-1", - Namespace: "ns-1-pool-admission", - }, - Spec: capsulev1beta2.ResourcePoolClaimSpec{ - Pool: pool.GetName(), - ResourceClaims: corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("1"), - corev1.ResourceLimitsMemory: resource.MustParse("1Gi"), - corev1.ResourceRequestsCPU: resource.MustParse("1"), - corev1.ResourceRequestsMemory: resource.MustParse("1Gi"), - }, - }, - } - - By("Create the Claim", func() { - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: claim.Namespace, - Labels: map[string]string{ - "e2e-resourcepoolclaims": "test", - "capsule.clastix.io/tenant": "admission-guards", - }, - }, - } - - err := k8sClient.Create(context.TODO(), ns) - Expect(err).Should(Succeed()) - - err = k8sClient.Create(context.TODO(), claim) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) - }) - - By("Create the ResourcePool", func() { - err := k8sClient.Create(context.TODO(), pool) - Expect(err).Should(Succeed(), "Failed to create ResourcePool %s", pool) - }) - - By("Get Applied revision", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: pool.Name}, pool) - Expect(err).Should(Succeed()) - }) - - By("Bind a claim", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - expectedPool := meta.LocalRFC1123ObjectReferenceWithUID{ - Name: meta.RFC1123Name(pool.Name), - UID: pool.GetUID(), - } - - isBoundAndUnusedCondition(claim) - Expect(claim.Status.Pool).To(Equal(expectedPool), "expected pool name to match") - }) - - By("Create a pod with resource requests/limits", func() { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "claim-pod", - Namespace: claim.Namespace, - Labels: map[string]string{ - "e2e": "claim-pod", - }, - }, - Spec: corev1.PodSpec{ - // optional: helps schedule quickly, avoid restarts - RestartPolicy: corev1.RestartPolicyNever, - Containers: []corev1.Container{ - { - Name: "pause", - Image: "registry.k8s.io/pause:3.9", - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("10m"), - corev1.ResourceMemory: resource.MustParse("16Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("20m"), - corev1.ResourceMemory: resource.MustParse("32Mi"), - }, - }, - }, - }, - }, - } - - Expect(k8sClient.Create(context.TODO(), pod)).To(Succeed()) - }) - - By("Verify the claim is used", func() { - time.Sleep(250 * time.Millisecond) - - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - isBoundAndUsedCondition(claim) - - err = k8sClient.Delete(context.TODO(), claim) - Expect(err).ShouldNot(Succeed()) - }) - - By("Error on patching resources for claim (Increase)", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - claim.Spec.ResourceClaims = corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("2"), - corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), - corev1.ResourceRequestsCPU: resource.MustParse("2"), - corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), - } - - err = k8sClient.Update(context.TODO(), claim) - Expect(err).ShouldNot(Succeed(), "Expected error when updating resources in bound state %s", claim) - }) - - By("Error on patching resources for claim (Decrease)", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - claim.Spec.ResourceClaims = corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("0"), - corev1.ResourceLimitsMemory: resource.MustParse("0Gi"), - corev1.ResourceRequestsCPU: resource.MustParse("0"), - corev1.ResourceRequestsMemory: resource.MustParse("0Gi"), - } - - err = k8sClient.Update(context.TODO(), claim) - Expect(err).ShouldNot(Succeed(), "Expected error when updating resources in bound state %s", claim) - }) - - By("Error on patching pool name", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - claim.Spec.Pool = "some-random-pool" - - err = k8sClient.Update(context.TODO(), claim) - Expect(err).ShouldNot(Succeed(), "Expected error when updating resources in bound state %s", claim) - }) - - By("Make the claim unused", func() { - key := client.ObjectKey{Name: "claim-pod", Namespace: claim.Namespace} - - pod := &corev1.Pod{} - err := k8sClient.Get(context.TODO(), key, pod) - Expect(err).To(Succeed(), "pod must exist before deleting") - - Expect(k8sClient.Delete(context.TODO(), pod, &client.DeleteOptions{ - GracePeriodSeconds: ptr.To(int64(0)), - })).To(Succeed()) - - Eventually(func() bool { - p := &corev1.Pod{} - err := k8sClient.Get( - context.TODO(), - key, - p, - ) - return apierrors.IsNotFound(err) - }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) - - }) - - By("Bind a claim", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - expectedPool := meta.LocalRFC1123ObjectReferenceWithUID{ - Name: meta.RFC1123Name(pool.Name), - UID: pool.GetUID(), - } - - isBoundAndUnusedCondition(claim) - Expect(claim.Status.Pool).To(Equal(expectedPool), "expected pool name to match") - }) - - By("Allow on patching resources for claim (Increase)", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - claim.Spec.ResourceClaims = corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("2"), - corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), - corev1.ResourceRequestsCPU: resource.MustParse("2"), - corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), - } - - err = k8sClient.Update(context.TODO(), claim) - Expect(err).Should(Succeed(), "Expected error when updating resources in bound state %s", claim) - }) - - By("Allow on patching resources for claim (Decrease)", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - claim.Spec.ResourceClaims = corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("0"), - corev1.ResourceLimitsMemory: resource.MustParse("0Gi"), - corev1.ResourceRequestsCPU: resource.MustParse("0"), - corev1.ResourceRequestsMemory: resource.MustParse("0Gi"), - } - - err = k8sClient.Update(context.TODO(), claim) - Expect(err).Should(Succeed(), "Expected error when updating resources in bound state %s", claim) - }) - - By("Allow on patching pool name", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - claim.Spec.Pool = "some-random-pool" - - err = k8sClient.Update(context.TODO(), claim) - Expect(err).Should(Succeed(), "Expected error when updating resources in bound state %s", claim) - }) - - By("Delete Pool", func() { - err := k8sClient.Delete(context.TODO(), pool) - Expect(err).Should(Succeed()) - }) - - By("Verify claim is no longer bound", func() { - isUnassignedCondition(claim) - }) - - By("Allow patching resources for claim (Increase)", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - claim.Spec.ResourceClaims = corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("2"), - corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), - corev1.ResourceRequestsCPU: resource.MustParse("2"), - corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), - } - - err = k8sClient.Update(context.TODO(), claim) - Expect(err).Should(Succeed(), "Expected error when updating resources in bound state %s", claim) - }) - - By("Allow patching resources for claim (Decrease)", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - claim.Spec.ResourceClaims = corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("0"), - corev1.ResourceLimitsMemory: resource.MustParse("0Gi"), - corev1.ResourceRequestsCPU: resource.MustParse("0"), - corev1.ResourceRequestsMemory: resource.MustParse("0Gi"), - } - - err = k8sClient.Update(context.TODO(), claim) - Expect(err).Should(Succeed(), "Expected error when updating resources in bound state %s", claim) - }) - - By("Allow patching pool name", func() { - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - claim.Spec.Pool = "some-random-pool" - - err = k8sClient.Update(context.TODO(), claim) - Expect(err).Should(Succeed(), "Expected no error when updating resources in bound state %s", claim) - }) - - }) - - It("Admission (Mutation) - Auto Pool Assign", Label("skip-on-openshift"), func() { - pool1 := &capsulev1beta2.ResourcePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-auto-assign-1", - Labels: map[string]string{ - "e2e-resourcepoolclaims": "test", - }, - }, - Spec: capsulev1beta2.ResourcePoolSpec{ - Config: capsulev1beta2.ResourcePoolSpecConfiguration{ - DeleteBoundResources: ptr.To(false), - }, - Selectors: []selectors.NamespaceSelector{ - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "admission-auto-assign", - }, - }, - }, - }, - Quota: corev1.ResourceQuotaSpec{ - Hard: corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("2"), - corev1.ResourceRequestsCPU: resource.MustParse("2"), - }, - }, - }, - } - - pool2 := &capsulev1beta2.ResourcePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-auto-assign-2", - Labels: map[string]string{ - "e2e-resourcepoolclaims": "test", - }, - }, - Spec: capsulev1beta2.ResourcePoolSpec{ - Config: capsulev1beta2.ResourcePoolSpecConfiguration{ - DeleteBoundResources: ptr.To(false), - }, - Selectors: []selectors.NamespaceSelector{ - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "capsule.clastix.io/tenant": "admission-auto-assign", - }, - }, - }, - }, - Quota: corev1.ResourceQuotaSpec{ - Hard: corev1.ResourceList{ - corev1.ResourceLimitsMemory: resource.MustParse("2"), - corev1.ResourceRequestsMemory: resource.MustParse("2"), - }, - }, - }, - } - - By("Create the ResourcePools", func() { - err := k8sClient.Create(context.TODO(), pool1) - Expect(err).Should(Succeed(), "Failed to create ResourcePool %s", pool1) - - err = k8sClient.Create(context.TODO(), pool2) - Expect(err).Should(Succeed(), "Failed to create ResourcePool %s", pool2) - }) - - By("Auto Assign Claim (CPU)", func() { - claim := &capsulev1beta2.ResourcePoolClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "auto-assign-1", - Namespace: "ns-1-pool-assign", - }, - Spec: capsulev1beta2.ResourcePoolClaimSpec{ - ResourceClaims: corev1.ResourceList{ - corev1.ResourceLimitsCPU: resource.MustParse("1"), - corev1.ResourceRequestsCPU: resource.MustParse("1"), - }, - }, - } - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: claim.Namespace, - Labels: map[string]string{ - "e2e-resourcepoolclaims": "test", - "capsule.clastix.io/tenant": "admission-auto-assign", - }, - }, - } - - err := k8sClient.Create(context.TODO(), ns) - Expect(err).Should(Succeed()) - - err = k8sClient.Create(context.TODO(), claim) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) - - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - Expect(claim.Spec.Pool).To(Equal(pool1.Name), "expected pool name to match") - }) - - By("Auto Assign Claim (Memory)", func() { - claim := &capsulev1beta2.ResourcePoolClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "auto-assign-1", - Namespace: "ns-2-pool-assign", - }, - Spec: capsulev1beta2.ResourcePoolClaimSpec{ - ResourceClaims: corev1.ResourceList{ - corev1.ResourceLimitsMemory: resource.MustParse("1"), - corev1.ResourceRequestsMemory: resource.MustParse("1"), - }, - }, - } - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: claim.Namespace, - Labels: map[string]string{ - "e2e-resourcepoolclaims": "test", - "capsule.clastix.io/tenant": "admission-auto-assign", - }, - }, - } - - err := k8sClient.Create(context.TODO(), ns) - Expect(err).Should(Succeed()) - - err = k8sClient.Create(context.TODO(), claim) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) - - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - Expect(claim.Spec.Pool).To(Equal(pool2.Name), "expected pool name to match") - }) - - By("No Default available (Storage)", func() { - claim := &capsulev1beta2.ResourcePoolClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "auto-assign-3", - Namespace: "ns-3-pool-assign", - }, - Spec: capsulev1beta2.ResourcePoolClaimSpec{ - ResourceClaims: corev1.ResourceList{ - corev1.ResourceRequestsStorage: resource.MustParse("1"), - }, - }, - } - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: claim.Namespace, - Labels: map[string]string{ - "e2e-resourcepoolclaims": "test", - "capsule.clastix.io/tenant": "admission-auto-assign", - }, - }, - } - - err := k8sClient.Create(context.TODO(), ns) - Expect(err).Should(Succeed()) - - err = k8sClient.Create(context.TODO(), claim) - Expect(err).Should(Succeed(), "Failed to create Claim %s", claim) - - err = k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, claim) - Expect(err).Should(Succeed()) - - Expect(claim.Spec.Pool).To(Equal(""), "expected pool name to match") - }) - - }) - -}) - -func isUnassignedCondition(claim *capsulev1beta2.ResourcePoolClaim) { - cl := &capsulev1beta2.ResourcePoolClaim{} - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, cl) - Expect(err).Should(Succeed()) - - assigned := cl.Status.Conditions.GetConditionByType(meta.ReadyCondition) - - Expect(assigned.Status).To(Equal(metav1.ConditionFalse), "failed to verify condition status") - Expect(assigned.Type).To(Equal(meta.ReadyCondition), "failed to verify condition type") - Expect(assigned.Reason).To(Equal(meta.FailedReason), "failed to verify condition reason") -} - -func isBoundAndUnusedCondition(claim *capsulev1beta2.ResourcePoolClaim) { - cl := &capsulev1beta2.ResourcePoolClaim{} - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, cl) - Expect(err).Should(Succeed()) - - bound := cl.Status.Conditions.GetConditionByType(meta.BoundCondition) - - Expect(bound.Type).To(Equal(meta.BoundCondition), "failed to verify condition type") - Expect(bound.Reason).To(Equal(meta.UnusedReason), "failed to verify condition reason") - Expect(bound.Status).To(Equal(metav1.ConditionFalse), "failed to verify condition status") -} - -func isBoundAndUsedCondition(claim *capsulev1beta2.ResourcePoolClaim) { - cl := &capsulev1beta2.ResourcePoolClaim{} - err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: claim.Name, Namespace: claim.Namespace}, cl) - Expect(err).Should(Succeed()) - - bound := cl.Status.Conditions.GetConditionByType(meta.BoundCondition) - - Expect(bound.Status).To(Equal(metav1.ConditionTrue), "failed to verify condition status") - Expect(bound.Type).To(Equal(meta.BoundCondition), "failed to verify condition type") - Expect(bound.Reason).To(Equal(meta.InUseReason), "failed to verify condition reason") -} diff --git a/e2e/rules_managed_test.go b/e2e/rules_managed_test.go index 04a3c1eb..5574386e 100644 --- a/e2e/rules_managed_test.go +++ b/e2e/rules_managed_test.go @@ -8,25 +8,30 @@ import ( . "github.com/onsi/gomega" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) -var _ = Describe("NamespaceStatus objects", Label("tenant", "rules"), func() { +var _ = Describe("NamespaceStatus objects", Ordered, Label("tenant", "rules", "status"), func() { ctx := context.Background() // Two tenants, each with one owner (reuse your existing ownerClient/NamespaceCreation helpers) tntA := &capsulev1beta2.Tenant{ - ObjectMeta: metav1.ObjectMeta{Name: "nsstatus-a"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-rule-status-a", + Labels: map[string]string{ + "env": "e2e", + }, + }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{Name: "matt", Kind: "User"}, + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{Name: "e2e-rule-status-a", Kind: "User"}, }, }, }, @@ -34,12 +39,17 @@ var _ = Describe("NamespaceStatus objects", Label("tenant", "rules"), func() { } tntB := &capsulev1beta2.Tenant{ - ObjectMeta: metav1.ObjectMeta{Name: "nsstatus-b"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-rule-status-b", + Labels: map[string]string{ + "env": "e2e", + }, + }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{Name: "matt", Kind: "User"}, + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{Name: "e2e-rule-status-b", Kind: "User"}, }, }, }, @@ -59,19 +69,22 @@ var _ = Describe("NamespaceStatus objects", Label("tenant", "rules"), func() { return k8sClient.Create(ctx, tntA) }).Should(Succeed()) + TenantReady(tntA, metav1.ConditionTrue, defaultTimeoutInterval) + EventuallyCreation(func() error { tntB.ResourceVersion = "" return k8sClient.Create(ctx, tntB) }).Should(Succeed()) - // Create namespaces for each tenant using your helper - nsA1 = NewNamespace("rule-status-ns1", map[string]string{ + TenantReady(tntB, metav1.ConditionTrue, defaultTimeoutInterval) + + nsA1 = NewNamespace("e2e-rule-status-a-1", map[string]string{ meta.TenantLabel: tntA.GetName(), }) - nsA2 = NewNamespace("rule-status-ns2", map[string]string{ + nsA2 = NewNamespace("e2e-rule-status-a-2", map[string]string{ meta.TenantLabel: tntA.GetName(), }) - nsB1 = NewNamespace("rule-status-ns3", map[string]string{ + nsB1 = NewNamespace("e2e-rule-status-b-1", map[string]string{ meta.TenantLabel: tntB.GetName(), }) @@ -80,8 +93,9 @@ var _ = Describe("NamespaceStatus objects", Label("tenant", "rules"), func() { NamespaceCreation(nsB1, tntB.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) // Wait until tenants list their namespaces (optional but makes debugging easier) - TenantNamespaceList(tntA, defaultTimeoutInterval).Should(ContainElements(nsA1.GetName(), nsA2.GetName())) - TenantNamespaceList(tntB, defaultTimeoutInterval).Should(ContainElement(nsB1.GetName())) + NamespaceIsPartOfTenant(tntA, nsA1).Should(Succeed()) + NamespaceIsPartOfTenant(tntA, nsA2).Should(Succeed()) + NamespaceIsPartOfTenant(tntB, nsB1).Should(Succeed()) }) JustAfterEach(func() { @@ -95,17 +109,15 @@ var _ = Describe("NamespaceStatus objects", Label("tenant", "rules"), func() { // Delete tenants if tntA != nil { - _ = k8sClient.Delete(ctx, tntA) + EventuallyDeletion(tntA) } if tntB != nil { - _ = k8sClient.Delete(ctx, tntB) + EventuallyDeletion(tntB) } }) - // --- Helpers --- - - expectNamespaceStatusFor := func(ns *corev1.Namespace, tenantName string) { - By(fmt.Sprintf("verifying NamespaceStatus for namespace %q (tenant=%q)", ns.Name, tenantName)) + expectNamespaceStatusFor := func(ns *corev1.Namespace, tenant *capsulev1beta2.Tenant) { + By(fmt.Sprintf("verifying NamespaceStatus for namespace %q (tenant=%q)", ns.Name, tenant.GetName())) Eventually(func(g Gomega) { // Re-read namespace to get UID reliably (in case local object is stale) @@ -120,29 +132,38 @@ var _ = Describe("NamespaceStatus objects", Label("tenant", "rules"), func() { var found bool for _, or := range nsStatus.OwnerReferences { - if or.APIVersion == "v1" && - or.Kind == "Namespace" && - or.Name == curNS.Name && - or.UID == curNS.UID { + if or.APIVersion == capsulev1beta2.GroupVersion.String() && + or.Kind == "Tenant" && + or.Name == tenant.Name && + or.UID == tenant.UID { found = true break } } - g.Expect(found).To(BeTrue(), "expected NamespaceStatus to have Namespace controller OwnerReference") + g.Expect(found).To(BeTrue(), "expected NamespaceStatus to have Tenant controller OwnerReference") }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } It("creates one NamespaceStatus per namespace, with correct Status.Tenant and Namespace controller OwnerReference", func() { - expectNamespaceStatusFor(nsA1, tntA.Name) - expectNamespaceStatusFor(nsA2, tntA.Name) - expectNamespaceStatusFor(nsB1, tntB.Name) + getTenantA := &capsulev1beta2.Tenant{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: tntA.GetName()}, getTenantA)).To(Succeed()) + + getTenantB := &capsulev1beta2.Tenant{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: tntB.GetName()}, getTenantB)).To(Succeed()) + + expectNamespaceStatusFor(nsA1, getTenantA) + expectNamespaceStatusFor(nsA2, getTenantA) + expectNamespaceStatusFor(nsB1, getTenantB) }) It("removes NamespaceStatus when the Namespace is deleted (ownerReference GC)", func() { + getTenantA := &capsulev1beta2.Tenant{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: tntA.GetName()}, getTenantA)).To(Succeed()) + // Ensure it exists first - expectNamespaceStatusFor(nsA1, tntA.Name) + expectNamespaceStatusFor(nsA1, getTenantA) // Delete namespace Expect(k8sClient.Delete(ctx, nsA1)).To(Succeed()) diff --git a/e2e/rules_registry_test.go b/e2e/rules_registry_test.go index 2c48a62d..7253a97a 100644 --- a/e2e/rules_registry_test.go +++ b/e2e/rules_registry_test.go @@ -20,30 +20,32 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "images", "registry"), func() { - originConfig := &capsulev1beta2.CapsuleConfiguration{} - +var _ = Describe("enforcing a Container Registry", Ordered, Label("tenant", "rules", "images", "registry"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "container-registry", + Name: "e2e-rule-registry", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "matt", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-rules-registry", Kind: "User", }, }, }, }, - Rules: []*capsulev1beta2.NamespaceRule{ + Rules: []*api.NamespaceRuleBodyTenant{ { - NamespaceRuleBody: capsulev1beta2.NamespaceRuleBody{ - Enforce: capsulev1beta2.NamespaceRuleEnforceBody{ + NamespaceRuleBodyNamespace: api.NamespaceRuleBodyNamespace{ + Enforce: api.NamespaceRuleEnforceBody{ Registries: []api.OCIRegistry{ // Global: allow any registry, but require PullPolicy Always (images+volumes) { @@ -72,8 +74,8 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima "environment": "prod", }, }, - NamespaceRuleBody: capsulev1beta2.NamespaceRuleBody{ - Enforce: capsulev1beta2.NamespaceRuleEnforceBody{ + NamespaceRuleBodyNamespace: api.NamespaceRuleBodyNamespace{ + Enforce: api.NamespaceRuleEnforceBody{ Registries: []api.OCIRegistry{ // Prod-only special-case { @@ -152,34 +154,26 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima } JustBeforeEach(func() { - Expect(k8sClient.Get(context.Background(), client.ObjectKey{Name: defaultConfigurationName}, originConfig)).To(Succeed()) - EventuallyCreation(func() error { tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) - - // Restore Configuration - Eventually(func() error { - c := &capsulev1beta2.CapsuleConfiguration{} - if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: originConfig.Name}, c); err != nil { - return err - } - c.Spec = originConfig.Spec - return k8sClient.Update(context.Background(), c) - }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + EventuallyDeletion(tnt) }) It("aggregates enforcement rules into NamespaceStatus for a non-prod namespace", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) // Non-prod: should include only the global rule body (two registries in order) expectNamespaceStatusRegistries(ns.GetName(), []string{ @@ -201,13 +195,14 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima It("aggregates enforcement rules into NamespaceStatus for a prod namespace", func() { ns := NewNamespace("", map[string]string{ - "environment": "prod", + "environment": "prod", + meta.TenantLabel: tnt.GetName(), }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) // Prod: should include global + prod rule (3 registries in order) expectNamespaceStatusRegistries(ns.GetName(), []string{ @@ -229,11 +224,15 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima }) It("denies a container image when pullPolicy is not explicitly set under restriction (dev)", func() { - ns := NewNamespace("") + ns := NewNamespace("", + map[string]string{ + meta.TenantLabel: tnt.GetName(), + }, + ) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) // No ImagePullPolicy set => "" => should be denied because global rule restricts policy to Always pod := &corev1.Pod{ @@ -253,20 +252,24 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima }) It("denies a harbor image with pullPolicy IfNotPresent because global Always must still apply (dev)", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "harbor-wrong-policy"}, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "c", Image: "harbor/some-team/app:1", ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -280,20 +283,24 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima }) It("allows a harbor image with pullPolicy Always (dev)", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "harbor-always"}, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "c", Image: "harbor/some-team/app:1", ImagePullPolicy: corev1.PullAlways, + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -303,27 +310,33 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima }) It("denies initContainers when they violate policy (dev) and includes the correct location in the message", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "init-deny"}, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), InitContainers: []corev1.Container{ { Name: "init", Image: "harbor/some-team/init:1", ImagePullPolicy: corev1.PullIfNotPresent, // should be denied + SecurityContext: restrictedContainerSecurityContext(), }, }, + Containers: []corev1.Container{ { Name: "c", Image: "harbor/some-team/app:1", ImagePullPolicy: corev1.PullAlways, + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -337,18 +350,25 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima }) It("denies volume image pullPolicy if not allowed (dev)", Label("skip-on-openshift"), func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "volume-deny"}, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ - // main container must exist - {Name: "c", Image: "harbor/some-team/app:1", ImagePullPolicy: corev1.PullAlways}, + { + Name: "c", + Image: "harbor/some-team/app:1", + ImagePullPolicy: corev1.PullAlways, + SecurityContext: restrictedContainerSecurityContext(), + }, }, Volumes: []corev1.Volume{ { @@ -373,20 +393,22 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima It("allows prod-specific image only with Always, still enforcing global policy", func() { ns := NewNamespace("", map[string]string{ - "environment": "prod", + "environment": "prod", + meta.TenantLabel: tnt.GetName(), }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) // Wrong policy => denied bad := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "prod-bad"}, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ - {Name: "c", Image: "harbor/production-image/app:1", ImagePullPolicy: corev1.PullNever}, + {Name: "c", Image: "harbor/production-image/app:1", ImagePullPolicy: corev1.PullNever, SecurityContext: restrictedContainerSecurityContext()}, }, }, } @@ -399,8 +421,9 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima good := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "prod-good"}, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ - {Name: "c", Image: "harbor/production-image/app:1", ImagePullPolicy: corev1.PullAlways}, + {Name: "c", Image: "harbor/production-image/app:1", ImagePullPolicy: corev1.PullAlways, SecurityContext: restrictedContainerSecurityContext()}, }, }, } @@ -408,11 +431,14 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima }) It("denies adding an ephemeral container with wrong pullPolicy on UPDATE", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + expectNamespaceStatusRegistries(ns.GetName(), []string{".*", "harbor/.*"}) cleanupRBAC := GrantEphemeralContainersUpdate(ns.Name, tnt.Spec.Owners[0].UserSpec.Name) @@ -422,8 +448,14 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "base"}, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ - {Name: "c", Image: "harbor/some-team/app:1", ImagePullPolicy: corev1.PullAlways}, + { + Name: "c", + Image: "harbor/some-team/app:1", + ImagePullPolicy: corev1.PullAlways, + SecurityContext: restrictedContainerSecurityContext(), + }, }, }, } @@ -435,6 +467,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima Name: "debug", Image: "harbor/some-team/debug:1", ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, } @@ -467,18 +500,22 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima }) It("denies a pod when volume image reference changes to a disallowed pullPolicy (recreate)", Label("skip-on-openshift"), func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + expectNamespaceStatusRegistries(ns.GetName(), []string{".*", "harbor/.*"}) pod1 := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "vol-ok"}, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ - {Name: "c", Image: "harbor/some-team/app:1", ImagePullPolicy: corev1.PullAlways}, + {Name: "c", Image: "harbor/some-team/app:1", ImagePullPolicy: corev1.PullAlways, SecurityContext: restrictedContainerSecurityContext()}, }, Volumes: []corev1.Volume{ { @@ -498,8 +535,9 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "rules", "ima pod2 := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "vol-bad"}, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ - {Name: "c", Image: "harbor/some-team/app:1", ImagePullPolicy: corev1.PullAlways}, + {Name: "c", Image: "harbor/some-team/app:1", ImagePullPolicy: corev1.PullAlways, SecurityContext: restrictedContainerSecurityContext()}, }, Volumes: []corev1.Volume{ { diff --git a/e2e/sa_prevent_privilege_escalation_test.go b/e2e/sa_prevent_privilege_escalation_test.go index 4ce733f7..73d1b4ae 100644 --- a/e2e/sa_prevent_privilege_escalation_test.go +++ b/e2e/sa_prevent_privilege_escalation_test.go @@ -18,20 +18,24 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("trying to escalate from a Tenant Namespace ServiceAccount", Label("tenant"), func() { +var _ = Describe("trying to escalate from a Tenant Namespace ServiceAccount", Ordered, Label("tenant"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "sa-privilege-escalation", + Name: "e2e-sa-privilege-escalation", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "mario", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-sa-escalation", Kind: "User", }, }, @@ -43,19 +47,21 @@ var _ = Describe("trying to escalate from a Tenant Namespace ServiceAccount", La }, } - ns := NewNamespace("attack") + ns := NewNamespace("e2e-sa-privilege-escalation-attack", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) JustBeforeEach(func() { EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) - + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should block Namespace changes", func() { diff --git a/e2e/sa_owner_promotion_test.go b/e2e/sa_promotion_owner_test.go similarity index 68% rename from e2e/sa_owner_promotion_test.go rename to e2e/sa_promotion_owner_test.go index 708e1f0b..c9127314 100644 --- a/e2e/sa_owner_promotion_test.go +++ b/e2e/sa_promotion_owner_test.go @@ -18,31 +18,37 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("Promoting ServiceAccounts to Owners", Label("config"), Label("promotion"), func() { +var _ = Describe("Promoting ServiceAccounts to Owners", Ordered, Label("config", "permissions", "owners", "promotion"), func() { originConfig := &capsulev1beta2.CapsuleConfiguration{} tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-owner-promotion", + Name: "e2e-tenant-owner-promotion", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Permissions: capsulev1beta2.Permissions{ + AllowOwnerPromotion: true, + }, + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "alice", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-sa-owner-promotion", Kind: "User", }, }, }, }, - AdditionalRoleBindings: []api.AdditionalRoleBindingsSpec{ + AdditionalRoleBindings: []rbac.AdditionalRoleBindingsSpec{ { - ClusterRoleName: "cluster-admin", + ClusterRoleName: "admin", Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", @@ -65,9 +71,11 @@ var _ = Describe("Promoting ServiceAccounts to Owners", Label("config"), Label(" tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) // Restore Configuration Eventually(func() error { @@ -86,9 +94,111 @@ var _ = Describe("Promoting ServiceAccounts to Owners", Label("config"), Label(" configuration.Spec.AllowServiceAccountPromotion = false }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - time.Sleep(250 * time.Millisecond) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + + // Create a ServiceAccount inside the tenant namespace + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sa", + Namespace: ns.Name, + }, + } + Expect(k8sClient.Create(context.TODO(), sa)).Should(Succeed()) + + // Table of personas: client + expected result + personas := map[string]struct { + client client.Client + matcher otypes.GomegaMatcher + }{ + "owner": {client: impersonationClient(tnt.Spec.Owners[0].Name, withDefaultGroups(make([]string, 0))), matcher: Not(Succeed())}, + "rb-user": {client: impersonationClient("bob", withDefaultGroups(make([]string, 0))), matcher: Not(Succeed())}, + "rb-sa": {client: impersonationClient("system:serviceaccount:"+sa.GetNamespace()+":default", withDefaultGroups(make([]string, 0))), matcher: Not(Succeed())}, + } + + for name, tc := range personas { + By(fmt.Sprintf("trying to promote SA as %s (Setting Trigger)", name)) + + Eventually(func() error { + saCopy := &corev1.ServiceAccount{} + Expect(tc.client.Get(context.TODO(), client.ObjectKeyFromObject(sa), saCopy)).To(Succeed()) + + if saCopy.Labels == nil { + saCopy.Labels = map[string]string{} + } + saCopy.Labels[meta.OwnerPromotionLabel] = meta.ValueTrue + + return tc.client.Update(context.TODO(), saCopy) + }, defaultTimeoutInterval, defaultPollInterval).Should(tc.matcher, "persona=%s", name) + } + + for name, tc := range personas { + By(fmt.Sprintf("trying to promote SA as %s (Setting Any Value)", name)) + + Eventually(func() error { + saCopy := &corev1.ServiceAccount{} + Expect(tc.client.Get(context.TODO(), client.ObjectKeyFromObject(sa), saCopy)).To(Succeed()) + + if saCopy.Labels == nil { + saCopy.Labels = map[string]string{} + } + saCopy.Labels[meta.OwnerPromotionLabel] = "false" + + return tc.client.Update(context.TODO(), saCopy) + }, defaultTimeoutInterval, defaultPollInterval).Should(tc.matcher, "persona=%s", name) + } + + for name, tc := range personas { + By(fmt.Sprintf("trying to allow deletion SA as %s (Setting Any Value)", name)) + + Eventually(func() error { + saCopy := &corev1.ServiceAccount{} + Expect(tc.client.Get(context.TODO(), client.ObjectKeyFromObject(sa), saCopy)).To(Succeed()) + + if saCopy.Labels == nil { + saCopy.Labels = map[string]string{} + } + saCopy.Labels[meta.OwnerPromotionLabel] = "false" + + return tc.client.Update(context.TODO(), saCopy) + }, defaultTimeoutInterval, defaultPollInterval).Should(tc.matcher, "persona=%s", name) + } + + for name, tc := range personas { + By(fmt.Sprintf("trying to allow deletion SA as %s (Setting Any Value)", name)) + + Eventually(func() error { + saCopy := &corev1.ServiceAccount{} + Expect(tc.client.Get(context.TODO(), client.ObjectKeyFromObject(sa), saCopy)).To(Succeed()) + + if saCopy.Labels == nil { + saCopy.Labels = map[string]string{} + } + saCopy.Labels[meta.OwnerPromotionLabel] = "false" + + return tc.client.Update(context.TODO(), saCopy) + }, defaultTimeoutInterval, defaultPollInterval).Should(tc.matcher, "persona=%s", name) + } + }) + + It("Deny Owner promotion even when feature is disabled on tenant", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.AllowServiceAccountPromotion = true + }) + + t := &capsulev1beta2.Tenant{} + Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).To(Succeed()) + t.Spec.Permissions.AllowOwnerPromotion = false + Expect(k8sClient.Update(context.TODO(), t)).To(Succeed()) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) // Create a ServiceAccount inside the tenant namespace sa := &corev1.ServiceAccount{ @@ -179,9 +289,11 @@ var _ = Describe("Promoting ServiceAccounts to Owners", Label("config"), Label(" configuration.Spec.AllowServiceAccountPromotion = true }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - time.Sleep(250 * time.Millisecond) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) // Create a ServiceAccount inside the tenant namespace sa := &corev1.ServiceAccount{ @@ -207,7 +319,10 @@ var _ = Describe("Promoting ServiceAccounts to Owners", Label("config"), Label(" Eventually(func() error { saCopy := &corev1.ServiceAccount{} - Expect(tc.client.Get(context.TODO(), client.ObjectKeyFromObject(sa), saCopy)).To(Succeed()) + err := tc.client.Get(context.TODO(), client.ObjectKeyFromObject(sa), saCopy) + if err != nil { + return err + } if saCopy.Labels == nil { saCopy.Labels = map[string]string{} @@ -240,9 +355,11 @@ var _ = Describe("Promoting ServiceAccounts to Owners", Label("config"), Label(" configuration.Spec.AllowServiceAccountPromotion = true }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - time.Sleep(250 * time.Millisecond) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) // Create a ServiceAccount inside the tenant namespace sa := &corev1.ServiceAccount{ @@ -297,11 +414,11 @@ var _ = Describe("Promoting ServiceAccounts to Owners", Label("config"), Label(" ) By("preventing the service account from deleting the namespace", func() { - newNs := NewNamespace("") + newNs := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) Expect(saClient.Create(context.TODO(), newNs)).To(Succeed()) - - TenantNamespaceList(tnt, defaultTimeoutInterval). - Should(ContainElements(ns.GetName(), newNs.GetName())) + NamespaceIsPartOfTenant(tnt, newNs).Should(Succeed()) Eventually(func(g Gomega) { // Deletion should eventually be forbidden / fail @@ -347,14 +464,14 @@ var _ = Describe("Promoting ServiceAccounts to Owners", Label("config"), Label(" Namespace: ns.Name, })), "expected ServiceAccount test-sa not to be present in CRB subjects") - time.Sleep(250 * time.Millisecond) - - secondNs := NewNamespace("") + secondNs := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) Eventually(func() error { return saClient.Create(context.TODO(), secondNs) }, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(Not(ContainElements(secondNs.GetName()))) + NamespaceIsNotPartOfTenant(tnt, secondNs).Should(Succeed()) Expect(saClient.Delete(context.TODO(), secondNs)).To(Not(Succeed())) diff --git a/e2e/sa_promotion_test.go b/e2e/sa_promotion_test.go new file mode 100644 index 00000000..fa2bd9fa --- /dev/null +++ b/e2e/sa_promotion_test.go @@ -0,0 +1,930 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + "sort" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + otypes "github.com/onsi/gomega/types" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" +) + +var serviceAccountPromotionClusterRoles = []string{ + "prod-view", + "prod-edit", + "dev-view", +} + +func expectPromotionTargets( + tenantName string, + serviceAccount *corev1.ServiceAccount, + clusterRoles []string, + targets []string, +) { + expectedClusterRoles := append([]string(nil), clusterRoles...) + expectedTargets := append([]string(nil), targets...) + + sort.Strings(expectedClusterRoles) + sort.Strings(expectedTargets) + + Eventually(func(g Gomega) rbac.PromotionStatusListSpec { + t := &capsulev1beta2.Tenant{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tenantName}, t) + g.Expect(err).NotTo(HaveOccurred()) + + return t.Status.Promotions + }, defaultTimeoutInterval, defaultPollInterval).Should( + ContainElement(rbac.PromotionSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, + Name: "system:serviceaccount:" + serviceAccount.GetNamespace() + ":" + serviceAccount.GetName(), + }, + ClusterRoles: expectedClusterRoles, + Targets: expectedTargets, + }), + "expected promotion for ServiceAccount %s/%s with clusterRoles=%v targets=%v", + serviceAccount.Namespace, + serviceAccount.Name, + expectedClusterRoles, + expectedTargets, + ) +} + +func expectNoPromotionTargets( + tenantName string, + serviceAccount *corev1.ServiceAccount, + clusterRoles []string, + targets []string, +) { + expectedClusterRoles := append([]string(nil), clusterRoles...) + expectedTargets := append([]string(nil), targets...) + + sort.Strings(expectedClusterRoles) + sort.Strings(expectedTargets) + + Consistently(func(g Gomega) rbac.PromotionStatusListSpec { + t := &capsulev1beta2.Tenant{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tenantName}, t) + g.Expect(err).NotTo(HaveOccurred()) + + return t.Status.Promotions + }, 2*time.Second, defaultPollInterval).ShouldNot( + ContainElement(rbac.PromotionSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, + Name: "system:serviceaccount:" + serviceAccount.GetNamespace() + ":" + serviceAccount.GetName(), + }, + ClusterRoles: expectedClusterRoles, + Targets: expectedTargets, + }), + "did not expect promotion for ServiceAccount %s/%s with clusterRoles=%v targets=%v", + serviceAccount.Namespace, + serviceAccount.Name, + expectedClusterRoles, + expectedTargets, + ) +} + +func containPromotion( + serviceAccount *corev1.ServiceAccount, + clusterRoles []string, + targets []string, +) otypes.GomegaMatcher { + return WithTransform(func(promotions rbac.PromotionStatusListSpec) bool { + expectedName := "system:serviceaccount:" + serviceAccount.GetNamespace() + ":" + serviceAccount.GetName() + + for _, promotion := range promotions { + if promotion.Kind != rbac.ServiceAccountOwner { + continue + } + + if promotion.Name != expectedName { + continue + } + + if !sameStringSet(promotion.ClusterRoles, clusterRoles) { + continue + } + + if !sameStringSet(promotion.Targets, targets) { + continue + } + + return true + } + + return false + }, BeTrue()) +} + +func sameStringSet(a, b []string) bool { + if len(a) != len(b) { + return false + } + + seen := map[string]int{} + + for _, item := range a { + seen[item]++ + } + + for _, item := range b { + seen[item]-- + } + + for _, count := range seen { + if count != 0 { + return false + } + } + + return true +} + +func promoteServiceAccount( + actor client.Client, + serviceAccount *corev1.ServiceAccount, + labels map[string]string, +) error { + saCopy := &corev1.ServiceAccount{} + if err := actor.Get(context.TODO(), client.ObjectKeyFromObject(serviceAccount), saCopy); err != nil { + return err + } + + if saCopy.Labels == nil { + saCopy.Labels = map[string]string{} + } + + saCopy.Labels[meta.ServiceAccountPromotionLabel] = meta.ValueTrue + + for key, value := range labels { + saCopy.Labels[key] = value + } + + return actor.Update(context.TODO(), saCopy) +} + +func setServiceAccountPromotionLabel( + actor client.Client, + serviceAccount *corev1.ServiceAccount, + value string, +) error { + saCopy := &corev1.ServiceAccount{} + if err := actor.Get(context.TODO(), client.ObjectKeyFromObject(serviceAccount), saCopy); err != nil { + return err + } + + if saCopy.Labels == nil { + saCopy.Labels = map[string]string{} + } + + saCopy.Labels[meta.ServiceAccountPromotionLabel] = value + + return actor.Update(context.TODO(), saCopy) +} + +func expectRoleBindingForPromotion( + namespace string, + clusterRole string, + serviceAccount *corev1.ServiceAccount, +) { + Eventually(func(g Gomega) { + roleBindings := &rbacv1.RoleBindingList{} + err := k8sClient.List(context.TODO(), roleBindings, client.InNamespace(namespace)) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(roleBindings.Items).To(ContainElement(SatisfyAll( + WithTransform(func(roleBinding rbacv1.RoleBinding) string { + return roleBinding.RoleRef.Kind + }, Equal("ClusterRole")), + WithTransform(func(roleBinding rbacv1.RoleBinding) string { + return roleBinding.RoleRef.Name + }, Equal(clusterRole)), + WithTransform(func(roleBinding rbacv1.RoleBinding) []rbacv1.Subject { + return roleBinding.Subjects + }, ContainElement(rbacv1.Subject{ + Kind: rbacv1.ServiceAccountKind, + Name: serviceAccount.Name, + Namespace: serviceAccount.Namespace, + })), + ))) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func expectNoRoleBindingForPromotion( + namespace string, + clusterRole string, + serviceAccount *corev1.ServiceAccount, +) { + Consistently(func(g Gomega) { + roleBindings := &rbacv1.RoleBindingList{} + err := k8sClient.List(context.TODO(), roleBindings, client.InNamespace(namespace)) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(roleBindings.Items).NotTo(ContainElement(SatisfyAll( + WithTransform(func(roleBinding rbacv1.RoleBinding) string { + return roleBinding.RoleRef.Kind + }, Equal("ClusterRole")), + WithTransform(func(roleBinding rbacv1.RoleBinding) string { + return roleBinding.RoleRef.Name + }, Equal(clusterRole)), + WithTransform(func(roleBinding rbacv1.RoleBinding) []rbacv1.Subject { + return roleBinding.Subjects + }, ContainElement(rbacv1.Subject{ + Kind: rbacv1.ServiceAccountKind, + Name: serviceAccount.Name, + Namespace: serviceAccount.Namespace, + })), + ))) + }, 2*time.Second, defaultPollInterval).Should(Succeed()) +} + +var _ = Describe("Promoting ServiceAccounts", Ordered, Label("config", "permissions", "promotion", "rbac"), func() { + originConfig := &capsulev1beta2.CapsuleConfiguration{} + + tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-sa-promotion", + Labels: map[string]string{ + "env": "e2e", + }, + }, + Spec: capsulev1beta2.TenantSpec{ + Rules: []*api.NamespaceRuleBodyTenant{ + { + Permissions: api.NamespaceRulePermissionBody{ + Promotions: []*api.NamespaceRulePromotionRule{ + { + ClusterRoles: []string{"view"}, + }, + { + ClusterRoles: []string{"edit"}, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "selective": "edit", + }, + }, + }, + }, + }, + }, + { + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "environment": "prod", + }, + }, + Permissions: api.NamespaceRulePermissionBody{ + Promotions: []*api.NamespaceRulePromotionRule{ + { + ClusterRoles: []string{"prod-view"}, + }, + { + ClusterRoles: []string{"prod-edit"}, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "super": "account", + }, + }, + }, + }, + }, + }, + { + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "environment": "dev", + }, + }, + Permissions: api.NamespaceRulePermissionBody{ + Promotions: []*api.NamespaceRulePromotionRule{ + { + ClusterRoles: []string{"dev-view"}, + }, + }, + }, + }, + }, + Owners: rbac.OwnerListSpec{ + { + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-sa-promotion", + Kind: "User", + }, + }, + }, + }, + AdditionalRoleBindings: []rbac.AdditionalRoleBindingsSpec{ + { + ClusterRoleName: "admin", + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "default", + }, + { + Kind: "User", + Name: "bob", + }, + }, + }, + }, + }, + } + + BeforeEach(func() { + for _, name := range serviceAccountPromotionClusterRoles { + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{ + "configmaps", + "secrets", + "serviceaccounts", + }, + Verbs: []string{ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + }, + }, + }, + } + + err := k8sClient.Create(context.TODO(), clusterRole) + if apierrors.IsAlreadyExists(err) { + continue + } + + Expect(err).NotTo(HaveOccurred()) + } + }) + + AfterEach(func() { + for _, name := range serviceAccountPromotionClusterRoles { + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + + err := k8sClient.Delete(context.TODO(), clusterRole) + if apierrors.IsNotFound(err) { + continue + } + + Expect(err).NotTo(HaveOccurred()) + } + }) + + JustBeforeEach(func() { + Expect(k8sClient.Get(context.Background(), client.ObjectKey{Name: defaultConfigurationName}, originConfig)).To(Succeed()) + + EventuallyCreation(func() error { + tnt.ResourceVersion = "" + return k8sClient.Create(context.TODO(), tnt) + }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) + }) + + JustAfterEach(func() { + EventuallyDeletion(tnt) + + Eventually(func() error { + c := &capsulev1beta2.CapsuleConfiguration{} + if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: originConfig.Name}, c); err != nil { + return err + } + + c.Spec = originConfig.Spec + + return k8sClient.Update(context.Background(), c) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + It("denies ServiceAccount promotion when the feature is globally disabled", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.AllowServiceAccountPromotion = false + }) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sa", + Namespace: ns.Name, + }, + } + Expect(k8sClient.Create(context.TODO(), sa)).Should(Succeed()) + + personas := map[string]struct { + client client.Client + matcher otypes.GomegaMatcher + }{ + "owner": { + client: impersonationClient(tnt.Spec.Owners[0].Name, withDefaultGroups(make([]string, 0))), + matcher: Not(Succeed()), + }, + "rb-user": { + client: impersonationClient("bob", withDefaultGroups(make([]string, 0))), + matcher: Not(Succeed()), + }, + "rb-sa": { + client: impersonationClient("system:serviceaccount:"+sa.GetNamespace()+":default", withDefaultGroups(make([]string, 0))), + matcher: Not(Succeed()), + }, + } + + for name, tc := range personas { + By(fmt.Sprintf("trying to promote ServiceAccount as %s", name)) + + Eventually(func() error { + return promoteServiceAccount(tc.client, sa, nil) + }, defaultTimeoutInterval, defaultPollInterval).Should(tc.matcher, "persona=%s", name) + } + + Eventually(func(g Gomega) { + t := &capsulev1beta2.Tenant{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(t.Status.Promotions).To(HaveLen(0), "expected no promotions to be present") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + for name, tc := range personas { + By(fmt.Sprintf("trying to set non-trigger promotion label as %s", name)) + + Eventually(func() error { + return setServiceAccountPromotionLabel(tc.client, sa, "false") + }, defaultTimeoutInterval, defaultPollInterval).Should(tc.matcher, "persona=%s", name) + } + + Eventually(func(g Gomega) { + t := &capsulev1beta2.Tenant{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(t.Status.Promotions).To(HaveLen(0), "expected no promotions to be present") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + It("allows ServiceAccount promotion only by tenant owners", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.AllowServiceAccountPromotion = true + }) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sa", + Namespace: ns.Name, + }, + } + Expect(k8sClient.Create(context.TODO(), sa)).Should(Succeed()) + + personas := map[string]struct { + client client.Client + matcher otypes.GomegaMatcher + }{ + "rb-user": { + client: impersonationClient("bob", withDefaultGroups(make([]string, 0))), + matcher: Not(Succeed()), + }, + "rb-sa": { + client: impersonationClient("system:serviceaccount:"+sa.GetNamespace()+":default", withDefaultGroups(make([]string, 0))), + matcher: Not(Succeed()), + }, + "owner": { + client: impersonationClient(tnt.Spec.Owners[0].Name, withDefaultGroups(make([]string, 0))), + matcher: Succeed(), + }, + } + + for name, tc := range personas { + By(fmt.Sprintf("trying to promote ServiceAccount as %s", name)) + + Eventually(func() error { + return promoteServiceAccount(tc.client, sa, nil) + }, defaultTimeoutInterval, defaultPollInterval).Should(tc.matcher, "persona=%s", name) + } + + expectPromotionTargets(tnt.GetName(), sa, []string{"view"}, []string{ns.Name}) + }) + It("does not apply namespace-scoped promotion rules to ServiceAccounts from non-matching source namespaces", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.AllowServiceAccountPromotion = true + }) + + dev := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "dev", + }) + NamespaceCreation(dev, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, dev).Should(Succeed()) + + test := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "test", + }) + NamespaceCreation(test, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, test).Should(Succeed()) + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dev-sa", + Namespace: dev.Name, + }, + } + Expect(k8sClient.Create(context.TODO(), sa)).Should(Succeed()) + + ownerClient := impersonationClient(tnt.Spec.Owners[0].Name, withDefaultGroups(make([]string, 0))) + + Eventually(func() error { + return promoteServiceAccount(ownerClient, sa, nil) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + expectPromotionTargets(tnt.GetName(), sa, []string{"view"}, []string{dev.Name, test.Name}) + expectPromotionTargets(tnt.GetName(), sa, []string{"dev-view"}, []string{dev.Name}) + }) + + It("promotes ServiceAccounts to all tenant namespaces for global promotion rules", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.AllowServiceAccountPromotion = true + }) + + dev := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "dev", + }) + NamespaceCreation(dev, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, dev).Should(Succeed()) + + prod := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "prod", + }) + NamespaceCreation(prod, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, prod).Should(Succeed()) + + stage := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "stage", + }) + NamespaceCreation(stage, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, stage).Should(Succeed()) + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sa", + Namespace: dev.Name, + }, + } + Expect(k8sClient.Create(context.TODO(), sa)).Should(Succeed()) + + ownerClient := impersonationClient(tnt.Spec.Owners[0].Name, withDefaultGroups(make([]string, 0))) + + Eventually(func() error { + return promoteServiceAccount(ownerClient, sa, nil) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + expectPromotionTargets(tnt.GetName(), sa, []string{"view"}, []string{dev.Name, prod.Name, stage.Name}) + }) + + It("promotes ServiceAccounts to all tenant namespaces for matching global selector rules", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.AllowServiceAccountPromotion = true + }) + + dev := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "dev", + }) + NamespaceCreation(dev, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, dev).Should(Succeed()) + + prod := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "prod", + }) + NamespaceCreation(prod, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, prod).Should(Succeed()) + + stage := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "stage", + }) + NamespaceCreation(stage, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, stage).Should(Succeed()) + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "selective-sa", + Namespace: dev.Name, + }, + } + Expect(k8sClient.Create(context.TODO(), sa)).Should(Succeed()) + + ownerClient := impersonationClient(tnt.Spec.Owners[0].Name, withDefaultGroups(make([]string, 0))) + + Eventually(func() error { + return promoteServiceAccount(ownerClient, sa, map[string]string{ + "selective": "edit", + }) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + expectPromotionTargets(tnt.GetName(), sa, []string{"view", "edit"}, []string{dev.Name, prod.Name, stage.Name}) + }) + + It("does not apply selector-based global promotion rules when the ServiceAccount labels do not match", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.AllowServiceAccountPromotion = true + }) + + dev := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "dev", + }) + NamespaceCreation(dev, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, dev).Should(Succeed()) + + prod := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "prod", + }) + NamespaceCreation(prod, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, prod).Should(Succeed()) + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "normal-sa", + Namespace: dev.Name, + }, + } + Expect(k8sClient.Create(context.TODO(), sa)).Should(Succeed()) + + ownerClient := impersonationClient(tnt.Spec.Owners[0].Name, withDefaultGroups(make([]string, 0))) + + Eventually(func() error { + return promoteServiceAccount(ownerClient, sa, nil) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + expectPromotionTargets(tnt.GetName(), sa, []string{"view"}, []string{dev.Name, prod.Name}) + expectNoPromotionTargets(tnt.GetName(), sa, []string{"edit"}, []string{dev.Name, prod.Name}) + }) + + It("promotes ServiceAccounts only to namespaces matching the namespace selector", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.AllowServiceAccountPromotion = true + }) + + dev := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "dev", + }) + NamespaceCreation(dev, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, dev).Should(Succeed()) + + prod := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "prod", + }) + NamespaceCreation(prod, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, prod).Should(Succeed()) + + stage := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "prod", + }) + NamespaceCreation(stage, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, stage).Should(Succeed()) + + saTest := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sa", + Namespace: dev.Name, + }, + } + Expect(k8sClient.Create(context.TODO(), saTest)).Should(Succeed()) + + saProd := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-prod", + Namespace: prod.Name, + }, + } + Expect(k8sClient.Create(context.TODO(), saProd)).Should(Succeed()) + + ownerClient := impersonationClient(tnt.Spec.Owners[0].Name, withDefaultGroups(make([]string, 0))) + + Eventually(func() error { + return promoteServiceAccount(ownerClient, saTest, nil) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Eventually(func() error { + return promoteServiceAccount(ownerClient, saProd, nil) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + expectPromotionTargets(tnt.GetName(), saTest, []string{"view"}, []string{dev.Name, prod.Name, stage.Name}) + expectPromotionTargets(tnt.GetName(), saTest, []string{"dev-view"}, []string{dev.Name}) + + expectPromotionTargets(tnt.GetName(), saProd, []string{"view"}, []string{dev.Name, prod.Name, stage.Name}) + expectPromotionTargets(tnt.GetName(), saProd, []string{"prod-view"}, []string{prod.Name, stage.Name}) + }) + + It("promotes ServiceAccounts by combined ServiceAccount selector and namespace selector", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.AllowServiceAccountPromotion = true + }) + + dev := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "dev", + }) + NamespaceCreation(dev, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, dev).Should(Succeed()) + + prodA := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "prod", + }) + NamespaceCreation(prodA, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, prodA).Should(Succeed()) + + prodB := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "prod", + }) + NamespaceCreation(prodB, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, prodB).Should(Succeed()) + + stage := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "stage", + }) + NamespaceCreation(stage, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + + saDev := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "super-sa", + Namespace: dev.Name, + }, + } + Expect(k8sClient.Create(context.TODO(), saDev)).Should(Succeed()) + + saProd := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "super-sa", + Namespace: prodA.Name, + }, + } + Expect(k8sClient.Create(context.TODO(), saProd)).Should(Succeed()) + + ownerClient := impersonationClient(tnt.Spec.Owners[0].Name, withDefaultGroups(make([]string, 0))) + + Eventually(func() error { + return promoteServiceAccount(ownerClient, saDev, map[string]string{ + "super": "account", + }) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Eventually(func() error { + return promoteServiceAccount(ownerClient, saProd, map[string]string{ + "super": "account", + }) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + expectPromotionTargets(tnt.GetName(), saDev, []string{"view"}, []string{dev.Name, prodA.Name, prodB.Name, stage.Name}) + expectPromotionTargets(tnt.GetName(), saDev, []string{"dev-view"}, []string{dev.Name}) + + expectPromotionTargets(tnt.GetName(), saProd, []string{"view"}, []string{dev.Name, prodA.Name, prodB.Name, stage.Name}) + expectPromotionTargets(tnt.GetName(), saProd, []string{"prod-edit", "prod-view"}, []string{prodA.Name, prodB.Name}) + }) + + It("does not apply combined selector promotion rules when the ServiceAccount selector does not match", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.AllowServiceAccountPromotion = true + }) + + dev := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "dev", + }) + NamespaceCreation(dev, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, dev).Should(Succeed()) + + prod := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "prod", + }) + NamespaceCreation(prod, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, prod).Should(Succeed()) + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "not-super-sa", + Namespace: prod.Name, + }, + } + Expect(k8sClient.Create(context.TODO(), sa)).Should(Succeed()) + + ownerClient := impersonationClient(tnt.Spec.Owners[0].Name, withDefaultGroups(make([]string, 0))) + + Eventually(func() error { + return promoteServiceAccount(ownerClient, sa, nil) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + expectPromotionTargets(tnt.GetName(), sa, []string{"view"}, []string{dev.Name, prod.Name}) + expectPromotionTargets(tnt.GetName(), sa, []string{"prod-view"}, []string{prod.Name}) + }) + + It("creates RoleBindings only in targeted namespaces", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.AllowServiceAccountPromotion = true + }) + + dev := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "dev", + }) + NamespaceCreation(dev, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, dev).Should(Succeed()) + + prod := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "prod", + }) + NamespaceCreation(prod, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, prod).Should(Succeed()) + + stage := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + "environment": "prod", + }) + NamespaceCreation(stage, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, stage).Should(Succeed()) + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "super-sa", + Namespace: prod.Name, + }, + } + Expect(k8sClient.Create(context.TODO(), sa)).Should(Succeed()) + + ownerClient := impersonationClient(tnt.Spec.Owners[0].Name, withDefaultGroups(make([]string, 0))) + + Eventually(func() error { + return promoteServiceAccount(ownerClient, sa, map[string]string{ + "super": "account", + }) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + expectPromotionTargets(tnt.GetName(), sa, []string{"view"}, []string{dev.Name, prod.Name, stage.Name}) + expectPromotionTargets(tnt.GetName(), sa, []string{"prod-view", "prod-edit"}, []string{prod.Name, stage.Name}) + + expectRoleBindingForPromotion(dev.Name, "view", sa) + expectRoleBindingForPromotion(prod.Name, "view", sa) + expectRoleBindingForPromotion(stage.Name, "view", sa) + + expectRoleBindingForPromotion(prod.Name, "prod-view", sa) + expectRoleBindingForPromotion(prod.Name, "prod-edit", sa) + + expectRoleBindingForPromotion(stage.Name, "prod-view", sa) + expectRoleBindingForPromotion(stage.Name, "prod-edit", sa) + }) +}) diff --git a/e2e/allowed_external_ips_test.go b/e2e/service_allowed_external_ips_test.go similarity index 79% rename from e2e/allowed_external_ips_test.go rename to e2e/service_allowed_external_ips_test.go index 6c756b0f..9df10d3b 100644 --- a/e2e/allowed_external_ips_test.go +++ b/e2e/service_allowed_external_ips_test.go @@ -14,19 +14,24 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("enforcing an allowed set of Service external IPs", Label("tenant"), func() { +var _ = Describe("enforcing an allowed set of Service external IPs", Ordered, Label("tenant", "networking", "service"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "allowed-external-ip", + Name: "e2e-allowed-external-ip", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "google", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-svc-external-svc", Kind: "User", }, }, @@ -48,13 +53,17 @@ var _ = Describe("enforcing an allowed set of Service external IPs", Label("tena tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should fail creating an evil service", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) svc := &corev1.Service{ @@ -87,8 +96,12 @@ var _ = Describe("enforcing an allowed set of Service external IPs", Label("tena }) It("should allow the first CIDR block", Label("skip-on-openshift"), func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -120,7 +133,9 @@ var _ = Describe("enforcing an allowed set of Service external IPs", Label("tena }) It("should allow the /32 CIDR block", Label("skip-on-openshift"), func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) svc := &corev1.Service{ diff --git a/e2e/disable_externalname_test.go b/e2e/service_disable_externalname_test.go similarity index 80% rename from e2e/disable_externalname_test.go rename to e2e/service_disable_externalname_test.go index ef5e38a0..65488912 100644 --- a/e2e/disable_externalname_test.go +++ b/e2e/service_disable_externalname_test.go @@ -15,19 +15,24 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating an ExternalName service when it is disabled for Tenant", Label("tenant"), func() { +var _ = Describe("creating an ExternalName service when it is disabled for Tenant", Ordered, Label("tenant", "networking", "service"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: "disable-external-service", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "google", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "disable-external-service", Kind: "User", }, }, @@ -46,15 +51,20 @@ var _ = Describe("creating an ExternalName service when it is disabled for Tenan tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should fail creating a service with ExternalService type", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) EventuallyCreation(func() error { svc := &corev1.Service{ diff --git a/e2e/disable_loadbalancer_test.go b/e2e/service_disable_loadbalancer_test.go similarity index 78% rename from e2e/disable_loadbalancer_test.go rename to e2e/service_disable_loadbalancer_test.go index 70df28dc..243d5edd 100644 --- a/e2e/disable_loadbalancer_test.go +++ b/e2e/service_disable_loadbalancer_test.go @@ -15,19 +15,24 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a LoadBalancer service when it is disabled for Tenant", Label("tenant"), func() { +var _ = Describe("creating a LoadBalancer service when it is disabled for Tenant", Ordered, Label("tenant", "networking", "service"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "disable-loadbalancer-service", + Name: "e2e-disable-loadbalancer-service", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "amazon", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-disable-loadbalancer-service", Kind: "User", }, }, @@ -45,16 +50,20 @@ var _ = Describe("creating a LoadBalancer service when it is disabled for Tenant EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should fail creating a service with LoadBalancer type", func() { - ns := NewNamespace("") - + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) EventuallyCreation(func() error { svc := &corev1.Service{ diff --git a/e2e/disable_node_ports_test.go b/e2e/service_disable_node_ports_test.go similarity index 73% rename from e2e/disable_node_ports_test.go rename to e2e/service_disable_node_ports_test.go index d70e14e4..721c594c 100644 --- a/e2e/disable_node_ports_test.go +++ b/e2e/service_disable_node_ports_test.go @@ -15,19 +15,24 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a nodePort service when it is disabled for Tenant", Label("tenant"), func() { +var _ = Describe("creating a nodePort service when it is disabled for Tenant", Ordered, Label("tenant", "networking", "service"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "disable-node-ports", + Name: "e2e-disable-node-ports", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "google", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-disable-node-ports", Kind: "User", }, }, @@ -46,14 +51,19 @@ var _ = Describe("creating a nodePort service when it is disabled for Tenant", L tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should fail creating a service with NodePort type", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ diff --git a/e2e/enable_loadbalancer_test.go b/e2e/service_enable_loadbalancer_test.go similarity index 73% rename from e2e/enable_loadbalancer_test.go rename to e2e/service_enable_loadbalancer_test.go index cb0fcca8..07d68a37 100644 --- a/e2e/enable_loadbalancer_test.go +++ b/e2e/service_enable_loadbalancer_test.go @@ -15,19 +15,24 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a LoadBalancer service when it is enabled for Tenant", Label("tenant"), func() { +var _ = Describe("creating a LoadBalancer service when it is enabled for Tenant", Ordered, Label("tenant", "networking", "service"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "enable-loadbalancer-service", + Name: "e2e-loadbalancer-service", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "netflix", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-loadbalancer-service", Kind: "User", }, }, @@ -45,16 +50,20 @@ var _ = Describe("creating a LoadBalancer service when it is enabled for Tenant" EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should succeed creating a service with LoadBalancer type", func() { - ns := NewNamespace("") - + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) EventuallyCreation(func() error { svc := &corev1.Service{ @@ -78,7 +87,6 @@ var _ = Describe("creating a LoadBalancer service when it is enabled for Tenant" } cs := ownerClient(tnt.Spec.Owners[0].UserSpec) - _, err := cs.CoreV1().Services(ns.Name).Create(context.Background(), svc, metav1.CreateOptions{}) return err diff --git a/e2e/service_forbidden_metadata_test.go b/e2e/service_forbidden_metadata_test.go index a36c3a02..2e39afb4 100644 --- a/e2e/service_forbidden_metadata_test.go +++ b/e2e/service_forbidden_metadata_test.go @@ -14,12 +14,17 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Service with user-specified labels and annotations", Label("tenant", "service"), func() { +var _ = Describe("creating a Service with user-specified labels and annotations", Ordered, Label("tenant", "networking", "service"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-user-metadata-forbidden", + Name: "e2e-service-user-metadata-forbidden", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ ServiceOptions: &api.ServiceOptions{ @@ -32,11 +37,11 @@ var _ = Describe("creating a Service with user-specified labels and annotations" Regex: "^gatsby-.*$", }, }, - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "gatsby", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-service-user-metadata-forbidden", Kind: "User", }, }, @@ -50,16 +55,20 @@ var _ = Describe("creating a Service with user-specified labels and annotations" tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should allow", func() { By("specifying non-forbidden labels", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) svc := NewService(types.NamespacedName{ Namespace: ns.GetName(), @@ -69,9 +78,11 @@ var _ = Describe("creating a Service with user-specified labels and annotations" ServiceCreation(svc, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) }) By("specifying non-forbidden annotations", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) svc := NewService(types.NamespacedName{ Namespace: ns.GetName(), @@ -84,9 +95,11 @@ var _ = Describe("creating a Service with user-specified labels and annotations" It("should fail when creating a Service", func() { By("specifying forbidden labels using exact match", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) svc := NewService(types.NamespacedName{ Namespace: ns.GetName(), @@ -96,9 +109,11 @@ var _ = Describe("creating a Service with user-specified labels and annotations" ServiceCreation(svc, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) }) By("specifying forbidden labels using regex match", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) svc := NewService(types.NamespacedName{ Namespace: ns.GetName(), @@ -108,9 +123,11 @@ var _ = Describe("creating a Service with user-specified labels and annotations" ServiceCreation(svc, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) }) By("specifying forbidden annotations using exact match", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) svc := NewService(types.NamespacedName{ Namespace: ns.GetName(), @@ -120,9 +137,11 @@ var _ = Describe("creating a Service with user-specified labels and annotations" ServiceCreation(svc, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) }) By("specifying forbidden annotations using regex match", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) svc := NewService(types.NamespacedName{ Namespace: ns.GetName(), @@ -137,9 +156,11 @@ var _ = Describe("creating a Service with user-specified labels and annotations" cs := ownerClient(tnt.Spec.Owners[0].UserSpec) By("specifying forbidden labels using exact match", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) svc := NewService(types.NamespacedName{ Namespace: ns.GetName(), @@ -159,9 +180,11 @@ var _ = Describe("creating a Service with user-specified labels and annotations" }, 10*time.Second, time.Second).ShouldNot(Succeed()) }) By("specifying forbidden labels using regex match", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) svc := NewService(types.NamespacedName{ Namespace: ns.GetName(), @@ -182,9 +205,11 @@ var _ = Describe("creating a Service with user-specified labels and annotations" }, 3*time.Second, time.Second).ShouldNot(Succeed()) }) By("specifying forbidden annotations using exact match", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) svc := NewService(types.NamespacedName{ Namespace: ns.GetName(), @@ -205,9 +230,11 @@ var _ = Describe("creating a Service with user-specified labels and annotations" }, 10*time.Second, time.Second).ShouldNot(Succeed()) }) By("specifying forbidden annotations using regex match", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) svc := NewService(types.NamespacedName{ Namespace: ns.GetName(), diff --git a/e2e/service_metadata_test.go b/e2e/service_metadata_test.go index 188090c9..69d0f3fd 100644 --- a/e2e/service_metadata_test.go +++ b/e2e/service_metadata_test.go @@ -20,20 +20,25 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" ) -var _ = Describe("adding metadata to Service objects", Label("tenant", "service"), func() { +var _ = Describe("adding metadata to Service objects", Ordered, Label("tenant", "networking", "service"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "service-metadata", + Name: "e2e-service-metadata", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "gatsby", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-service-metadata", Kind: "User", }, }, @@ -60,16 +65,20 @@ var _ = Describe("adding metadata to Service objects", Label("tenant", "service" tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should apply them to Service", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -150,9 +159,12 @@ var _ = Describe("adding metadata to Service objects", Label("tenant", "service" } } - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + // Waiting for the reconciliation of required RBAC EventuallyCreation(func() (err error) { pod := &corev1.Pod{ @@ -160,10 +172,12 @@ var _ = Describe("adding metadata to Service objects", Label("tenant", "service" Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "quay.io/google-containers/pause-amd64:3.0", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, diff --git a/e2e/enable_node_ports_test.go b/e2e/service_node_ports_test.go similarity index 71% rename from e2e/enable_node_ports_test.go rename to e2e/service_node_ports_test.go index 523a1d01..52a37f8b 100644 --- a/e2e/enable_node_ports_test.go +++ b/e2e/service_node_ports_test.go @@ -13,20 +13,24 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a nodePort service when it is enabled for Tenant", Label("tenant"), func() { +var _ = Describe("creating a nodePort service when it is enabled for Tenant", Ordered, Label("tenant", "networking", "service"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "enable-node-ports", + Name: "e2e-enable-node-ports", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "google", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-enable-node-ports", Kind: "User", }, }, @@ -40,14 +44,19 @@ var _ = Describe("creating a nodePort service when it is enabled for Tenant", La tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should allow creating a service with NodePort type", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ diff --git a/e2e/storage_class_test.go b/e2e/storage_class_test.go index bf205275..53b257c6 100644 --- a/e2e/storage_class_test.go +++ b/e2e/storage_class_test.go @@ -25,19 +25,24 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes", "storage"), func() { +var _ = Describe("when Tenant handles Storage classes", Ordered, Label("tenant", "storage", "classes", "storageclass"), func() { tntNoDefaults := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "storage-class-selector", + Name: "e2e-storage-class-selector", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "selector", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-storage-class-selector", Kind: "User", }, }, @@ -51,7 +56,7 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes }, LabelSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ - "env": "customer", + "environment": "customer", }, }, }, @@ -61,14 +66,17 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes tntWithDefault := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "storage-class-default", + Name: "e2e-storage-class-default", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "default", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-storage-class-default", Kind: "User", }, }, @@ -90,13 +98,16 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes tntNoRestrictions := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-storage-no-restrictions", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "no-restrictions", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-storage-no-restrictions", Kind: "User", }, }, @@ -109,8 +120,9 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes ObjectMeta: metav1.ObjectMeta{ Name: "cephfs", Labels: map[string]string{ - "name": "cephfs", - "env": "e2e", + "name": "cephfs", + "environment": "internal", + "env": "e2e", }, }, Provisioner: "kubernetes.io/no-provisioner", @@ -120,8 +132,9 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes ObjectMeta: metav1.ObjectMeta{ Name: "tenant-default", Labels: map[string]string{ - "name": "tenant-default", - "env": "e2e", + "name": "tenant-default", + "environment": "internal", + "env": "e2e", }, }, Provisioner: "kubernetes.io/no-provisioner", @@ -130,7 +143,8 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes ObjectMeta: metav1.ObjectMeta{ Name: "global-default", Labels: map[string]string{ - "env": "customer", + "environment": "customer", + "env": "e2e", }, }, Provisioner: "kubernetes.io/no-provisioner", @@ -139,8 +153,9 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes ObjectMeta: metav1.ObjectMeta{ Name: "disallowed-global-default", Labels: map[string]string{ - "name": "disallowed-global-default", - "env": "e2e", + "name": "disallowed-global-default", + "environment": "internal", + "env": "e2e", }, }, Provisioner: "kubernetes.io/no-provisioner", @@ -153,6 +168,8 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) } for _, class := range []storagev1.StorageClass{exact, tenantDefault, globalDefault, disallowedGlobalDefault} { @@ -164,18 +181,24 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes }) JustAfterEach(func() { for _, tnt := range []*capsulev1beta2.Tenant{tntNoDefaults, tntWithDefault, tntNoRestrictions} { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) } - Eventually(func() (err error) { - req, _ := labels.NewRequirement("env", selection.Exists, nil) + req, err := labels.NewRequirement("env", selection.Equals, []string{"e2e"}) + Expect(err).NotTo(HaveOccurred()) - return k8sClient.DeleteAllOf(context.TODO(), &storagev1.StorageClass{}, &client.DeleteAllOfOptions{ - ListOptions: client.ListOptions{ - LabelSelector: labels.NewSelector().Add(*req), - }, - }) - }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + var list storagev1.StorageClassList + Expect(k8sClient.List( + context.TODO(), + &list, + client.MatchingLabelsSelector{ + Selector: labels.NewSelector().Add(*req), + }, + )).Should(Succeed()) + + for i := range list.Items { + EventuallyDeletion(&list.Items[i]) + } }) It("should allow all classes", Label("skip-on-openshift"), func() { @@ -195,9 +218,11 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes Should(ConsistOf("standard", exact.GetName(), tenantDefault.GetName(), globalDefault.GetName(), disallowedGlobalDefault.GetName())) }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoRestrictions.GetName(), + }) NamespaceCreation(ns, tntNoRestrictions.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoRestrictions, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntNoRestrictions, ns).Should(Succeed()) By("providing any storageclass", func() { for _, class := range []storagev1.StorageClass{exact, tenantDefault, globalDefault, disallowedGlobalDefault} { @@ -226,7 +251,7 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes By("Verify Status (Deletion)", func() { for _, class := range []storagev1.StorageClass{exact, tenantDefault} { - Expect(ignoreNotFound(k8sClient.Delete(context.TODO(), &class))).To(Succeed()) + EventuallyDeletion(&class) } Eventually(func() ([]string, error) { t := &capsulev1beta2.Tenant{} @@ -263,9 +288,11 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes Should(ConsistOf(exact.GetName(), globalDefault.GetName())) }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefaults.GetName(), + }) NamespaceCreation(ns, tntNoDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoDefaults, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntNoDefaults, ns).Should(Succeed()) By("non-specifying it", func() { Eventually(func() (err error) { @@ -314,13 +341,30 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("sc-%s", storageName), Labels: map[string]string{ - "env": "internal", + "environment": "internal", + "env": "e2e", }, }, Provisioner: "kubernetes.io/no-provisioner", } Expect(k8sClient.Create(context.TODO(), class)).Should(Succeed()) + By("Verify Status", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntNoDefaults.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.StorageClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + ShouldNot(ContainElement(class.GetName())) + }) + p := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: storageName, @@ -366,11 +410,14 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes Should(ConsistOf(exact.GetName(), globalDefault.GetName())) }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntNoDefaults.GetName(), + }) cs := ownerClient(tntNoDefaults.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tntNoDefaults.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntNoDefaults, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntNoDefaults, ns).Should(Succeed()) + By("using exact matches", func() { for _, c := range tntNoDefaults.Spec.StorageClasses.Exact { @@ -422,13 +469,30 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes ObjectMeta: metav1.ObjectMeta{ Name: storageName, Labels: map[string]string{ - "env": "customer", + "environment": "customer", + "env": "e2e", }, }, Provisioner: "kubernetes.io/no-provisioner", } Expect(k8sClient.Create(context.TODO(), class)).Should(Succeed()) + By("Verify Status", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntNoDefaults.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.StorageClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ContainElement(class.GetName())) + }) + EventuallyCreation(func() error { p := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -487,9 +551,11 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes Should(ConsistOf(tenantDefault.GetName())) }) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) By("Patch Tenant Default", func() { p := &corev1.PersistentVolumeClaim{ @@ -514,9 +580,27 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes }) It("should mutate to default tenant StorageClass (class exists)", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) + + By("Verify Status (Creation)", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntWithDefault.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.StorageClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ConsistOf(tenantDefault.GetName())) + }) p := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -540,9 +624,27 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes }) It("should mutate to default tenant StorageClass although cluster global ons is not allowed", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) + + By("Verify Status (Creation)", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntWithDefault.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.StorageClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ConsistOf(tenantDefault.GetName())) + }) p := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -565,9 +667,27 @@ var _ = Describe("when Tenant handles Storage classes", Label("tenant", "classes }) It("should mutate to default tenant StorageClass although cluster global ons is allowed", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tntWithDefault.GetName(), + }) NamespaceCreation(ns, tntWithDefault.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tntWithDefault, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tntWithDefault, ns).Should(Succeed()) + + By("Verify Status (Creation)", func() { + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tntWithDefault.GetName()}, + t, + ); err != nil { + return nil, err + } + + return t.Status.Classes.StorageClasses, nil + }, defaultTimeoutInterval, defaultPollInterval). + Should(ConsistOf(tenantDefault.GetName())) + }) p := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ diff --git a/e2e/storage_pv_cross_tenant_mount_test.go b/e2e/storage_pv_cross_tenant_mount_test.go new file mode 100644 index 00000000..af87b5e5 --- /dev/null +++ b/e2e/storage_pv_cross_tenant_mount_test.go @@ -0,0 +1,503 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" +) + +var _ = Describe("preventing PersistentVolume cross-tenant mount", Ordered, Label("tenant", "storage", "persistentvolumeclaim"), func() { + tnt1 := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-pv-cross-one", + Labels: map[string]string{ + "env": "e2e", + }, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{ + { + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-pv-cross-one", + Kind: "User", + }, + }, + }, + }, + }, + } + + tnt2 := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-pv-cross-two", + Labels: map[string]string{ + "env": "e2e", + }, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{ + { + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-pv-cross-two", + Kind: "User", + }, + }, + }, + }, + }, + } + + JustBeforeEach(func() { + for _, tnt := range []*capsulev1beta2.Tenant{tnt1, tnt2} { + EventuallyCreation(func() error { + tnt.ResourceVersion = "" + + return k8sClient.Create(context.TODO(), tnt) + }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) + } + }) + + JustAfterEach(func() { + for _, tnt := range []*capsulev1beta2.Tenant{tnt1, tnt2} { + EventuallyDeletion(tnt) + } + }) + + It("should add labels to PersistentVolume and prevent cross-Tenant mount", func() { + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt1.GetName(), + }) + NamespaceCreation(ns, tnt1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt1, ns).Should(Succeed()) + + pvc := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "arrakis", + Namespace: ns.Name, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: ptr.To("standard"), + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(context.Background(), &pvc) + }).Should(Succeed()) + + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "arrakis-pod", + Namespace: ns.Name, + }, + Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + Containers: []corev1.Container{ + { + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + ImagePullPolicy: corev1.PullAlways, + SecurityContext: restrictedContainerSecurityContext(), + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/tmp", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.Name, + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(context.Background(), &pod) + }).Should(Succeed()) + + Eventually(func() int { + nsName := types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace} + + if err := k8sClient.Get(context.Background(), nsName, &pvc); err != nil { + return 0 + } + + return len(pvc.Spec.VolumeName) + }, defaultTimeoutInterval, defaultPollInterval).Should(BeNumerically(">", 0)) + + pv := corev1.PersistentVolume{} + defer func() { + _ = k8sClient.Delete(context.Background(), &pv) + }() + + Eventually(func() string { + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: pvc.Spec.VolumeName}, &pv); err != nil { + return "not-found" + } + + if pv.GetLabels() == nil { + return "no-labels" + } + + return pv.GetLabels()["capsule.clastix.io/tenant"] + }, defaultTimeoutInterval, defaultPollInterval).Should(Equal(tnt1.Name)) + + Eventually(func() error { + nsName := types.NamespacedName{Name: pv.Name} + + if err := k8sClient.Get(context.Background(), nsName, &pv); err != nil { + return err + } + + pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRecycle + + return k8sClient.Update(context.Background(), &pv) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Expect(k8sClient.Delete(context.Background(), &pod, &client.DeleteOptions{GracePeriodSeconds: ptr.To(int64(0))})).ToNot(HaveOccurred()) + + ns2 := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt2.GetName(), + }) + NamespaceCreation(ns2, tnt2.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt2, ns2).Should(Succeed()) + + Consistently(func() error { + pvc := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "caladan", + Namespace: ns2.Name, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: ptr.To("standard"), + VolumeName: pv.Name, + }, + } + + return k8sClient.Create(context.Background(), &pvc) + }, defaultTimeoutInterval, defaultPollInterval).Should(HaveOccurred()) + }) + + It("should not add a selector when updating an already-bound dynamic PVC without selector", func() { + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt1.GetName(), + }) + NamespaceCreation(ns, tnt1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt1, ns).Should(Succeed()) + + pvc := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dynamic-pvc", + Namespace: ns.Name, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: ptr.To("standard"), + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(context.Background(), &pvc) + }).Should(Succeed()) + + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get( + context.Background(), + types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace}, + &pvc, + )).To(Succeed()) + + g.Expect(pvc.Spec.Selector).To(BeNil()) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + pvc.Labels = map[string]string{ + "updated": "true", + } + + Eventually(func() error { + return k8sClient.Update(context.Background(), &pvc) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + Eventually(func(g Gomega) { + updated := corev1.PersistentVolumeClaim{} + + g.Expect(k8sClient.Get( + context.Background(), + types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace}, + &updated, + )).To(Succeed()) + + g.Expect(updated.Labels).To(HaveKeyWithValue("updated", "true")) + g.Expect(updated.Spec.Selector).To(BeNil()) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + It("should add the Tenant selector to a PVC with an existing selector", func() { + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt1.GetName(), + }) + NamespaceCreation(ns, tnt1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt1, ns).Should(Succeed()) + + pvc := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "selector-pvc", + Namespace: ns.Name, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: ptr.To("standard"), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "storage-tier": "gold", + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(context.Background(), &pvc) + }).Should(Succeed()) + + Eventually(func(g Gomega) { + created := corev1.PersistentVolumeClaim{} + + g.Expect(k8sClient.Get( + context.Background(), + types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace}, + &created, + )).To(Succeed()) + + g.Expect(created.Spec.Selector).ToNot(BeNil()) + g.Expect(created.Spec.Selector.MatchLabels).To(HaveKeyWithValue("storage-tier", "gold")) + g.Expect(created.Spec.Selector.MatchLabels).ToNot(HaveKey(meta.TenantLabel)) + g.Expect(created.Spec.Selector.MatchExpressions).To(ContainElement(metav1.LabelSelectorRequirement{ + Key: meta.TenantLabel, + Operator: metav1.LabelSelectorOpIn, + Values: []string{tnt1.Name}, + })) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + It("should overwrite conflicting Tenant selectors on PVC creation", func() { + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt1.GetName(), + }) + NamespaceCreation(ns, tnt1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt1, ns).Should(Succeed()) + + pvc := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "conflicting-selector-pvc", + Namespace: ns.Name, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: ptr.To("standard"), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + meta.TenantLabel: tnt2.Name, + "storage-tier": "gold", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: meta.TenantLabel, + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{tnt1.Name}, + }, + { + Key: "environment", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"test"}, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(context.Background(), &pvc) + }).Should(Succeed()) + + Eventually(func(g Gomega) { + created := corev1.PersistentVolumeClaim{} + + g.Expect(k8sClient.Get( + context.Background(), + types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace}, + &created, + )).To(Succeed()) + + g.Expect(created.Spec.Selector).ToNot(BeNil()) + + // The mutating webhook must preserve unrelated selector requirements. + g.Expect(created.Spec.Selector.MatchLabels).To(HaveKeyWithValue("storage-tier", "gold")) + g.Expect(created.Spec.Selector.MatchExpressions).To(ContainElement(metav1.LabelSelectorRequirement{ + Key: "environment", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"test"}, + })) + + // The tenant selector must be canonicalized. + g.Expect(created.Spec.Selector.MatchLabels).ToNot(HaveKey(meta.TenantLabel)) + + tenantExpressions := 0 + for _, expression := range created.Spec.Selector.MatchExpressions { + if expression.Key != meta.TenantLabel { + continue + } + + tenantExpressions++ + + g.Expect(expression.Operator).To(Equal(metav1.LabelSelectorOpIn)) + g.Expect(expression.Values).To(Equal([]string{tnt1.Name})) + } + + g.Expect(tenantExpressions).To(Equal(1)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + + It("should add the Tenant selector to a pre-bound PVC", func() { + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt1.GetName(), + }) + NamespaceCreation(ns, tnt1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt1, ns).Should(Succeed()) + + pv := corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "prebound-pv", + Labels: map[string]string{ + meta.TenantLabel: tnt1.Name, + }, + }, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: "manual", + PersistentVolumeSource: corev1.PersistentVolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/tmp/capsule-e2e-prebound-pv", + }, + }, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(context.Background(), &pv) + }).Should(Succeed()) + + defer func() { + _ = k8sClient.Delete(context.Background(), &pv) + }() + + pvc := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "prebound-pvc", + Namespace: ns.Name, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + StorageClassName: ptr.To("manual"), + VolumeName: pv.Name, + }, + } + + EventuallyCreation(func() error { + return k8sClient.Create(context.Background(), &pvc) + }).Should(Succeed()) + + Eventually(func(g Gomega) { + created := corev1.PersistentVolumeClaim{} + + g.Expect(k8sClient.Get( + context.Background(), + types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace}, + &created, + )).To(Succeed()) + + g.Expect(created.Spec.VolumeName).To(Equal(pv.Name)) + g.Expect(created.Spec.Selector).ToNot(BeNil()) + g.Expect(created.Spec.Selector.MatchExpressions).To(ContainElement(metav1.LabelSelectorRequirement{ + Key: meta.TenantLabel, + Operator: metav1.LabelSelectorOpIn, + Values: []string{tnt1.Name}, + })) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + }) + +}) diff --git a/e2e/suite_test.go b/e2e/suite_test.go index 21e41293..900025f9 100644 --- a/e2e/suite_test.go +++ b/e2e/suite_test.go @@ -24,8 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" - "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to @@ -40,6 +39,13 @@ var ( var log = ctrl.Log.WithName("e2e-tests") +const ( + ControllerNamespace string = "capsule-system" + ControllerServiceAccount string = "capsule" +) + +var ControllerServiceAccountFull = "system:serviceaccount:" + ControllerNamespace + ":" + ControllerServiceAccount + func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) @@ -61,57 +67,82 @@ var _ = BeforeSuite(func() { Expect(capsulev1beta2.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred()) + tuneE2ERestConfig(cfg) + ctrlClient, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).ToNot(HaveOccurred()) Expect(ctrlClient).ToNot(BeNil()) k8sClient = &e2eClient{Client: ctrlClient} - - ModifyCapsuleConfigurationOpts(func(cfg *capsulev1beta2.CapsuleConfiguration) { - cfg.Spec = configuration.DefaultCapsuleConfiguration() - }) - }) -var _ = AfterSuite(func() { - Eventually(func() error { - var nsList corev1.NamespaceList +var _ = SynchronizedAfterSuite( + func() { + // Runs on every parallel process. + // Keep this empty, or put per-worker cleanup here. + }, + func() { + Eventually(func() error { + var tnts capsulev1beta2.TenantList - // List all namespaces with env=e2e - if err := k8sClient.List( - context.TODO(), - &nsList, - client.MatchingLabels{"env": "e2e"}, - ); err != nil { - return err - } - - // If none left, we’re done - if len(nsList.Items) == 0 { - return nil - } - - // Try deleting all; if any delete fails with something other than NotFound, - // return the error so Eventually keeps retrying. - for i := range nsList.Items { - ns := &nsList.Items[i] - if err := k8sClient.Delete(context.TODO(), ns); err != nil && !apierrors.IsNotFound(err) { + if err := k8sClient.List( + context.TODO(), + &tnts, + client.MatchingLabels{"env": "e2e"}, + ); err != nil { return err } - } - // Return a non-nil error to tell Eventually "not done yet" - return fmt.Errorf("still have %d namespaces with env=e2e", len(nsList.Items)) - }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + if len(tnts.Items) == 0 { + return nil + } - By("tearing down the test environment") + for i := range tnts.Items { + ns := &tnts.Items[i] + if err := k8sClient.Delete(context.TODO(), ns); err != nil && !apierrors.IsNotFound(err) { + return err + } + } - Expect(testEnv.Stop()).ToNot(HaveOccurred()) -}) + return fmt.Errorf("still have %d tenants with env=e2e", len(tnts.Items)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) -func ownerClient(owner api.UserSpec) (cs kubernetes.Interface) { + Eventually(func() error { + var nsList corev1.NamespaceList + + if err := k8sClient.List( + context.TODO(), + &nsList, + client.MatchingLabels{"env": "e2e"}, + ); err != nil { + return err + } + + if len(nsList.Items) == 0 { + return nil + } + + for i := range nsList.Items { + ns := &nsList.Items[i] + if err := k8sClient.Delete(context.TODO(), ns); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + + return fmt.Errorf("still have %d namespaces with env=e2e", len(nsList.Items)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + By("tearing down the test environment") + + Expect(testEnv.Stop()).ToNot(HaveOccurred()) + }, +) + +func ownerClient(owner rbac.UserSpec) (cs kubernetes.Interface) { c, err := config.GetConfig() Expect(err).ToNot(HaveOccurred()) + tuneE2ERestConfig(c) + c.Impersonate.Groups = []string{"projectcapsule.dev", owner.Name} c.Impersonate.UserName = owner.Name cs, err = kubernetes.NewForConfig(c) @@ -120,8 +151,23 @@ func ownerClient(owner api.UserSpec) (cs kubernetes.Interface) { return cs } +func impersonationClientSet(user string, groups []string) (cs kubernetes.Interface) { + c, err := config.GetConfig() + Expect(err).ToNot(HaveOccurred()) + tuneE2ERestConfig(c) + + c.Impersonate.Groups = groups + c.Impersonate.UserName = user + cs, err = kubernetes.NewForConfig(c) + Expect(err).ToNot(HaveOccurred()) + + return cs +} + func impersonationClient(user string, groups []string) client.Client { impersonatedCfg := rest.CopyConfig(cfg) + tuneE2ERestConfig(impersonatedCfg) + impersonatedCfg.Impersonate = rest.ImpersonationConfig{ UserName: user, Groups: groups, diff --git a/e2e/container_registry_test.go b/e2e/tenant_container_registry_test.go similarity index 76% rename from e2e/container_registry_test.go rename to e2e/tenant_container_registry_test.go index 0e643db6..050b6376 100644 --- a/e2e/container_registry_test.go +++ b/e2e/tenant_container_registry_test.go @@ -17,6 +17,8 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) type Patch struct { @@ -25,19 +27,22 @@ type Patch struct { Value string `json:"value"` } -var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "registry"), func() { +var _ = Describe("enforcing a Container Registry", Ordered, Label("tenant", "images", "registry"), func() { originConfig := &capsulev1beta2.CapsuleConfiguration{} tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "container-registry", + Name: "e2e-container-registry", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "matt", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-container-registry", Kind: "User", }, }, @@ -57,25 +62,18 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) - - // Restore Configuration - Eventually(func() error { - c := &capsulev1beta2.CapsuleConfiguration{} - if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: originConfig.Name}, c); err != nil { - return err - } - // Apply the initial configuration from originConfig to c - c.Spec = originConfig.Spec - return k8sClient.Update(context.Background(), c) - }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should add labels to Namespace", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) Eventually(func() (ok bool) { Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: ns.Name}, ns)).Should(Succeed()) @@ -92,17 +90,21 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re }) It("should deny running a gcr.io container", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "container", - Image: "gcr.io/google_containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -111,7 +113,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) EventuallyCreation(func() error { _, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{}) @@ -121,17 +123,21 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re }) It("should allow using a registry only match", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "container", - Image: "myregistry.azurecr.io/myapp:latest", + Name: "container", + Image: "myregistry.azurecr.io/myapp:latest", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -140,7 +146,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) EventuallyCreation(func() error { _, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{}) @@ -161,17 +167,21 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re }) It("should deny patching a not matching registry after applying with a matching (Container)", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "container", - Image: "myregistry.azurecr.io/myapp:latest", + Name: "container", + Image: "myregistry.azurecr.io/myapp:latest", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -180,7 +190,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) EventuallyCreation(func() error { _, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{}) @@ -215,17 +225,21 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re }) It("should deny patching a not matching registry after applying with a matching (EphemeralContainer)", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "container", - Image: "docker.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "docker.io/google-containers/pause-amd64:3.0", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -234,7 +248,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) role := &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ @@ -298,23 +312,28 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re }) It("should deny patching a not matching registry after applying with a matching (initContainer)", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), InitContainers: []corev1.Container{ { - Name: "init", - Image: "myregistry.azurecr.io/myapp:latest", + Name: "init", + Image: "myregistry.azurecr.io/myapp:latest", + SecurityContext: restrictedContainerSecurityContext(), }, }, Containers: []corev1.Container{ { - Name: "container", - Image: "myregistry.azurecr.io/myapp:latest", + Name: "container", + Image: "myregistry.azurecr.io/myapp:latest", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -323,7 +342,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) EventuallyCreation(func() error { _, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{}) @@ -347,17 +366,21 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re }) It("should deny patching a not matching registry after applying with a matching (Container)", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "container", - Image: "myregistry.azurecr.io/myapp:latest", + Name: "container", + Image: "myregistry.azurecr.io/myapp:latest", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -366,7 +389,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) EventuallyCreation(func() error { _, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{}) @@ -390,17 +413,21 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re }) It("should allow patching a matching registry after applying with a matching (EphemeralContainer)", Label("skip-on-openshift"), func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "container", - Image: "docker.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "docker.io/google-containers/pause-amd64:3.0", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -409,7 +436,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) role := &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ @@ -473,23 +500,29 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re }) It("should allow patching a matching registry after applying with a matching (initContainer)", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), InitContainers: []corev1.Container{ { - Name: "init", - Image: "myregistry.azurecr.io/myapp:latest", + Name: "init", + Image: "myregistry.azurecr.io/myapp:latest", + SecurityContext: restrictedContainerSecurityContext(), }, }, + Containers: []corev1.Container{ { - Name: "container", - Image: "docker.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "docker.io/google-containers/pause-amd64:3.0", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -498,7 +531,7 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) EventuallyCreation(func() error { _, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{}) @@ -522,7 +555,9 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re }) It("should allow using an exact match", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) pod := &corev1.Pod{ @@ -547,7 +582,9 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re }) It("should allow using a regex match", func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) pod := &corev1.Pod{ @@ -555,10 +592,12 @@ var _ = Describe("enforcing a Container Registry", Label("tenant", "images", "re Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "quay.io/google-containers/pause-amd64:3.0", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, diff --git a/e2e/tenant_cordoning_test.go b/e2e/tenant_cordoning_test.go index 8da6bad2..3db8fd5c 100644 --- a/e2e/tenant_cordoning_test.go +++ b/e2e/tenant_cordoning_test.go @@ -5,31 +5,36 @@ package e2e import ( "context" - "time" + "encoding/json" - "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" ) -var _ = Describe("cordoning a Tenant", Label("tenant"), func() { +var _ = Describe("cordoning a Tenant", Ordered, Label("tenant", "operations", "cordoning"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-cordoning", + Name: "e2e-cordoning", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "jim", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-cordoning", Kind: "User", }, }, @@ -38,128 +43,286 @@ var _ = Describe("cordoning a Tenant", Label("tenant"), func() { }, } + patchNamespaceCordonedLabel := func(cs kubernetes.Interface, nsName string, value string) error { + labels := map[string]any{ + meta.CordonedLabel: value, + } + + if value == "" { + labels[meta.CordonedLabel] = nil + } + + patch, err := json.Marshal(map[string]any{ + "metadata": map[string]any{ + "labels": labels, + }, + }) + if err != nil { + return err + } + + _, err = cs.CoreV1().Namespaces().Patch( + context.TODO(), + nsName, + types.StrategicMergePatchType, + patch, + metav1.PatchOptions{}, + ) + + return err + } + + expectNamespaceExists := func(name string) { + Eventually(func() error { + current := &corev1.Namespace{} + + return k8sClient.Get(context.TODO(), types.NamespacedName{Name: name}, current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + expectNamespaceDeleted := func(name string) { + Eventually(func() bool { + current := &corev1.Namespace{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: name}, current) + + return apierrors.IsNotFound(err) + }, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue()) + } + + expectTenantCordonedCondition := func(expectedStatus metav1.ConditionStatus, expectedReason string) { + Eventually(func(g Gomega) { + t := &capsulev1beta2.Tenant{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t) + g.Expect(err).NotTo(HaveOccurred()) + + condition := t.Status.Conditions.GetConditionByType(meta.CordonedCondition) + g.Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + + g.Expect(condition.Type).To(Equal(meta.CordonedCondition)) + g.Expect(condition.Status).To(Equal(expectedStatus)) + g.Expect(condition.Reason).To(Equal(expectedReason)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + setTenantCordoned := func(cordoned bool) { + Eventually(func() error { + current := &capsulev1beta2.Tenant{} + if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, current); err != nil { + return err + } + + current.Spec.Cordoned = cordoned + + return k8sClient.Update(context.TODO(), current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + + expectNamespaceCordonedLabel := func(nsName string, expected bool) { + Eventually(func(g Gomega) { + current := &corev1.Namespace{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: nsName}, current) + g.Expect(err).NotTo(HaveOccurred()) + + if expected { + g.Expect(current.Labels).To(HaveKey(meta.CordonedLabel)) + } else { + g.Expect(current.Labels).NotTo(HaveKey(meta.CordonedLabel)) + } + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + } + JustBeforeEach(func() { EventuallyCreation(func() error { + tnt.ResourceVersion = "" + return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) + + It("should allow tenant owner to cordon a namespace by patching the cordoned label", func() { + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + + NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + + By("patching the namespace cordoned label as tenant owner") + Expect(patchNamespaceCordonedLabel(cs, ns.GetName(), meta.ValueTrue)).To(Succeed()) + expectNamespaceCordonedLabel(ns.GetName(), true) + }) + + It("should block namespace membership changes while Tenant is cordoned", func() { + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + + existing := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + + By("creating an initial namespace before cordoning") + NamespaceCreation(existing, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, existing).Should(Succeed()) + + By("cordoning the Tenant") + setTenantCordoned(true) + expectTenantCordonedCondition(metav1.ConditionTrue, meta.CordonedReason) + expectNamespaceCordonedLabel(existing.GetName(), true) + + By("rejecting new namespace assignment to the cordoned Tenant") + blocked := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + + _, err := cs.CoreV1().Namespaces().Create(context.TODO(), blocked, metav1.CreateOptions{}) + Expect(err).To(HaveOccurred()) + + By("rejecting deletion of an existing namespace while cordoned") + err = cs.CoreV1().Namespaces().Delete(context.TODO(), existing.GetName(), metav1.DeleteOptions{}) + Expect(err).To(HaveOccurred()) + + expectNamespaceExists(existing.GetName()) + + By("uncordoning the Tenant allows namespace deletion again") + setTenantCordoned(false) + expectTenantCordonedCondition(metav1.ConditionFalse, meta.ActiveReason) + expectNamespaceCordonedLabel(existing.GetName(), false) + + Expect(cs.CoreV1().Namespaces().Delete(context.TODO(), existing.GetName(), metav1.DeleteOptions{})).To(Succeed()) + expectNamespaceDeleted(existing.GetName()) + }) + + It("should block content changes inside a namespace cordoned by tenant owner", func() { + cs := ownerClient(tnt.Spec.Owners[0].UserSpec) + + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) + + NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) + + By("cordoning the namespace directly") + Expect(patchNamespaceCordonedLabel(cs, ns.GetName(), meta.ValueTrue)).To(Succeed()) + expectNamespaceCordonedLabel(ns.GetName(), true) + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "blocked-create", + }, + Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + Containers: []corev1.Container{ + { + Name: "container", + Image: "quay.io/google-containers/pause-amd64:3.0", + SecurityContext: restrictedContainerSecurityContext(), + }, + }, + }, + } + + By("rejecting content creation inside the cordoned namespace") + _, err := cs.CoreV1().Pods(ns.GetName()).Create(context.TODO(), pod, metav1.CreateOptions{}) + Expect(err).To(HaveOccurred()) + + By("uncordoning the namespace allows content creation again") + Expect(patchNamespaceCordonedLabel(cs, ns.GetName(), "")).To(Succeed()) + expectNamespaceCordonedLabel(ns.GetName(), false) + + _, err = cs.CoreV1().Pods(ns.GetName()).Create(context.TODO(), pod, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + It("should block or allow operations", func() { cs := ownerClient(tnt.Spec.Owners[0].UserSpec) - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "container", - Image: "gcr.io/google_containers/pause-amd64:3.0", + Name: "container", + Image: "quay.io/google-containers/pause-amd64:3.0", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, } - By("Verifing Tenant Status", func() { - t := &capsulev1beta2.Tenant{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).Should(Succeed()) - - condition := t.Status.Conditions.GetConditionByType(meta.CordonedCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") - - Expect(condition.Status).To(Equal(metav1.ConditionFalse), "Expected tenant condition status to be True") - Expect(condition.Type).To(Equal(meta.CordonedCondition), "Expected tenant condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.ActiveReason), "Expected tenant condition reason to be Succeeded") + By("Verifying Tenant Status", func() { + expectTenantCordonedCondition(metav1.ConditionFalse, meta.ActiveReason) }) - By("creating a Namespace", func() { + By("creating a Namespace and Pod", func() { NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) EventuallyCreation(func() error { _, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err }).Should(Succeed()) }) - By("should contain the cordoned Capsule label", func() { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.Name}, tnt)).Should(Succeed()) - - tnt.Spec.Cordoned = true - - Expect(k8sClient.Update(context.TODO(), tnt)).Should(Succeed()) - - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) - - Expect(ns.Labels).To(HaveKey(meta.CordonedLabel)) - + By("cordoning the Tenant should add the cordoned Capsule label", func() { + setTenantCordoned(true) + expectNamespaceCordonedLabel(ns.GetName(), true) }) - By("Verifing Tenant Status", func() { - t := &capsulev1beta2.Tenant{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).Should(Succeed()) - - condition := t.Status.Conditions.GetConditionByType(meta.CordonedCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") - - Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected tenant condition status to be True") - Expect(condition.Type).To(Equal(meta.CordonedCondition), "Expected tenant condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.CordonedReason), "Expected tenant condition reason to be Succeeded") + By("Verifying Tenant Status after cordoning", func() { + expectTenantCordonedCondition(metav1.ConditionTrue, meta.CordonedReason) }) By("cordoning the Tenant deletion must be blocked", func() { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.Name}, tnt)).Should(Succeed()) + setTenantCordoned(true) + expectNamespaceCordonedLabel(ns.GetName(), true) - tnt.Spec.Cordoned = true - - Expect(k8sClient.Update(context.TODO(), tnt)).Should(Succeed()) - - time.Sleep(2 * time.Second) - - Expect(cs.CoreV1().Pods(ns.Name).Delete(context.Background(), pod.Name, metav1.DeleteOptions{})).ShouldNot(Succeed()) + Eventually(func() error { + return cs.CoreV1().Pods(ns.Name).Delete(context.Background(), pod.Name, metav1.DeleteOptions{}) + }, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed()) }) By("uncordoning the Tenant deletion must be allowed", func() { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.Name}, tnt)).Should(Succeed()) + setTenantCordoned(false) + expectNamespaceCordonedLabel(ns.GetName(), false) - tnt.Spec.Cordoned = false + Eventually(func() error { + err := cs.CoreV1().Pods(ns.Name).Delete(context.Background(), pod.Name, metav1.DeleteOptions{}) + if apierrors.IsNotFound(err) { + return nil + } - Expect(k8sClient.Update(context.TODO(), tnt)).Should(Succeed()) - - time.Sleep(2 * time.Second) - - Expect(cs.CoreV1().Pods(ns.Name).Delete(context.Background(), pod.Name, metav1.DeleteOptions{})).Should(Succeed()) + return err + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) }) By("should not contain the cordoned Capsule label", func() { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.Name}, tnt)).Should(Succeed()) - - tnt.Spec.Cordoned = false - - Expect(k8sClient.Update(context.TODO(), tnt)).Should(Succeed()) - - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, ns)).Should(Succeed()) - - Expect(ns.Labels).ToNot(HaveKey(meta.CordonedLabel)) - + setTenantCordoned(false) + expectNamespaceCordonedLabel(ns.GetName(), false) }) - By("Verifing Tenant Status", func() { - t := &capsulev1beta2.Tenant{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).Should(Succeed()) - - condition := t.Status.Conditions.GetConditionByType(meta.CordonedCondition) - Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") - - Expect(condition.Status).To(Equal(metav1.ConditionFalse), "Expected tenant condition status to be True") - Expect(condition.Type).To(Equal(meta.CordonedCondition), "Expected tenant condition type to be Ready") - Expect(condition.Reason).To(Equal(meta.ActiveReason), "Expected tenant condition reason to be Succeeded") + By("Verifying Tenant Status after uncordoning", func() { + expectTenantCordonedCondition(metav1.ConditionFalse, meta.ActiveReason) }) }) + }) diff --git a/e2e/tenant_forbidden_annotations_regex_test.go b/e2e/tenant_forbidden_annotations_regex_test.go new file mode 100644 index 00000000..29fbe9ba --- /dev/null +++ b/e2e/tenant_forbidden_annotations_regex_test.go @@ -0,0 +1,72 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" +) + +var _ = Describe("creating a tenant with various forbidden regexes", Ordered, Label("tenant", "metadata", "forbidden"), func() { + successRegexes := []string{ + "", + "(.*gitops|.*nsm)", + } + for i, annotationValue := range successRegexes { + It("should succeed using a valid regex on the annotation", func() { + tnt := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-namespace-regex-" + strconv.Itoa(i), + Labels: map[string]string{ + "env": "e2e", + }, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: rbac.OwnerListSpec{ + { + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-namespace-regex", + Kind: "User", + }, + }, + }, + }, + }, + } + + EventuallyCreation(func() error { + tnt.SetResourceVersion("") + + tnt.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{ + ForbiddenLabels: api.ForbiddenListSpec{ + Regex: annotationValue, + }, + } + return k8sClient.Create(context.TODO(), tnt) + }).Should(Succeed()) + EventuallyDeletion(tnt) + + EventuallyCreation(func() error { + tnt.SetResourceVersion("") + + tnt.Spec.NamespaceOptions = &capsulev1beta2.NamespaceOptions{ + ForbiddenAnnotations: api.ForbiddenListSpec{ + Regex: annotationValue, + }, + } + return k8sClient.Create(context.TODO(), tnt) + }).Should(Succeed()) + EventuallyDeletion(tnt) + }) + } +}) diff --git a/e2e/force_tenant_prefix_tenant_scope_test.go b/e2e/tenant_force_prefix_test.go similarity index 60% rename from e2e/force_tenant_prefix_tenant_scope_test.go rename to e2e/tenant_force_prefix_test.go index 3bfd1116..f3d4e0fe 100644 --- a/e2e/force_tenant_prefix_tenant_scope_test.go +++ b/e2e/tenant_force_prefix_test.go @@ -11,21 +11,25 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Namespace with Tenant name prefix enforcement at Tenant scope", Label("tenant", "config"), func() { +var _ = Describe("creating a Namespace with Tenant name prefix enforcement at Tenant scope", Ordered, Label("config", "tenant", "prefix"), func() { t1 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "awesome", + Name: "e2e-tenant-force-prefix", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ ForceTenantPrefix: &[]bool{true}[0], - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "john", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-force-prefix", Kind: "User", }, }, @@ -35,15 +39,15 @@ var _ = Describe("creating a Namespace with Tenant name prefix enforcement at Te } t2 := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "awesome-tenant", + Name: "e2e-tenant-force-prefix-tenant", }, Spec: capsulev1beta2.TenantSpec{ ForceTenantPrefix: &[]bool{false}[0], - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "john", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-force-prefix", Kind: "User", }, }, @@ -57,14 +61,20 @@ var _ = Describe("creating a Namespace with Tenant name prefix enforcement at Te t1.ResourceVersion = "" return k8sClient.Create(context.TODO(), t1) }).Should(Succeed()) + + TenantReady(t1, metav1.ConditionTrue, defaultTimeoutInterval) + EventuallyCreation(func() error { t2.ResourceVersion = "" return k8sClient.Create(context.TODO(), t2) }).Should(Succeed()) + + TenantReady(t2, metav1.ConditionTrue, defaultTimeoutInterval) + }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), t1)).Should(Succeed()) - Expect(k8sClient.Delete(context.TODO(), t2)).Should(Succeed()) + EventuallyDeletion(t1) + EventuallyDeletion(t2) ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.ForceTenantPrefix = false @@ -75,81 +85,100 @@ var _ = Describe("creating a Namespace with Tenant name prefix enforcement at Te ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.ForceTenantPrefix = false }) - labels := map[string]string{ - "capsule.clastix.io/tenant": t1.GetName(), - } - ns := NewNamespace("awesome", labels) + + ns := NewNamespace("e2e-tenant-force-prefix", map[string]string{ + meta.TenantLabel: t1.GetName(), + }) NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + NamespaceIsNotPartOfTenant(t1, ns).Should(Succeed()) }) It("should fail using prefix without capsule.clastix.io/tenant label, where the user owns more than one Tenant, for a tenant with ForceTenantPrefix true and global ForceTenantPrefix false", func() { - ns := NewNamespace("awesome-namespace") - NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + ns := NewNamespace("e2e-tenant-force-prefix-namespace") + NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) }) It("should fail using prefix without capsule.clastix.io/tenant label, where the user owns more than one Tenant, for a tenant with ForceTenantPrefix false and global ForceTenantPrefix true", func() { - ns := NewNamespace("awesome-namespace") - NamespaceCreation(ns, t2.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + ns := NewNamespace("e2e-tenant-force-prefix-tenant-namespace") + NamespaceCreation(ns, t2.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t2, ns).Should(Succeed()) }) It("should succeed and be assigned with prefix and label, for a tenant with ForceTenantPrefix true and global ForceTenantPrefix false", func() { - labels := map[string]string{ - "capsule.clastix.io/tenant": t1.GetName(), - } - ns := NewNamespace("awesome-tenant", labels) + ns := NewNamespace("e2e-tenant-force-prefix-tenant", map[string]string{ + meta.TenantLabel: t1.GetName(), + }) NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) - TenantNamespaceList(t1, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) }) It("should fail when not using prefix, with tenant label for a tenant with ForceTenantPrefix true and global ForceTenantPrefix true", func() { ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.ForceTenantPrefix = true }) - labels := map[string]string{ - "capsule.clastix.io/tenant": t1.GetName(), - } - ns := NewNamespace("awesome", labels) + + ns := NewNamespace("e2e-tenant-force-prefix", map[string]string{ + meta.TenantLabel: t1.GetName(), + }) + NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + NamespaceIsPartOfTenant(t1, ns).ShouldNot(Succeed()) + }) + + It("should fail when not using prefix, with tenant label for a tenant with ForceTenantPrefix true and global ForceTenantPrefix true", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.ForceTenantPrefix = true + }) + + ns := NewNamespace("e2e-tenant-force-none", map[string]string{ + meta.TenantLabel: t1.GetName(), + }) + + NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + NamespaceIsPartOfTenant(t1, ns).ShouldNot(Succeed()) }) It("should succeed and be assigned with prefix and label, for a tenant with ForceTenantPrefix true and global ForceTenantPrefix true", func() { ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.ForceTenantPrefix = true }) - labels := map[string]string{ - "capsule.clastix.io/tenant": t1.GetName(), - } - ns := NewNamespace("awesome-tenant", labels) - NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(t1, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) - }) - - It("should fail using prefix without capsule.clastix.io/tenant label, for a tenant with ForceTenantPrefix true and global ForceTenantPrefix true", func() { - ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { - configuration.Spec.ForceTenantPrefix = true + ns := NewNamespace("e2e-tenant-force-prefix-tenant", map[string]string{ + meta.TenantLabel: t1.GetName(), }) - ns := NewNamespace("awesome-namespace") - NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).ShouldNot(Succeed()) + NamespaceCreation(ns, t1.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t1, ns).Should(Succeed()) }) It("should succeed when not using prefix, with tenant label for a tenant with ForceTenantPrefix false and global ForceTenantPrefix false", func() { - labels := map[string]string{ - "capsule.clastix.io/tenant": t2.GetName(), - } - ns := NewNamespace("awesome", labels) + ns := NewNamespace("e2e-tenant-force-prefix", map[string]string{ + meta.TenantLabel: t2.GetName(), + }) NamespaceCreation(ns, t2.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t2, ns).Should(Succeed()) }) It("should succeed when not using prefix, with tenant label for a tenant with ForceTenantPrefix false and global ForceTenantPrefix true", func() { ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { configuration.Spec.ForceTenantPrefix = true }) - labels := map[string]string{ - "capsule.clastix.io/tenant": t2.GetName(), - } - ns := NewNamespace("awesome", labels) + + ns := NewNamespace("e2e-tenant-force-prefix", map[string]string{ + meta.TenantLabel: t2.GetName(), + }) NamespaceCreation(ns, t2.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t2, ns).Should(Succeed()) + }) + + It("should fail using prefix without capsule.clastix.io/tenant label, for a tenant with ForceTenantPrefix true and global ForceTenantPrefix true", func() { + ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) { + configuration.Spec.ForceTenantPrefix = true + }) + ns := NewNamespace("e2e-tenant-force-prefix-tenant-namespace") + NamespaceCreation(ns, t2.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(t2, ns).Should(Succeed()) + NamespaceIsNotPartOfTenant(t1, ns).Should(Succeed()) }) }) diff --git a/e2e/imagepullpolicy_multiple_test.go b/e2e/tenant_imagepullpolicy_multiple_test.go similarity index 81% rename from e2e/imagepullpolicy_multiple_test.go rename to e2e/tenant_imagepullpolicy_multiple_test.go index 6510381c..78251d69 100644 --- a/e2e/imagepullpolicy_multiple_test.go +++ b/e2e/tenant_imagepullpolicy_multiple_test.go @@ -14,19 +14,24 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("enforcing some defined ImagePullPolicy", Label("tenant", "images", "policy"), func() { +var _ = Describe("enforcing some defined ImagePullPolicy", Ordered, Label("tenant", "pods", "images", "policy"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "image-pull-policies", + Name: "e2e-image-pull-policies", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "alex", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-image-pull-policies", Kind: "User", }, }, @@ -41,15 +46,20 @@ var _ = Describe("enforcing some defined ImagePullPolicy", Label("tenant", "imag tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should just allow the defined policies", Label("skip-on-openshift"), func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) @@ -95,11 +105,13 @@ var _ = Describe("enforcing some defined ImagePullPolicy", Label("tenant", "imag Name: "pull-always", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "container", Image: "gcr.io/google_containers/pause-amd64:3.0", ImagePullPolicy: corev1.PullAlways, + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -118,6 +130,7 @@ var _ = Describe("enforcing some defined ImagePullPolicy", Label("tenant", "imag Name: "dbg", Image: "gcr.io/google_containers/pause-amd64:3.0", ImagePullPolicy: corev1.PullAlways, + SecurityContext: restrictedContainerSecurityContext(), }, }, } @@ -137,11 +150,13 @@ var _ = Describe("enforcing some defined ImagePullPolicy", Label("tenant", "imag Name: "if-not-present", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "container", Image: "gcr.io/google_containers/pause-amd64:3.0", ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -160,6 +175,7 @@ var _ = Describe("enforcing some defined ImagePullPolicy", Label("tenant", "imag Name: "dbg", Image: "gcr.io/google_containers/pause-amd64:3.0", ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, } @@ -178,11 +194,13 @@ var _ = Describe("enforcing some defined ImagePullPolicy", Label("tenant", "imag Name: "never", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "container", Image: "gcr.io/google_containers/pause-amd64:3.0", ImagePullPolicy: corev1.PullNever, + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -201,6 +219,7 @@ var _ = Describe("enforcing some defined ImagePullPolicy", Label("tenant", "imag Name: "dbg", Image: "gcr.io/google_containers/pause-amd64:3.0", ImagePullPolicy: corev1.PullNever, + SecurityContext: restrictedContainerSecurityContext(), }, }, } diff --git a/e2e/imagepullpolicy_single_test.go b/e2e/tenant_imagepullpolicy_single_test.go similarity index 81% rename from e2e/imagepullpolicy_single_test.go rename to e2e/tenant_imagepullpolicy_single_test.go index f1b06cd3..8bc99ef0 100644 --- a/e2e/imagepullpolicy_single_test.go +++ b/e2e/tenant_imagepullpolicy_single_test.go @@ -14,19 +14,24 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("enforcing a defined ImagePullPolicy", Label("tenant", "images", "policy"), func() { +var _ = Describe("enforcing a defined ImagePullPolicy", Ordered, Label("tenant", "pods", "images", "policy"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "image-pull-policy", + Name: "e2e-image-pull-policy", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "axel", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-image-pull-policy", Kind: "User", }, }, @@ -41,15 +46,20 @@ var _ = Describe("enforcing a defined ImagePullPolicy", Label("tenant", "images" tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should just allow the defined policy", Label("skip-on-openshift"), func() { - ns := NewNamespace("") + ns := NewNamespace("", map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) @@ -95,11 +105,13 @@ var _ = Describe("enforcing a defined ImagePullPolicy", Label("tenant", "images" Name: "pull-always", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "container", Image: "gcr.io/google_containers/pause-amd64:3.0", ImagePullPolicy: corev1.PullAlways, + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -118,6 +130,7 @@ var _ = Describe("enforcing a defined ImagePullPolicy", Label("tenant", "images" Name: "dbg", Image: "gcr.io/google_containers/pause-amd64:3.0", ImagePullPolicy: corev1.PullAlways, + SecurityContext: restrictedContainerSecurityContext(), }, }, } @@ -136,11 +149,13 @@ var _ = Describe("enforcing a defined ImagePullPolicy", Label("tenant", "images" Name: "if-not-present", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "container", Image: "gcr.io/google_containers/pause-amd64:3.0", ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -159,6 +174,7 @@ var _ = Describe("enforcing a defined ImagePullPolicy", Label("tenant", "images" Name: "dbg", Image: "gcr.io/google_containers/pause-amd64:3.0", ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: restrictedContainerSecurityContext(), }, }, } @@ -177,11 +193,13 @@ var _ = Describe("enforcing a defined ImagePullPolicy", Label("tenant", "images" Name: "never", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "container", Image: "gcr.io/google_containers/pause-amd64:3.0", ImagePullPolicy: corev1.PullNever, + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -200,6 +218,7 @@ var _ = Describe("enforcing a defined ImagePullPolicy", Label("tenant", "images" Name: "dbg", Image: "gcr.io/google_containers/pause-amd64:3.0", ImagePullPolicy: corev1.PullNever, + SecurityContext: restrictedContainerSecurityContext(), }, }, } diff --git a/e2e/tenant_resources_changes_test.go b/e2e/tenant_managed_resources_changes_test.go similarity index 59% rename from e2e/tenant_resources_changes_test.go rename to e2e/tenant_managed_resources_changes_test.go index dabbe98e..8537dda9 100644 --- a/e2e/tenant_resources_changes_test.go +++ b/e2e/tenant_managed_resources_changes_test.go @@ -8,11 +8,15 @@ import ( "fmt" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -21,17 +25,20 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" ) -var _ = Describe("changing Tenant managed Kubernetes resources", Label("tenant", "managed", "current"), func() { +var _ = Describe("changing Tenant managed Kubernetes resources", Ordered, Label("tenant", "managed"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-resources-changes", + Name: "e2e-tenant-managed-changes", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "laura", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-managed-changes", Kind: "User", }, }, @@ -100,7 +107,7 @@ var _ = Describe("changing Tenant managed Kubernetes resources", Label("tenant", }, { IPBlock: &networkingv1.IPBlock{ - CIDR: "192.168.0.0/12", + CIDR: "192.160.0.0/12", }, }, }, @@ -113,7 +120,7 @@ var _ = Describe("changing Tenant managed Kubernetes resources", Label("tenant", IPBlock: &networkingv1.IPBlock{ CIDR: "0.0.0.0/0", Except: []string{ - "192.168.0.0/12", + "192.160.0.0/12", }, }, }, @@ -163,20 +170,41 @@ var _ = Describe("changing Tenant managed Kubernetes resources", Label("tenant", tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) + By("creating the Namespaces", func() { for _, i := range nsl { - ns := NewNamespace(i) + ns := NewNamespace(i, map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) } }) }) + JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) + It("should reapply the original resources upon third party change", func() { + + sampleUser := "test@user.com" + for _, ns := range nsl { By("changing Limit Range", func() { + + ensureTamperRoleBinding( + context.TODO(), + k8sClient, + ns, + sampleUser, + "allow-tamper-limitrange", + "", + "limitranges", + ) + for i, s := range tnt.Spec.LimitRanges.Items { n := fmt.Sprintf("capsule-%s-%d", tnt.GetName(), i) lr := &corev1.LimitRange{} @@ -184,13 +212,31 @@ var _ = Describe("changing Tenant managed Kubernetes resources", Label("tenant", return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, lr) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) - cs := ownerClient(tnt.Spec.Owners[0].UserSpec) - err := cs.CoreV1().LimitRanges(ns).Delete(context.TODO(), n, metav1.DeleteOptions{}) - Expect(err).To(HaveOccurred()) + // Delete As owner in the namespace should fails + cs := impersonationClient(tnt.Spec.Owners[0].UserSpec.Name, withDefaultGroups(nil)) + + By(fmt.Sprintf("owner cannot delete limitrange"), func() { + obj := &corev1.LimitRange{ObjectMeta: metav1.ObjectMeta{Name: n, Namespace: ns}} + err := cs.Delete(context.TODO(), obj) + Expect(err).To(HaveOccurred()) + }) + + By(fmt.Sprintf("owner cannot update limitrange"), func() { + current := &corev1.LimitRange{} + Expect(cs.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, current)).To(Succeed()) + + mut := current.DeepCopy() + mut.Spec.Limits = []corev1.LimitRangeItem{} + err := cs.Update(context.TODO(), mut) + Expect(err).To(HaveOccurred()) + }) c := lr.DeepCopy() c.Spec.Limits = []corev1.LimitRangeItem{} - Expect(k8sClient.Update(context.TODO(), c, &client.UpdateOptions{})).Should(Succeed()) + Expect(k8sClient.Update(context.TODO(), c, &client.UpdateOptions{})).ShouldNot(Succeed()) + + controllerAccount := impersonationClient(ControllerServiceAccountFull, nil) + Expect(controllerAccount.Update(context.TODO(), c, &client.UpdateOptions{})).Should(Succeed()) Eventually(func() corev1.LimitRangeSpec { Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, lr)).Should(Succeed()) @@ -207,14 +253,36 @@ var _ = Describe("changing Tenant managed Kubernetes resources", Label("tenant", }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) Expect(np.Spec).Should(Equal(s)) - cs := ownerClient(tnt.Spec.Owners[0].UserSpec) - err := cs.NetworkingV1().NetworkPolicies(ns).Delete(context.TODO(), n, metav1.DeleteOptions{}) - Expect(err).To(HaveOccurred()) + // Delete As owner in the namespace should fails + cs := impersonationClient(tnt.Spec.Owners[0].UserSpec.Name, withDefaultGroups(nil)) + + By(fmt.Sprintf("owner cannot delete netpol"), func() { + obj := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: n, Namespace: ns}} + err := cs.Delete(context.TODO(), obj) + Expect(err).To(HaveOccurred()) + }) + + By(fmt.Sprintf("owner cannot update netpol"), func() { + current := &networkingv1.NetworkPolicy{} + Expect(cs.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, current)).To(Succeed()) + + mut := current.DeepCopy() + mut.Spec.PodSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{ + "something": "custom", + }, + } + err := cs.Update(context.TODO(), mut) + Expect(err).To(HaveOccurred()) + }) c := np.DeepCopy() c.Spec.Egress = []networkingv1.NetworkPolicyEgressRule{} c.Spec.Ingress = []networkingv1.NetworkPolicyIngressRule{} - Expect(k8sClient.Update(context.TODO(), c, &client.UpdateOptions{})).Should(Succeed()) + Expect(k8sClient.Update(context.TODO(), c, &client.UpdateOptions{})).ShouldNot(Succeed()) + + controllerAccount := impersonationClient(ControllerServiceAccountFull, nil) + Expect(controllerAccount.Update(context.TODO(), c, &client.UpdateOptions{})).Should(Succeed()) Eventually(func() networkingv1.NetworkPolicySpec { Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, np)).Should(Succeed()) @@ -230,13 +298,34 @@ var _ = Describe("changing Tenant managed Kubernetes resources", Label("tenant", return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, rq) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) - cs := ownerClient(tnt.Spec.Owners[0].UserSpec) - err := cs.CoreV1().ResourceQuotas(ns).Delete(context.TODO(), n, metav1.DeleteOptions{}) - Expect(err).To(HaveOccurred()) + // Delete As owner in the namespace should fails + cs := impersonationClient(tnt.Spec.Owners[0].UserSpec.Name, withDefaultGroups(nil)) + + By(fmt.Sprintf("owner cannot delete resourcequota"), func() { + obj := &corev1.ResourceQuota{ObjectMeta: metav1.ObjectMeta{Name: n, Namespace: ns}} + err := cs.Delete(context.TODO(), obj) + Expect(err).To(HaveOccurred()) + }) + + By(fmt.Sprintf("owner cannot update resourcequota"), func() { + current := &corev1.ResourceQuota{} + Expect(cs.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, current)).To(Succeed()) + + mut := current.DeepCopy() + mut.SetLabels(map[string]string{ + meta.ManagedByCapsuleLabel: "someone-else", + }) + + err := cs.Update(context.TODO(), mut) + Expect(err).To(HaveOccurred()) + }) c := rq.DeepCopy() c.Spec.Hard = map[corev1.ResourceName]resource.Quantity{} - Expect(k8sClient.Update(context.TODO(), c, &client.UpdateOptions{})).Should(Succeed()) + Expect(k8sClient.Update(context.TODO(), c, &client.UpdateOptions{})).ShouldNot(Succeed()) + + controllerAccount := impersonationClient(ControllerServiceAccountFull, nil) + Expect(controllerAccount.Update(context.TODO(), c, &client.UpdateOptions{})).Should(Succeed()) Eventually(func() corev1.ResourceQuotaSpec { Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, rq)).Should(Succeed()) @@ -247,3 +336,50 @@ var _ = Describe("changing Tenant managed Kubernetes resources", Label("tenant", } }) }) + +func ensureTamperRoleBinding( + ctx context.Context, + k8sClient client.Client, + ns string, + user string, + roleName string, + apiGroup string, + resource string, +) { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleName, + Namespace: ns, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{apiGroup}, + Resources: []string{resource}, + Verbs: []string{"get", "list", "watch", "update", "patch", "delete"}, + }, + }, + } + + Expect(k8sClient.Create(ctx, role)).To(SatisfyAny(Succeed(), WithTransform(apierrors.IsAlreadyExists, BeTrue()))) + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleName, + Namespace: ns, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "Role", + Name: roleName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.UserKind, + APIGroup: rbacv1.GroupName, + Name: user, + }, + }, + } + + Expect(k8sClient.Create(ctx, rb)).To(SatisfyAny(Succeed(), WithTransform(apierrors.IsAlreadyExists, BeTrue()))) +} diff --git a/e2e/tenant_resources_test.go b/e2e/tenant_managed_resources_test.go similarity index 91% rename from e2e/tenant_resources_test.go rename to e2e/tenant_managed_resources_test.go index ed9a0393..58c61665 100644 --- a/e2e/tenant_resources_test.go +++ b/e2e/tenant_managed_resources_test.go @@ -9,6 +9,8 @@ import ( "strings" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -21,17 +23,20 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" ) -var _ = Describe("creating namespaces within a Tenant with resources", Label("tenant"), func() { +var _ = Describe("creating namespaces within a Tenant with resources", Ordered, Label("tenant", "managed"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-resources", + Name: "e2e-tenant-managed-resources", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "john", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-managed-resources", Kind: "User", }, }, @@ -162,16 +167,21 @@ var _ = Describe("creating namespaces within a Tenant with resources", Label("te EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) + By("creating the Namespaces", func() { for _, i := range nsl { - ns := NewNamespace(i) + ns := NewNamespace(i, map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) } }) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should contains all replicated resources", func() { for _, name := range nsl { diff --git a/e2e/tenant_metadata_test.go b/e2e/tenant_metadata_test.go index 0fc195d7..f0476fc4 100644 --- a/e2e/tenant_metadata_test.go +++ b/e2e/tenant_metadata_test.go @@ -12,7 +12,7 @@ import ( "k8s.io/apimachinery/pkg/types" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) func getLabels(tnt capsulev1beta2.Tenant) (map[string]string, error) { @@ -24,20 +24,21 @@ func getLabels(tnt capsulev1beta2.Tenant) (map[string]string, error) { return current.GetLabels(), nil } -var _ = Describe("adding metadata to a Tenant", Label("tenant"), func() { +var _ = Describe("adding metadata to a Tenant", Ordered, Label("tenant"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-metadata", + Name: "e2e-tenant-metadata", Labels: map[string]string{ "custom-label": "test", + "env": "e2e", }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "jim", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-tenant-metadata", Kind: "User", }, }, @@ -49,16 +50,18 @@ var _ = Describe("adding metadata to a Tenant", Label("tenant"), func() { EventuallyCreation(func() error { return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("Should ensure label metadata", func() { By("Default labels", func() { currentlabels, _ := getLabels(*tnt) - Expect(currentlabels["kubernetes.io/metadata.name"]).To(Equal("tenant-metadata")) + Expect(currentlabels["kubernetes.io/metadata.name"]).To(Equal(tnt.GetName())) Expect(currentlabels["custom-label"]).To(Equal("test")) }) By("Disallow name overwrite", func() { diff --git a/e2e/tenant_name_webhook_test.go b/e2e/tenant_name_webhook_test.go index c3ba101e..eccf5366 100644 --- a/e2e/tenant_name_webhook_test.go +++ b/e2e/tenant_name_webhook_test.go @@ -11,19 +11,19 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("creating a Tenant with wrong name", Label("tenant"), func() { +var _ = Describe("creating a Tenant with wrong name", Ordered, Label("tenant"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: "non_rfc_dns_1123", }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: "john", Kind: "User", }, diff --git a/e2e/tenant_protected_webhook_test.go b/e2e/tenant_protected_webhook_test.go index 2c0d6a7e..b0b52c46 100644 --- a/e2e/tenant_protected_webhook_test.go +++ b/e2e/tenant_protected_webhook_test.go @@ -12,21 +12,21 @@ import ( "k8s.io/apimachinery/pkg/types" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -var _ = Describe("Deleting a tenant with protected annotation", Label("tenant"), func() { +var _ = Describe("Deleting a tenant with protected annotation", Ordered, Label("tenant"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "protected-tenant", + Name: "e2e-protected-tenant", }, Spec: capsulev1beta2.TenantSpec{ PreventDeletion: true, - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "john", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-protected-tenant", Kind: "User", }, }, @@ -44,6 +44,7 @@ var _ = Describe("Deleting a tenant with protected annotation", Label("tenant"), It("should fail", func() { Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed()) + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) Expect(k8sClient.Delete(context.TODO(), tnt)).ShouldNot(Succeed()) }) }) diff --git a/e2e/resource_quota_exceeded_test.go b/e2e/tenant_resource_quota_exceeded_test.go similarity index 83% rename from e2e/resource_quota_exceeded_test.go rename to e2e/tenant_resource_quota_exceeded_test.go index b5017c9a..42a19237 100644 --- a/e2e/resource_quota_exceeded_test.go +++ b/e2e/tenant_resource_quota_exceeded_test.go @@ -8,6 +8,8 @@ import ( "fmt" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -21,17 +23,20 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" ) -var _ = Describe("exceeding a Tenant resource quota", Label("resourcequota"), func() { +var _ = Describe("exceeding a Tenant resource quota", Ordered, Label("resourcequota"), func() { tnt := &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-resources-changes", + Name: "e2e-quota-changes", + Labels: map[string]string{ + "env": "e2e", + }, }, Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ + Owners: rbac.OwnerListSpec{ { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "bobby", + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Name: "e2e-quota-changes", Kind: "User", }, }, @@ -116,16 +121,21 @@ var _ = Describe("exceeding a Tenant resource quota", Label("resourcequota"), fu tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) + + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) + By("creating the Namespaces", func() { for _, i := range nsl { - ns := NewNamespace(i) + ns := NewNamespace(i, map[string]string{ + meta.TenantLabel: tnt.GetName(), + }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - TenantNamespaceList(tnt, defaultTimeoutInterval).Should(ContainElement(ns.GetName())) + NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) } }) }) JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + EventuallyDeletion(tnt) }) It("should block new Pods", func() { @@ -150,10 +160,12 @@ var _ = Describe("exceeding a Tenant resource quota", Label("resourcequota"), fu }, }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "my-pause", - Image: "gcr.io/google_containers/pause-amd64:3.0", + Name: "my-pause", + Image: "gcr.io/google_containers/pause-amd64:3.0", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, @@ -184,10 +196,12 @@ var _ = Describe("exceeding a Tenant resource quota", Label("resourcequota"), fu Name: "container", }, Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { - Name: "container", - Image: "quay.io/google-containers/pause-amd64:3.0", + Name: "container", + Image: "gcr.io/google_containers/pause-amd64:3.0", + SecurityContext: restrictedContainerSecurityContext(), }, }, }, diff --git a/e2e/tenantresource_test.go b/e2e/tenantresource_test.go deleted file mode 100644 index 0e6282ab..00000000 --- a/e2e/tenantresource_test.go +++ /dev/null @@ -1,399 +0,0 @@ -// Copyright 2020-2023 Project Capsule Authors. -// SPDX-License-Identifier: Apache-2.0 - -package e2e - -import ( - "context" - "fmt" - "math/rand" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/selection" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" -) - -var _ = Describe("Creating a TenantResource object", Label("tenantresource"), func() { - solar := &capsulev1beta2.Tenant{ - ObjectMeta: metav1.ObjectMeta{ - Name: "energy-solar", - }, - Spec: capsulev1beta2.TenantSpec{ - Owners: api.OwnerListSpec{ - { - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Name: "solar-user", - Kind: "User", - }, - }, - }, - }, - }, - } - - tntItem := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "dummy-secret", - Namespace: "solar-system", - Labels: map[string]string{ - "replicate": "true", - }, - }, - Type: corev1.SecretTypeOpaque, - } - - crossNamespaceItem := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cross-reference-secret", - Namespace: "default", - Labels: map[string]string{ - "replicate": "true", - }, - }, - Type: corev1.SecretTypeOpaque, - } - - testLabels := map[string]string{ - "labels.energy.io": "namespaced", - } - testAnnotations := map[string]string{ - "annotations.energy.io": "namespaced", - } - - tr := &capsulev1beta2.TenantResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "replicate-energies", - Namespace: "solar-system", - }, - Spec: capsulev1beta2.TenantResourceSpec{ - ResyncPeriod: metav1.Duration{Duration: time.Minute}, - PruningOnDelete: ptr.To(true), - Resources: []capsulev1beta2.ResourceSpec{ - { - NamespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "replicate": "true", - }, - }, - NamespacedItems: []capsulev1beta2.ObjectReference{ - { - ObjectReferenceAbstract: capsulev1beta2.ObjectReferenceAbstract{ - Kind: "Secret", - Namespace: "solar-system", - APIVersion: "v1", - }, - Selector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "replicate": "true", - }, - }, - }, - }, - RawItems: []capsulev1beta2.RawExtension{ - { - RawExtension: runtime.RawExtension{ - Object: &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "raw-secret-1", - Labels: testLabels, - Annotations: testAnnotations, - }, - Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{ - "{{ tenant.name }}": []byte("Cg=="), - "{{ namespace }}": []byte("Cg=="), - }, - }, - }, - }, - { - RawExtension: runtime.RawExtension{ - Object: &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "raw-secret-2", - Labels: testLabels, - Annotations: testAnnotations, - }, - Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{ - "{{ tenant.name }}": []byte("Cg=="), - "{{ namespace }}": []byte("Cg=="), - }, - }, - }, - }, - { - RawExtension: runtime.RawExtension{ - Object: &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "raw-secret-3", - Labels: testLabels, - Annotations: testAnnotations, - }, - Type: corev1.SecretTypeOpaque, - Data: map[string][]byte{ - "{{ tenant.name }}": []byte("Cg=="), - "{{ namespace }}": []byte("Cg=="), - }, - }, - }, - }, - }, - AdditionalMetadata: &api.AdditionalMetadataSpec{ - Labels: map[string]string{ - "labels.energy.io": "replicate", - }, - Annotations: map[string]string{ - "annotations.energy.io": "replicate", - }, - }, - }, - }, - }, - } - - JustBeforeEach(func() { - EventuallyCreation(func() error { - return k8sClient.Create(context.TODO(), solar) - }).Should(Succeed()) - - EventuallyCreation(func() error { - return k8sClient.Create(context.TODO(), crossNamespaceItem) - }).Should(Succeed()) - }) - - JustAfterEach(func() { - Expect(k8sClient.Delete(context.TODO(), crossNamespaceItem)).Should(Succeed()) - _ = k8sClient.Delete(context.TODO(), solar) - }) - - It("should replicate resources to all Tenant Namespaces", Label("skip"), func() { - solarNs := []string{"solar-one", "solar-two", "solar-three"} - - By("creating solar Namespaces", func() { - for _, ns := range append(solarNs, "solar-system") { - NamespaceCreation(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, solar.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) - } - }) - - By("labelling Namespaces", func() { - for _, name := range []string{"solar-one", "solar-two", "solar-three"} { - EventuallyWithOffset(1, func() error { - ns := corev1.Namespace{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: name}, &ns)).Should(Succeed()) - - labels := ns.GetLabels() - if labels == nil { - return fmt.Errorf("missing labels") - } - labels["replicate"] = "true" - ns.SetLabels(labels) - - return k8sClient.Update(context.TODO(), &ns) - }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) - } - }) - - By("creating the namespaced item", func() { - EventuallyCreation(func() error { - return k8sClient.Create(context.TODO(), tntItem) - }).Should(Succeed()) - }) - - By("creating the TenantResource", func() { - EventuallyCreation(func() error { - return k8sClient.Create(context.TODO(), tr) - }).Should(Succeed()) - }) - - for _, ns := range solarNs { - By(fmt.Sprintf("waiting for replicated resources in %s Namespace", ns), func() { - Eventually(func() []corev1.Secret { - r, err := labels.NewRequirement("labels.energy.io", selection.DoubleEquals, []string{"replicate"}) - if err != nil { - return nil - } - - secrets := corev1.SecretList{} - err = k8sClient.List(context.TODO(), &secrets, &client.ListOptions{LabelSelector: labels.NewSelector().Add(*r), Namespace: ns}) - if err != nil { - return nil - } - - return secrets.Items - }, defaultTimeoutInterval, defaultPollInterval).Should(HaveLen(4)) - }) - - By(fmt.Sprintf("ensuring raw items are templated in %s Namespace", ns), func() { - for _, name := range []string{"raw-secret-1", "raw-secret-2", "raw-secret-3"} { - secret := corev1.Secret{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: ns}, &secret)).ToNot(HaveOccurred()) - - Expect(secret.Data).To(HaveKey(solar.Name)) - Expect(secret.Data).To(HaveKey(ns)) - } - }) - } - - By("using a Namespace selector", func() { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tr.GetName(), Namespace: "solar-system"}, tr)).ToNot(HaveOccurred()) - - tr.Spec.Resources[0].NamespaceSelector = &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "kubernetes.io/metadata.name": "solar-three", - }, - } - - Expect(k8sClient.Update(context.TODO(), tr)).ToNot(HaveOccurred()) - - checkFn := func(ns string) func() []corev1.Secret { - return func() []corev1.Secret { - r, err := labels.NewRequirement("labels.energy.io", selection.DoubleEquals, []string{"replicate"}) - if err != nil { - return nil - } - - secrets := corev1.SecretList{} - err = k8sClient.List(context.TODO(), &secrets, &client.ListOptions{LabelSelector: labels.NewSelector().Add(*r), Namespace: ns}) - if err != nil { - return nil - } - - return secrets.Items - } - } - - for _, ns := range []string{"solar-one", "solar-two"} { - Eventually(checkFn(ns), defaultTimeoutInterval, defaultPollInterval).Should(HaveLen(0)) - } - - Eventually(checkFn("solar-three"), defaultTimeoutInterval, defaultPollInterval).Should(HaveLen(4)) - }) - - By("checking if replicated object have annotations and labels", func() { - for _, name := range []string{"dummy-secret", "raw-secret-1", "raw-secret-2", "raw-secret-3"} { - secret := corev1.Secret{} - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: "solar-three"}, &secret)).ToNot(HaveOccurred()) - - for k, v := range tr.Spec.Resources[0].AdditionalMetadata.Labels { - _, err := HaveKeyWithValue(k, v).Match(secret.GetLabels()) - Expect(err).ToNot(HaveOccurred()) - } - for k, v := range testLabels { - _, err := HaveKeyWithValue(k, v).Match(secret.GetLabels()) - Expect(err).ToNot(HaveOccurred()) - } - for k, v := range tr.Spec.Resources[0].AdditionalMetadata.Annotations { - _, err := HaveKeyWithValue(k, v).Match(secret.GetAnnotations()) - Expect(err).ToNot(HaveOccurred()) - } - for k, v := range testAnnotations { - _, err := HaveKeyWithValue(k, v).Match(secret.GetAnnotations()) - Expect(err).ToNot(HaveOccurred()) - } - } - }) - - By("checking replicated object cannot be deleted by a Tenant Owner", func() { - for _, name := range []string{"dummy-secret", "raw-secret-1", "raw-secret-2", "raw-secret-3"} { - cs := ownerClient(solar.Spec.Owners[0].UserSpec) - - Consistently(func() error { - return cs.CoreV1().Secrets("solar-three").Delete(context.TODO(), name, metav1.DeleteOptions{}) - }, 10*time.Second, time.Second).Should(HaveOccurred()) - } - }) - - By("checking replicated object cannot be update by a Tenant Owner", func() { - for _, name := range []string{"dummy-secret", "raw-secret-1", "raw-secret-2", "raw-secret-3"} { - cs := ownerClient(solar.Spec.Owners[0].UserSpec) - - Consistently(func() error { - secret, err := cs.CoreV1().Secrets("solar-three").Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil { - return err - } - - secret.SetLabels(nil) - secret.SetAnnotations(nil) - - _, err = cs.CoreV1().Secrets("solar-three").Update(context.TODO(), secret, metav1.UpdateOptions{}) - - return err - }, 10*time.Second, time.Second).Should(HaveOccurred()) - } - }) - - By("checking that cross-namespace objects are not replicated", func() { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tr.GetName(), Namespace: "solar-system"}, tr)).ToNot(HaveOccurred()) - tr.Spec.Resources[0].NamespacedItems = append(tr.Spec.Resources[0].NamespacedItems, capsulev1beta2.ObjectReference{ - ObjectReferenceAbstract: capsulev1beta2.ObjectReferenceAbstract{ - Kind: crossNamespaceItem.Kind, - Namespace: crossNamespaceItem.GetName(), - APIVersion: crossNamespaceItem.APIVersion, - }, - Selector: metav1.LabelSelector{ - MatchLabels: crossNamespaceItem.GetLabels(), - }, - }) - - Expect(k8sClient.Update(context.TODO(), tr)).ToNot(HaveOccurred()) - // Ensuring that although the deletion of TenantResource object, - // the replicated objects are not deleted. - Consistently(func() error { - return k8sClient.Get(context.TODO(), types.NamespacedName{Namespace: solarNs[rand.Intn(len(solarNs))], Name: crossNamespaceItem.GetName()}, &corev1.Secret{}) - }, 10*time.Second, time.Second).Should(HaveOccurred()) - }) - - By("checking pruning is deleted", func() { - Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tr.GetName(), Namespace: "solar-system"}, tr)).ToNot(HaveOccurred()) - Expect(*tr.Spec.PruningOnDelete).Should(BeTrue()) - - tr.Spec.PruningOnDelete = ptr.To(false) - - Expect(k8sClient.Update(context.TODO(), tr)).ToNot(HaveOccurred()) - - By("deleting the TenantResource", func() { - // Ensuring that although the deletion of TenantResource object, - // the replicated objects are not deleted. - Expect(k8sClient.Delete(context.TODO(), tr)).Should(Succeed()) - - r, err := labels.NewRequirement("labels.energy.io", selection.DoubleEquals, []string{"replicate"}) - Expect(err).ToNot(HaveOccurred()) - - Consistently(func() []corev1.Secret { - secrets := corev1.SecretList{} - - err = k8sClient.List(context.TODO(), &secrets, &client.ListOptions{LabelSelector: labels.NewSelector().Add(*r), Namespace: "solar-three"}) - Expect(err).ToNot(HaveOccurred()) - - return secrets.Items - }, 10*time.Second, time.Second).Should(HaveLen(4)) - }) - }) - }) -}) diff --git a/e2e/suite_client_test.go b/e2e/utils_suite_client_test.go similarity index 97% rename from e2e/suite_client_test.go rename to e2e/utils_suite_client_test.go index 0594bcd8..73769e68 100644 --- a/e2e/suite_client_test.go +++ b/e2e/utils_suite_client_test.go @@ -15,7 +15,7 @@ type e2eClient struct { } func (e *e2eClient) sleep() { - time.Sleep(250 * time.Millisecond) + time.Sleep(50 * time.Millisecond) } func (e *e2eClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { diff --git a/e2e/utils_test.go b/e2e/utils_test.go index e75798e4..e7d04d15 100644 --- a/e2e/utils_test.go +++ b/e2e/utils_test.go @@ -13,31 +13,59 @@ import ( "time" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" . "github.com/onsi/gomega" "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + nodev1 "k8s.io/api/node/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" versionUtil "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/util/retry" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" ) const ( - defaultTimeoutInterval = 40 * time.Second - defaultPollInterval = time.Second - defaultConfigurationName = "default" + defaultTimeoutInterval = 60 * time.Second + defaultTerminationTimeoutInterval = 60 * time.Second + defaultPollInterval = 2 * time.Second + defaultConfigurationName = "default" + e2eClientQPS float32 = 1000 + e2eClientBurst int = 2000 ) +func tuneE2ERestConfig(c *rest.Config) *rest.Config { + c.QPS = e2eClientQPS + c.Burst = e2eClientBurst + + return c +} + +func mergeMaps(base map[string]string, extra map[string]string) map[string]string { + out := map[string]string{} + for k, v := range base { + out[k] = v + } + for k, v := range extra { + out[k] = v + } + return out +} + func ignoreNotFound(err error) error { if apierrors.IsNotFound(err) { return nil @@ -59,7 +87,7 @@ func NewService(svc types.NamespacedName) *corev1.Service { } } -func ServiceCreation(svc *corev1.Service, owner api.UserSpec, timeout time.Duration) AsyncAssertion { +func ServiceCreation(svc *corev1.Service, owner rbac.UserSpec, timeout time.Duration) AsyncAssertion { cs := ownerClient(owner) return Eventually(func() (err error) { _, err = cs.CoreV1().Services(svc.Namespace).Create(context.TODO(), svc, metav1.CreateOptions{}) @@ -92,7 +120,59 @@ func NewNamespace(name string, labels ...map[string]string) *corev1.Namespace { } } -func NamespaceCreation(ns *corev1.Namespace, owner api.UserSpec, timeout time.Duration) AsyncAssertion { +func NamespaceCreationAdmin(ns *corev1.Namespace, timeout time.Duration) AsyncAssertion { + return Eventually(func() (err error) { + return k8sClient.Create( + context.TODO(), + ns, + ) + }, timeout, defaultPollInterval) +} + +func NamespaceDeletionAdmin(ns *corev1.Namespace, timeout time.Duration) AsyncAssertion { + return Eventually(func() (err error) { + return k8sClient.Delete( + context.TODO(), + ns, + ) + }, timeout, defaultPollInterval) +} + +func ForceDeleteNamespace(ctx context.Context, name string) { + Eventually(func() error { + ns := &corev1.Namespace{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, ns) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + + // Trigger deletion if not already happening + if ns.DeletionTimestamp.IsZero() { + if err := k8sClient.Delete(ctx, ns); err != nil && !apierrors.IsNotFound(err) { + return err + } + return fmt.Errorf("namespace %s deletion triggered", name) + } + + // Force-remove finalizers (THIS is the key part) + if len(ns.Finalizers) > 0 { + ns.Finalizers = nil + if err := k8sClient.Update(ctx, ns); err != nil { + return err + } + return fmt.Errorf("namespace %s finalizers removed", name) + } + + // wait until fully gone + return fmt.Errorf("namespace %s still terminating", name) + }, defaultTerminationTimeoutInterval, defaultPollInterval).Should(Succeed(), + "failed to force delete namespace %s", name) +} + +func NamespaceCreation(ns *corev1.Namespace, owner rbac.UserSpec, timeout time.Duration) AsyncAssertion { cs := ownerClient(owner) return Eventually(func() (err error) { _, err = cs.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) @@ -100,12 +180,64 @@ func NamespaceCreation(ns *corev1.Namespace, owner api.UserSpec, timeout time.Du }, timeout, defaultPollInterval) } -func NamespaceIsPartOfTenant( +func TenantNamespaceReady( tnt *capsulev1beta2.Tenant, ns *corev1.Namespace, -) func() error { + expectedSize uint, +) { + Eventually(func(g Gomega) { + t := &capsulev1beta2.Tenant{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(t.Status.Size).To( + Equal(expectedSize), + "expected tenant %s status size to be %d, got %d", + t.GetName(), + expectedSize, + t.Status.Size, + ) + + currentNS := &corev1.Namespace{} + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, currentNS) + g.Expect(err).NotTo(HaveOccurred()) + + instance := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ + Name: currentNS.GetName(), + UID: currentNS.GetUID(), + }) + g.Expect(instance).NotTo(BeNil(), "Namespace instance should not be nil") + + condition := instance.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil(), "Condition instance should not be nil") + + g.Expect(instance.Name).To(Equal(currentNS.GetName())) + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue), "Expected namespace condition status to be True") + g.Expect(condition.Type).To(Equal(meta.ReadyCondition), "Expected namespace condition type to be Ready") + g.Expect(condition.Reason).To(Equal(meta.SucceededReason), "Expected namespace condition reason to be Succeeded") + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func NamespaceIsNotPartOfTenant( + tnt *capsulev1beta2.Tenant, + ns *corev1.Namespace, +) AsyncAssertion { + return Eventually(func() error { + currentNS := &corev1.Namespace{} + nsUID := ns.GetUID() + + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: ns.GetName()}, + currentNS, + ); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to get namespace %s: %w", ns.GetName(), err) + } + } else { + nsUID = currentNS.GetUID() + } - return func() error { t := &capsulev1beta2.Tenant{} if err := k8sClient.Get( context.TODO(), @@ -115,31 +247,84 @@ func NamespaceIsPartOfTenant( return fmt.Errorf("failed to get tenant: %w", err) } - // reuse existing helper - namespaces := TenantNamespaceList(t, defaultTimeoutInterval) - if ok, _ := ContainElements(ns.GetName()).Match(namespaces); ok { + namespaces := t.Status.Namespaces + if ok, _ := ContainElement(ns.GetName()).Match(namespaces); ok { return fmt.Errorf( - "expected tenant %s to contain namespace %s, but got: %v", - t.GetName(), ns.GetName(), namespaces, + "expected tenant %s not to contain namespace %s, but got: %v", + t.GetName(), + ns.GetName(), + namespaces, ) } - // reuse your existing method - instance := t.Status.GetInstance( - &capsulev1beta2.TenantStatusNamespaceItem{ - Name: ns.GetName(), - UID: ns.GetUID(), - }) - - if instance == nil { + instance := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ + Name: ns.GetName(), + UID: nsUID, + }) + if instance != nil { return fmt.Errorf( - "tenant %s does not contain instance for namespace %s (uid=%s)", - t.GetName(), ns.GetName(), ns.GetUID(), + "expected tenant %s not to contain instance for namespace %s (uid=%s), but got: %+v", + t.GetName(), + ns.GetName(), + nsUID, + instance, ) } return nil - } + }, defaultTimeoutInterval, defaultPollInterval) +} + +func NamespaceIsPartOfTenant( + tnt *capsulev1beta2.Tenant, + ns *corev1.Namespace, +) AsyncAssertion { + return Eventually(func() error { + currentNS := &corev1.Namespace{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: ns.GetName()}, + currentNS, + ); err != nil { + return fmt.Errorf("failed to get namespace %s: %w", ns.GetName(), err) + } + + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get( + context.TODO(), + types.NamespacedName{Name: tnt.GetName()}, + t, + ); err != nil { + return fmt.Errorf("failed to get tenant: %w", err) + } + + namespaces := t.Status.Namespaces + if ok, _ := ContainElement(currentNS.GetName()).Match(namespaces); !ok { + return fmt.Errorf( + "expected tenant %s to contain namespace %s, but got: %v", + t.GetName(), + currentNS.GetName(), + namespaces, + ) + } + + instance := t.Status.GetInstance( + &capsulev1beta2.TenantStatusNamespaceItem{ + Name: currentNS.GetName(), + UID: currentNS.GetUID(), + }, + ) + if instance == nil { + return fmt.Errorf( + "expected tenant %s to contain instance for namespace %s (uid=%s)", + t.GetName(), + currentNS.GetName(), + currentNS.GetUID(), + ) + } + + return nil + }, defaultTimeoutInterval, defaultPollInterval) } func GetTenantOwnerReference( @@ -255,6 +440,66 @@ func PatchTenantOwnerReferenceForNamespace( }, timeout, defaultPollInterval) } +func GetTenantEventually(tnt *capsulev1beta2.Tenant) *capsulev1beta2.Tenant { + t := &capsulev1beta2.Tenant{} + + Eventually(func() error { + return k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + return t +} + +func GetNamespaceEventually(name string) *corev1.Namespace { + ns := &corev1.Namespace{} + + Eventually(func() error { + return k8sClient.Get(context.TODO(), types.NamespacedName{Name: name}, ns) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + + return ns +} + +func UpdateTenantEventually(tnt *capsulev1beta2.Tenant, mutator func(*capsulev1beta2.Tenant)) { + Eventually(func() error { + current := &capsulev1beta2.Tenant{} + if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, current); err != nil { + return err + } + + mutator(current) + + return k8sClient.Update(context.TODO(), current) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func UpdateTenantEventuallyShouldFail(tnt *capsulev1beta2.Tenant, mutator func(*capsulev1beta2.Tenant)) { + Eventually(func() error { + current := &capsulev1beta2.Tenant{} + if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, current); err != nil { + return err + } + + mutator(current) + + return k8sClient.Update(context.TODO(), current) + }, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed()) +} + +func PatchNamespaceEventually(ns *corev1.Namespace, mutator func(*corev1.Namespace)) { + Eventually(func() error { + current := &corev1.Namespace{} + if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: ns.GetName()}, current); err != nil { + return err + } + + before := current.DeepCopy() + mutator(current) + + return k8sClient.Patch(context.TODO(), current, client.MergeFrom(before)) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + func TenantNamespaceList(tnt *capsulev1beta2.Tenant, timeout time.Duration) AsyncAssertion { t := &capsulev1beta2.Tenant{} return Eventually(func() []string { @@ -275,16 +520,47 @@ func EventuallyCreation(f interface{}) AsyncAssertion { return Eventually(f, defaultTimeoutInterval, defaultPollInterval) } -func ModifyCapsuleConfigurationOpts(fn func(configuration *capsulev1beta2.CapsuleConfiguration)) { - config := &capsulev1beta2.CapsuleConfiguration{} - Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: defaultConfigurationName}, config)).ToNot(HaveOccurred()) +func EventuallyDeletion(obj client.Object) { + key := client.ObjectKeyFromObject(obj) - fn(config) + Eventually(func() error { + // Retry delete until the object is really gone. + err := k8sClient.Delete(context.TODO(), obj) + if err != nil && !apierrors.IsNotFound(err) { + return err + } - Expect(k8sClient.Update(context.Background(), config)).ToNot(HaveOccurred()) + // Read into a fresh copy to avoid stale in-memory state. + current := obj.DeepCopyObject().(client.Object) + err = k8sClient.Get(context.TODO(), key, current) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + + return fmt.Errorf("%T %q still exists", obj, obj.GetName()) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } -func CheckForOwnerRoleBindings(ns *corev1.Namespace, owner api.OwnerSpec, roles map[string]bool) func() error { +func ModifyCapsuleConfigurationOpts(fn func(configuration *capsulev1beta2.CapsuleConfiguration)) { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + config := &capsulev1beta2.CapsuleConfiguration{} + + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: defaultConfigurationName}, config); err != nil { + return err + } + + fn(config) + + return k8sClient.Update(context.Background(), config) + }) + + Expect(err).ToNot(HaveOccurred()) +} + +func CheckForOwnerRoleBindings(ns *corev1.Namespace, owner rbac.OwnerSpec, roles map[string]bool) func() error { if roles == nil { roles = map[string]bool{ "admin": false, @@ -301,7 +577,7 @@ func CheckForOwnerRoleBindings(ns *corev1.Namespace, owner api.OwnerSpec, roles var ownerName string - if owner.Kind == api.ServiceAccountOwner { + if owner.Kind == rbac.ServiceAccountOwner { parts := strings.Split(owner.Name, ":") ownerName = parts[3] @@ -373,13 +649,13 @@ func VerifyTenantRoleBindings( }).WithTimeout(30 * time.Second).WithPolling(500 * time.Millisecond).Should(Succeed()) } -func normalizeOwners(in api.OwnerStatusListSpec) api.OwnerStatusListSpec { +func normalizeOwners(in rbac.OwnerStatusListSpec) rbac.OwnerStatusListSpec { // copy to avoid mutating the original - out := make(api.OwnerStatusListSpec, len(in)) + out := make(rbac.OwnerStatusListSpec, len(in)) copy(out, in) // sort outer slice by kind+name - sort.Sort(api.GetByKindAndName(out)) + sort.Sort(rbac.GetByKindAndName(out)) // sort roles inside each owner so role order doesn't matter for i := range out { @@ -389,6 +665,67 @@ func normalizeOwners(in api.OwnerStatusListSpec) api.OwnerStatusListSpec { return out } +func EnsureServiceAccount(ctx context.Context, c client.Client, name string, namespace string) { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + + err := c.Create(ctx, sa) + if err != nil && !apierrors.IsAlreadyExists(err) { + Expect(err).ToNot(HaveOccurred()) + } +} + +func EnsureRoleAndBindingForNamespaces(ctx context.Context, c client.Client, saName string, saNamespace string, namespaces []string) { + for _, ns := range namespaces { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName + "-" + saNamespace, + Namespace: ns, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + }, + } + + err := c.Create(ctx, role) + if err != nil && !apierrors.IsAlreadyExists(err) { + Expect(err).ToNot(HaveOccurred()) + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName + "-" + saNamespace, + Namespace: ns, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: saName, + Namespace: saNamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: saName + "-" + saNamespace, + }, + } + + err = c.Create(ctx, rb) + if err != nil && !apierrors.IsAlreadyExists(err) { + Expect(err).ToNot(HaveOccurred()) + } + } +} + func GetKubernetesVersion() *versionUtil.Version { var serverVersion *version.Info var err error @@ -463,7 +800,7 @@ func GrantEphemeralContainersUpdate(ns string, username string) (cleanup func()) // Give RBAC a moment to propagate in the apiserver authorizer cache Eventually(func() error { - cs := ownerClient(api.UserSpec{Name: username, Kind: "User"}) + cs := ownerClient(rbac.UserSpec{Name: username, Kind: "User"}) _, err := cs.CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{Limit: 1}) return err }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) @@ -528,3 +865,262 @@ type dummyT struct { func (d *dummyT) Errorf(format string, args ...interface{}) { d.errors = append(d.errors, fmt.Sprintf(format, args...)) } + +func MakePod(namespace, name string, labels map[string]string, annotations map[string]string, image string, cpuRequest string, emptyDirSize string) *corev1.Pod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + Containers: []corev1.Container{ + { + Name: "main", + Image: image, + SecurityContext: restrictedContainerSecurityContext(), + }, + }, + RestartPolicy: corev1.RestartPolicyAlways, + }, + } + + if cpuRequest != "" { + pod.Spec.Containers[0].Resources.Requests = corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(cpuRequest), + } + } + + if emptyDirSize != "" { + pod.Spec.Volumes = []corev1.Volume{ + { + Name: "cache", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + SizeLimit: ptr.To(resource.MustParse(emptyDirSize)), + }, + }, + }, + } + pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ + { + Name: "cache", + MountPath: "/cache", + }, + } + } + + return pod +} + +func MakeDeployment(namespace, name string, replicas int32, labels map[string]string, cpuRequest string) *appsv1.Deployment { + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(replicas), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": name, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: mergeMaps(map[string]string{"app": name}, labels), + }, + Spec: corev1.PodSpec{ + SecurityContext: nobodyPodSecurityContext(), + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx:1.27.0", + SecurityContext: restrictedContainerSecurityContext(), + }, + }, + }, + }, + }, + } + + if cpuRequest != "" { + dep.Spec.Template.Spec.Containers[0].Resources.Requests = corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(cpuRequest), + } + } + + return dep +} + +func ExpectPodsForDeployment(ctx context.Context, namespace, app string, expected int) { + Eventually(func(g Gomega) { + pods := &corev1.PodList{} + + g.Expect(k8sClient.List(ctx, pods, + client.InNamespace(namespace), + client.MatchingLabels{ + "app": app, + }, + )).To(Succeed()) + + g.Expect(len(pods.Items)).To(Equal(expected), + "unexpected pod count for deployment app=%q in namespace %s", + app, + namespace, + ) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func nobodyPodSecurityContext() *corev1.PodSecurityContext { + return &corev1.PodSecurityContext{ + RunAsNonRoot: ptr.To(true), + RunAsUser: ptr.To[int64](65534), + RunAsGroup: ptr.To[int64](65534), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + } +} + +func restrictedContainerSecurityContext() *corev1.SecurityContext { + return &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{ + "ALL", + }, + }, + } +} + +func MakePVC(namespace, name, size string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(size), + }, + }, + }, + } +} + +func ScaleDeployment(ctx context.Context, namespace, name string, replicas int32) { + Eventually(func() error { + dep := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, dep); err != nil { + return err + } + dep.Spec.Replicas = ptr.To(replicas) + return k8sClient.Update(ctx, dep) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func UpdatePodLabels(ctx context.Context, namespace, name string, labels map[string]string) { + Eventually(func() error { + pod := &corev1.Pod{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, pod); err != nil { + return err + } + pod.Labels = labels + return k8sClient.Update(ctx, pod) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func UpdatePodImage(ctx context.Context, namespace, name, image string) { + Eventually(func() error { + pod := &corev1.Pod{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, pod); err != nil { + return err + } + pod.Spec.Containers[0].Image = image + return k8sClient.Update(ctx, pod) + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func TenantReadyTrue(tnt *capsulev1beta2.Tenant) { + TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) +} + +func TenantReadyFalse(tnt *capsulev1beta2.Tenant) { + TenantReady(tnt, metav1.ConditionFalse, defaultTimeoutInterval) +} + +func TenantReady( + tnt *capsulev1beta2.Tenant, + expected metav1.ConditionStatus, + timeoutInterval time.Duration, +) { + Eventually(func(g Gomega) { + current := &capsulev1beta2.Tenant{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: tnt.GetName()}, current) + g.Expect(err).NotTo(HaveOccurred()) + + condition := current.Status.Conditions.GetConditionByType(meta.ReadyCondition) + g.Expect(condition).NotTo(BeNil(), "expected Tenant %q to have Ready condition", tnt.GetName()) + + g.Expect(condition.Status).To( + Equal(expected), + "expected Tenant %q Ready condition to be %s, got %s: reason=%q message=%q", + tnt.GetName(), + expected, + condition.Status, + condition.Reason, + condition.Message, + ) + + for _, owner := range current.Spec.Owners { + g.Expect(current.Status.Owners).To( + ContainElement(owner.CoreOwnerSpec), + "expected Tenant %q status.owners to contain spec owner %+v; current status.owners=%+v", + tnt.GetName(), + owner.CoreOwnerSpec, + current.Status.Owners, + ) + } + }, timeoutInterval, defaultPollInterval).Should(Succeed()) +} + +func EnsureRuntimeClass(ctx context.Context, rtc *nodev1.RuntimeClass) { + Eventually(func() error { + desired := rtc.DeepCopy() + desired.ResourceVersion = "" + + _, err := controllerutil.CreateOrUpdate(ctx, k8sClient, desired, func() error { + desired.Handler = rtc.Handler + desired.Overhead = rtc.Overhead + desired.Scheduling = rtc.Scheduling + + labels := desired.GetLabels() + if labels == nil { + labels = map[string]string{} + } + for key, value := range rtc.GetLabels() { + labels[key] = value + } + labels["env"] = "e2e" + desired.SetLabels(labels) + + annotations := desired.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + for key, value := range rtc.GetAnnotations() { + annotations[key] = value + } + desired.SetAnnotations(annotations) + + return nil + }) + + return err + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) +} diff --git a/go.mod b/go.mod index 77bda2b3..80ec11b4 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,13 @@ module github.com/projectcapsule/capsule -go 1.25.4 +go 1.26.3 require ( + github.com/BurntSushi/toml v1.6.0 + github.com/fluxcd/pkg/apis/kustomize v1.15.0 + github.com/fluxcd/pkg/ssa v0.64.0 github.com/go-logr/logr v1.4.3 + github.com/go-sprout/sprout v1.0.3 github.com/onsi/ginkgo/v2 v2.27.5 github.com/onsi/gomega v1.39.0 github.com/pkg/errors v0.9.1 @@ -13,34 +17,41 @@ require ( github.com/valyala/fasttemplate v1.2.2 go.uber.org/automaxprocs v1.6.0 go.uber.org/zap v1.27.1 - golang.org/x/sync v0.19.0 - k8s.io/api v0.35.0 - k8s.io/apiextensions-apiserver v0.35.0 - k8s.io/apimachinery v0.35.0 - k8s.io/apiserver v0.35.0 - k8s.io/client-go v0.35.0 + golang.org/x/sync v0.20.0 + gomodules.xyz/jsonpatch/v2 v2.5.0 + k8s.io/api v0.35.5 + k8s.io/apiextensions-apiserver v0.35.5 + k8s.io/apimachinery v0.35.5 + k8s.io/apiserver v0.35.5 + k8s.io/client-go v0.35.5 k8s.io/utils v0.0.0-20260108192941-914a6e750570 sigs.k8s.io/cluster-api v1.12.2 sigs.k8s.io/controller-runtime v0.23.0 sigs.k8s.io/gateway-api v1.4.1 + sigs.k8s.io/yaml v1.6.0 ) require ( + cel.dev/expr v0.25.1 // indirect dario.cat/mergo v1.0.2 // indirect - github.com/BurntSushi/toml v1.6.0 // indirect + filippo.io/age v1.3.1 // indirect + filippo.io/hpke v0.4.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chai2010/gettext-go v1.0.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fluxcd/cli-utils v0.37.1-flux.1 // indirect - github.com/fluxcd/pkg/apis/kustomize v1.15.0 // indirect - github.com/fluxcd/pkg/ssa v0.64.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-errors/errors v1.5.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.22.4 // indirect github.com/go-openapi/jsonreference v0.21.4 // indirect @@ -56,17 +67,20 @@ require ( github.com/go-openapi/swag/stringutils v0.25.4 // indirect github.com/go-openapi/swag/typeutils v0.25.4 // indirect github.com/go-openapi/swag/yamlutils v0.25.4 // indirect - github.com/go-sprout/sprout v1.0.3 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gobuffalo/flect v1.0.3 // indirect github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.26.1 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect @@ -76,39 +90,53 @@ require ( github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect - github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/match v1.2.0 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/wI2L/jsondiff v0.6.1 // indirect + github.com/wI2L/jsondiff v0.7.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xlab/treeprint v1.2.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.opentelemetry.io/proto/otlp v1.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.47.0 // indirect + golang.org/x/crypto v0.51.0 // indirect golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect - golang.org/x/mod v0.31.0 // indirect - golang.org/x/net v0.49.0 // indirect - golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sys v0.40.0 // indirect - golang.org/x/term v0.39.0 // indirect - golang.org/x/text v0.33.0 // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/term v0.43.0 // indirect + golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.14.0 // indirect - golang.org/x/tools v0.40.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect + golang.org/x/tools v0.44.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.81.1 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/cli-runtime v0.35.0 // indirect + k8s.io/component-base v0.35.5 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect - sigs.k8s.io/kustomize/api v0.21.0 // indirect - sigs.k8s.io/kustomize/kyaml v0.21.0 // indirect + sigs.k8s.io/kustomize/api v0.21.1 // indirect + sigs.k8s.io/kustomize/kyaml v0.21.1 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 954a9da9..51250c7c 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,15 @@ cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= -dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= -dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +filippo.io/age v1.3.1 h1:hbzdQOJkuaMEpRCLSN1/C5DX74RPcNCk6oqhKMXmZi0= +filippo.io/age v1.3.1/go.mod h1:EZorDTYUxt836i3zdori5IJX/v2Lj6kWFU0cfh6C0D4= +filippo.io/hpke v0.4.0 h1:p575VVQ6ted4pL+it6M00V/f2qTZITO0zgmdKCkd5+A= +filippo.io/hpke v0.4.0/go.mod h1:EmAN849/P3qdeK+PCMkDpDm83vRHM5cDipBJ8xbQLVY= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= @@ -25,10 +31,13 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v1.0.3 h1:9liNh8t+u26xl5ddmWLmsOsdNLwkdRTg5AG+JnTiM80= +github.com/chai2010/gettext-go v1.0.3/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0= github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4= github.com/coredns/corefile-migration v1.0.29 h1:g4cPYMXXDDs9uLE2gFYrJaPBuUAR07eEMGyh9JBE13w= github.com/coredns/corefile-migration v1.0.29/go.mod h1:56DPqONc3njpVPsdilEnfijCwNGC3/kTJLl7i7SPavY= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -41,6 +50,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fluxcd/cli-utils v0.37.1-flux.1 h1:WnG2mHxCPZMj/soIq/S/1zvbrGCJN3GJGbNfG06X55M= @@ -49,6 +60,8 @@ github.com/fluxcd/pkg/apis/kustomize v1.15.0 h1:p8wPIxdmn0vy0a664rsE9JKCfnliZz4H github.com/fluxcd/pkg/apis/kustomize v1.15.0/go.mod h1:XWdsx8P15OiMaQIvmUjYWdmD3zAwhl5q9osl5iCqcOk= github.com/fluxcd/pkg/ssa v0.64.0 h1:B/8VYMIYMeRmolup2HOoWNqXh4UeXi6w2LvXXvl6MZM= github.com/fluxcd/pkg/ssa v0.64.0/go.mod h1:RjvVjJIoRo1ecsv91yMuiqzO6cpNag80M6MOB/vrJdc= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= @@ -61,69 +74,41 @@ github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01 github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.22.3 h1:dKMwfV4fmt6Ah90zloTbUKWMD+0he+12XYAsPotrkn8= -github.com/go-openapi/jsonpointer v0.22.3/go.mod h1:0lBbqeRsQ5lIanv3LHZBrmRGHLHcQoOXQnf88fHlGWo= github.com/go-openapi/jsonpointer v0.22.4 h1:dZtK82WlNpVLDW2jlA1YCiVJFVqkED1MegOUy9kR5T4= github.com/go-openapi/jsonpointer v0.22.4/go.mod h1:elX9+UgznpFhgBuaMQ7iu4lvvX1nvNsesQ3oxmYTw80= -github.com/go-openapi/jsonreference v0.21.3 h1:96Dn+MRPa0nYAR8DR1E03SblB5FJvh7W6krPI0Z7qMc= -github.com/go-openapi/jsonreference v0.21.3/go.mod h1:RqkUP0MrLf37HqxZxrIAtTWW4ZJIK1VzduhXYBEeGc4= github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmGjjySRCHtC8= github.com/go-openapi/jsonreference v0.21.4/go.mod h1:rIENPTjDbLpzQmQWCj5kKj3ZlmEh+EFVbz3RTUh30/4= -github.com/go-openapi/swag v0.25.3 h1:FAa5wJXyDtI7yUztKDfZxDrSx+8WTg31MfCQ9s3PV+s= -github.com/go-openapi/swag v0.25.3/go.mod h1:tX9vI8Mj8Ny+uCEk39I1QADvIPI7lkndX4qCsEqhkS8= github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= -github.com/go-openapi/swag/cmdutils v0.25.3 h1:EIwGxN143JCThNHnqfqs85R8lJcJG06qjJRZp3VvjLI= -github.com/go-openapi/swag/cmdutils v0.25.3/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= -github.com/go-openapi/swag/conv v0.25.3 h1:PcB18wwfba7MN5BVlBIV+VxvUUeC2kEuCEyJ2/t2X7E= -github.com/go-openapi/swag/conv v0.25.3/go.mod h1:n4Ibfwhn8NJnPXNRhBO5Cqb9ez7alBR40JS4rbASUPU= github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= -github.com/go-openapi/swag/fileutils v0.25.3 h1:P52Uhd7GShkeU/a1cBOuqIcHMHBrA54Z2t5fLlE85SQ= -github.com/go-openapi/swag/fileutils v0.25.3/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= -github.com/go-openapi/swag/jsonname v0.25.3 h1:U20VKDS74HiPaLV7UZkztpyVOw3JNVsit+w+gTXRj0A= -github.com/go-openapi/swag/jsonname v0.25.3/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= -github.com/go-openapi/swag/jsonutils v0.25.3 h1:kV7wer79KXUM4Ea4tBdAVTU842Rg6tWstX3QbM4fGdw= -github.com/go-openapi/swag/jsonutils v0.25.3/go.mod h1:ILcKqe4HC1VEZmJx51cVuZQ6MF8QvdfXsQfiaCs0z9o= github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.3 h1:/i3E9hBujtXfHy91rjtwJ7Fgv5TuDHgnSrYjhFxwxOw= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.3/go.mod h1:8kYfCR2rHyOj25HVvxL5Nm8wkfzggddgjZm6RgjT8Ao= github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= -github.com/go-openapi/swag/loading v0.25.3 h1:Nn65Zlzf4854MY6Ft0JdNrtnHh2bdcS/tXckpSnOb2Y= -github.com/go-openapi/swag/loading v0.25.3/go.mod h1:xajJ5P4Ang+cwM5gKFrHBgkEDWfLcsAKepIuzTmOb/c= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= -github.com/go-openapi/swag/mangling v0.25.3 h1:rGIrEzXaYWuUW1MkFmG3pcH+EIA0/CoUkQnIyB6TUyo= -github.com/go-openapi/swag/mangling v0.25.3/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= -github.com/go-openapi/swag/netutils v0.25.3 h1:XWXHZfL/65ABiv8rvGp9dtE0C6QHTYkCrNV77jTl358= -github.com/go-openapi/swag/netutils v0.25.3/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= -github.com/go-openapi/swag/stringutils v0.25.3 h1:nAmWq1fUTWl/XiaEPwALjp/8BPZJun70iDHRNq/sH6w= -github.com/go-openapi/swag/stringutils v0.25.3/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= -github.com/go-openapi/swag/typeutils v0.25.3 h1:2w4mEEo7DQt3V4veWMZw0yTPQibiL3ri2fdDV4t2TQc= -github.com/go-openapi/swag/typeutils v0.25.3/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= -github.com/go-openapi/swag/yamlutils v0.25.3 h1:LKTJjCn/W1ZfMec0XDL4Vxh8kyAnv1orH5F2OREDUrg= -github.com/go-openapi/swag/yamlutils v0.25.3/go.mod h1:Y7QN6Wc5DOBXK14/xeo1cQlq0EA0wvLoSv13gDQoCao= github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= @@ -138,12 +123,12 @@ github.com/gobuffalo/flect v1.0.3 h1:xeWBM2nui+qnVvNM4S3foBhCAL2XgPU+a7FdpelbTq4 github.com/gobuffalo/flect v1.0.3/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= -github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= -github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= -github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/cel-go v0.26.1 h1:iPbVVEdkhTX++hpe3lzSk7D3G3QSYqLGoHOcEio+UXQ= +github.com/google/cel-go v0.26.1/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -155,6 +140,8 @@ github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pI github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= @@ -173,14 +160,20 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -191,16 +184,14 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.27.3 h1:ICsZJ8JoYafeXFFlFAG75a7CxMsJHwgKwtO+82SE9L8= -github.com/onsi/ginkgo/v2 v2.27.3/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/ginkgo/v2 v2.27.5 h1:ZeVgZMx2PDMdJm/+w5fE/OyG6ILo1Y3e+QX4zSR0zTE= github.com/onsi/ginkgo/v2 v2.27.5/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= -github.com/onsi/gomega v1.38.3 h1:eTX+W6dobAYfFeGC2PV6RwXRu/MyT+cQguijutvkpSM= -github.com/onsi/gomega v1.38.3/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -212,20 +203,23 @@ github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.67.2 h1:PcBAckGFTIHt2+L3I33uNRTlKTplNzFctXcWhPyAEN8= -github.com/prometheus/common v0.67.2/go.mod h1:63W3KZb1JOKgcjlIr64WW/LvFGAqKPj0atm+knVGEko= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= -github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= -github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= @@ -233,9 +227,8 @@ github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8w github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -246,8 +239,9 @@ github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -257,28 +251,39 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/wI2L/jsondiff v0.6.1 h1:ISZb9oNWbP64LHnu4AUhsMF5W0FIj5Ok3Krip9Shqpw= -github.com/wI2L/jsondiff v0.6.1/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM= +github.com/wI2L/jsondiff v0.7.0 h1:1lH1G37GhBPqCfp/lrs91rf/2j3DktX6qYAKZkLuCQQ= +github.com/wI2L/jsondiff v0.7.0/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= @@ -293,54 +298,65 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= -golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= -golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= -golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= -golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 h1:FiusG7LWj+4byqhbvmB+Q93B/mOxJLN2DTozDuZm4EU= +google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:kXqgZtrWaf6qS3jZOCnCH7WYfrvFjkC51bM8fz3RsCA= +google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 h1:tu/dtnW1o3wfaxCOjSLn5IRX4YDcJrtlpzYkhHhGaC4= +google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171/go.mod h1:M5krXqk4GhBKvB596udGL3UyjL4I1+cTbK0orROM9ng= google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1 h1:pmJpJEvT846VzausCQ5d7KreSROcDqmO388w5YbnltA= google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1/go.mod h1:GmFNa4BdJZ2a8G+wCe9Bg3wwThLrJun751XstdJt5Og= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -353,54 +369,46 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= -k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= -k8s.io/apiextensions-apiserver v0.35.0 h1:3xHk2rTOdWXXJM+RDQZJvdx0yEOgC0FgQ1PlJatA5T4= -k8s.io/apiextensions-apiserver v0.35.0/go.mod h1:E1Ahk9SADaLQ4qtzYFkwUqusXTcaV2uw3l14aqpL2LU= -k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= -k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= -k8s.io/apiserver v0.35.0 h1:CUGo5o+7hW9GcAEF3x3usT3fX4f9r8xmgQeCBDaOgX4= -k8s.io/apiserver v0.35.0/go.mod h1:QUy1U4+PrzbJaM3XGu2tQ7U9A4udRRo5cyxkFX0GEds= +k8s.io/api v0.35.5 h1:BrFeUDGY/LBtlA1R5RoxhlYRHs76RnQBc6xbm/y7hsQ= +k8s.io/api v0.35.5/go.mod h1:xWkFhMnoPZdTAQh95Rlw3zZpUUNVlFHcuESUYd06BWM= +k8s.io/apiextensions-apiserver v0.35.5 h1:HttlJjgsx3ddLsASCqklkKvfBlwUoXma8VLpeMG5YL8= +k8s.io/apiextensions-apiserver v0.35.5/go.mod h1:4xbAgP/jbt8sVHE3H4DfE1gSPLUoSzXrNqhZz1lTHKc= +k8s.io/apimachinery v0.35.5 h1:lbjjjUfVeVqFbiOpyhqZHc8DhiYkWOxSNij7lHx2U8Y= +k8s.io/apimachinery v0.35.5/go.mod h1:NNi1taPOpep0jOj+oRha3mBJPqvi0hGdaV8TCqGQ+cc= +k8s.io/apiserver v0.35.5 h1:ZtFpSEmxf/VmOdbL3bo7hLxyNRorRegqOLmYSW0mxEo= +k8s.io/apiserver v0.35.5/go.mod h1:6NNWFTq/UosCwUmqhQDC+3ApzSx5ekeYMIwzSG+49VU= k8s.io/cli-runtime v0.35.0 h1:PEJtYS/Zr4p20PfZSLCbY6YvaoLrfByd6THQzPworUE= k8s.io/cli-runtime v0.35.0/go.mod h1:VBRvHzosVAoVdP3XwUQn1Oqkvaa8facnokNkD7jOTMY= -k8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE= -k8s.io/client-go v0.35.0/go.mod h1:q2E5AAyqcbeLGPdoRB+Nxe3KYTfPce1Dnu1myQdqz9o= +k8s.io/client-go v0.35.5 h1:wUrgqVSmFRw75bgSHY7X0G/hZM/QYpV0Hg7SYYOYpFk= +k8s.io/client-go v0.35.5/go.mod h1:Z0mDcAJsX1Y7RQfuQlJipiRtqf8Mhk2VDu1/JvRqdGo= k8s.io/cluster-bootstrap v0.34.2 h1:oKckPeunVCns37BntcsxaOesDul32yzGd3DFLjW2fc8= k8s.io/cluster-bootstrap v0.34.2/go.mod h1:f21byPR7X5nt12ivZi+J3pb4sG4SH6VySX8KAAJA8BY= -k8s.io/component-base v0.35.0 h1:+yBrOhzri2S1BVqyVSvcM3PtPyx5GUxCK2tinZz1G94= -k8s.io/component-base v0.35.0/go.mod h1:85SCX4UCa6SCFt6p3IKAPej7jSnF3L8EbfSyMZayJR0= +k8s.io/component-base v0.35.5 h1:1y1xxfpFNkNi4RMi6bvPNN4aDr9VhOijtEfrqnhPijs= +k8s.io/component-base v0.35.5/go.mod h1:n/+aL98XYINubqIu/Okh6mS/kZT2nMeN4IQkQR4VXRg= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= -k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e h1:iW9ChlU0cU16w8MpVYjXk12dqQ4BPFBEgif+ap7/hqQ= k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= -k8s.io/utils v0.0.0-20251222233032-718f0e51e6d2 h1:OfgiEo21hGiwx1oJUU5MpEaeOEg6coWndBkZF/lkFuE= -k8s.io/utils v0.0.0-20251222233032-718f0e51e6d2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +k8s.io/kubectl v0.35.0 h1:cL/wJKHDe8E8+rP3G7avnymcMg6bH6JEcR5w5uo06wc= +k8s.io/kubectl v0.35.0/go.mod h1:VR5/TSkYyxZwrRwY5I5dDq6l5KXmiCb+9w8IKplk3Qo= k8s.io/utils v0.0.0-20260108192941-914a6e750570 h1:JT4W8lsdrGENg9W+YwwdLJxklIuKWdRm+BC+xt33FOY= k8s.io/utils v0.0.0-20260108192941-914a6e750570/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/cluster-api v1.12.1 h1:s3DivSZjXdu2HPyOtV/n6XwSZBaIycZdKNs4y8X+3lY= -sigs.k8s.io/cluster-api v1.12.1/go.mod h1:+S6WJdi8UPdqv5q9nka5al3ed/Qa0zAcSBgzTaa9VKA= sigs.k8s.io/cluster-api v1.12.2 h1:+b+M2IygfvFZJq7bsaloNakimMEVNf81zkGR1IiuxXs= sigs.k8s.io/cluster-api v1.12.2/go.mod h1:2XuF/dmN3c/1VITb6DB44N5+Ecvsvd5KOWqrY9Q53nU= -sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= -sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= sigs.k8s.io/controller-runtime v0.23.0 h1:Ubi7klJWiwEWqDY+odSVZiFA0aDSevOCXpa38yCSYu8= sigs.k8s.io/controller-runtime v0.23.0/go.mod h1:DBOIr9NsprUqCZ1ZhsuJ0wAnQSIxY/C6VjZbmLgw0j0= sigs.k8s.io/gateway-api v1.4.1 h1:NPxFutNkKNa8UfLd2CMlEuhIPMQgDQ6DXNKG9sHbJU8= sigs.k8s.io/gateway-api v1.4.1/go.mod h1:AR5RSqciWP98OPckEjOjh2XJhAe2Na4LHyXD2FUY7Qk= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/kustomize/api v0.21.0 h1:I7nry5p8iDJbuRdYS7ez8MUvw7XVNPcIP5GkzzuXIIQ= -sigs.k8s.io/kustomize/api v0.21.0/go.mod h1:XGVQuR5n2pXKWbzXHweZU683pALGw/AMVO4zU4iS8SE= -sigs.k8s.io/kustomize/kyaml v0.21.0 h1:7mQAf3dUwf0wBerWJd8rXhVcnkk5Tvn/q91cGkaP6HQ= -sigs.k8s.io/kustomize/kyaml v0.21.0/go.mod h1:hmxADesM3yUN2vbA5z1/YTBnzLJ1dajdqpQonwBL1FQ= +sigs.k8s.io/kustomize/api v0.21.1 h1:lzqbzvz2CSvsjIUZUBNFKtIMsEw7hVLJp0JeSIVmuJs= +sigs.k8s.io/kustomize/api v0.21.1/go.mod h1:f3wkKByTrgpgltLgySCntrYoq5d3q7aaxveSagwTlwI= +sigs.k8s.io/kustomize/kyaml v0.21.1 h1:IVlbmhC076nf6foyL6Taw4BkrLuEsXUXNpsE+ScX7fI= +sigs.k8s.io/kustomize/kyaml v0.21.1/go.mod h1:hmxADesM3yUN2vbA5z1/YTBnzLJ1dajdqpQonwBL1FQ= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= sigs.k8s.io/structured-merge-diff/v6 v6.3.1/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= diff --git a/hack/distro/argocd/release.flux.yaml b/hack/distro/argocd/release.flux.yaml index 9312f4d1..fb35bec2 100644 --- a/hack/distro/argocd/release.flux.yaml +++ b/hack/distro/argocd/release.flux.yaml @@ -220,6 +220,134 @@ spec: return obj + resource.customizations.health.capsule.clastix.io_GlobalTenantResource: | + hs = {} + if obj.status ~= nil then + if obj.status.conditions ~= nil then + for i, condition in ipairs(obj.status.conditions) do + if condition.type == "Cordoned" and condition.status == "True" then + hs.status = "Suspended" + hs.message = condition.message + return hs + end + end + for i, condition in ipairs(obj.status.conditions) do + if condition.type == "Ready" and condition.status == "False" then + hs.status = "Degraded" + hs.message = condition.message + return hs + end + if condition.type == "Ready" and condition.status == "True" then + hs.status = "Healthy" + hs.message = condition.message + return hs + end + end + end + end + + hs.status = "Progressing" + hs.message = "Waiting for Status" + return hs + + resource.customizations.actions.capsule.clastix.io_GlobalTenantResource: | + mergeBuiltinActions: true + discovery.lua: | + actions = {} + + actions["cordon"] = { + ["iconClass"] = "fa fa-solid fa-pause", + ["disabled"] = true, + } + actions["uncordon"] = { + ["iconClass"] = "fa fa-solid fa-play", + ["disabled"] = true, + } + actions["reconcile"] = { + ["iconClass"] = "fa fa-solid fa-sync", + ["disabled"] = false, + } + + local suspend = false + if obj.spec ~= nil and obj.spec.cordoned ~= nil then + suspend = obj.spec.cordoned + end + + if suspend then + actions["uncordon"]["disabled"] = false + else + actions["cordon"]["disabled"] = false + end + + return actions + + definitions: + - name: cordon + action.lua: | + if obj.spec == nil then + obj.spec = {} + end + obj.spec.cordoned = true + return obj + + - name: uncordon + action.lua: | + if obj.spec ~= nil and obj.spec.cordoned ~= nil and obj.spec.cordoned then + obj.spec.cordoned = false + end + return obj + + - name: reconcile + action.lua: | + if obj.metadata == nil then + obj.metadata = {} + end + if obj.metadata.annotations == nil then + obj.metadata.annotations = {} + end + + local key = "reconcile.projectcapsule.dev/requested" + + local ts = nil + if os ~= nil and os.time ~= nil then + ts = tostring(os.time()) + else + ts = "true" + end + + obj.metadata.annotations[key] = ts + return obj + + + resource.customizations.health.capsule.clastix.io_TenantResource: | + hs = {} + if obj.status ~= nil then + if obj.status.conditions ~= nil then + for i, condition in ipairs(obj.status.conditions) do + if condition.type == "Cordoned" and condition.status == "True" then + hs.status = "Suspended" + hs.message = condition.message + return hs + end + end + for i, condition in ipairs(obj.status.conditions) do + if condition.type == "Ready" and condition.status == "False" then + hs.status = "Degraded" + hs.message = condition.message + return hs + end + if condition.type == "Ready" and condition.status == "True" then + hs.status = "Healthy" + hs.message = condition.message + return hs + end + end + end + end + + hs.status = "Progressing" + hs.message = "Waiting for Status" + return hs --- apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository diff --git a/hack/distro/capsule/example-setup/custom-quotas.yaml b/hack/distro/capsule/example-setup/custom-quotas.yaml new file mode 100644 index 00000000..2819df86 --- /dev/null +++ b/hack/distro/capsule/example-setup/custom-quotas.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: capsule.clastix.io/v1beta2 +kind: GlobalCustomQuota +metadata: + name: storage-aggregate +spec: + limit: 5Gi + namespaceSelectors: + - matchLabels: + capsule.clastix.io/tenant: wind + sources: + - apiVersion: v1 + kind: Pod + op: add + path: ".spec.volumes[*].ephemeral.volumeClaimTemplate.spec.resources.requests.storage" + + - apiVersion: v1 + kind: PersistentVolumeClaim + op: add + path: ".spec.resources.requests.storage" + selectors: + - fieldSelectors: + - '.spec.accessModes[?(@=="ReadWriteOnce")]' +--- +apiVersion: capsule.clastix.io/v1beta2 +kind: GlobalCustomQuota +metadata: + name: cpu-limits +spec: + limit: 5 + namespaceSelectors: + - matchLabels: + capsule.clastix.io/tenant: wind + sources: + - apiVersion: "v1" + kind: Pod + op: add + path: .spec.containers[*].resources.limits.cpu + - apiVersion: "v1" + kind: Pod + op: add + path: .spec.initContainers[*].resources.limits.cpu diff --git a/hack/distro/capsule/example-setup/kustomization.yaml b/hack/distro/capsule/example-setup/kustomization.yaml index 693c6041..a60fba4d 100644 --- a/hack/distro/capsule/example-setup/kustomization.yaml +++ b/hack/distro/capsule/example-setup/kustomization.yaml @@ -5,3 +5,5 @@ resources: - tenants.yaml - resource.yaml - pools.yaml + - rbac.yaml + - custom-quotas.yaml diff --git a/hack/distro/capsule/example-setup/rbac.yaml b/hack/distro/capsule/example-setup/rbac.yaml new file mode 100644 index 00000000..c645da71 --- /dev/null +++ b/hack/distro/capsule/example-setup/rbac.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: configmap-replicator + labels: + projectcapsule.dev/aggregate-to-controller: "true" +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "patch", "watch", "list", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: secret-replicator + labels: + projectcapsule.dev/aggregate-to-controller: "true" +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "patch", "watch", "list", "delete"] diff --git a/hack/distro/capsule/example-setup/resource-legacy.yaml b/hack/distro/capsule/example-setup/resource-legacy.yaml new file mode 100644 index 00000000..2f0e3a5c --- /dev/null +++ b/hack/distro/capsule/example-setup/resource-legacy.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: capsule.clastix.io/v1beta2 +kind: GlobalTenantResource +metadata: + name: custom-cm +spec: + resyncPeriod: 60s + resources: + - additionalMetadata: + labels: + "replicated-by": "capsule" + rawItems: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: game-demo + data: + # property-like keys; each key maps to a simple value + player_initial_lives: "3" + ui_properties_file_name: "user-interface.properties" +--- +apiVersion: capsule.clastix.io/v1beta2 +kind: GlobalTenantResource +metadata: + name: gitops-owners +spec: + resyncPeriod: 60s + resources: + - additionalMetadata: + labels: + "replicated-by": "capsule" + rawItems: + - apiVersion: capsule.clastix.io/v1beta2 + kind: TenantOwner + metadata: + name: "{{tenant.name}}-{{namespace}}" + spec: + clusterRoles: + - capsule-namespace-deleter + - admin + kind: ServiceAccount + name: "system:serviceaccount:{{namespace}}:gitops-reconciler" diff --git a/hack/distro/capsule/example-setup/resource.yaml b/hack/distro/capsule/example-setup/resource.yaml index afc33992..37546296 100644 --- a/hack/distro/capsule/example-setup/resource.yaml +++ b/hack/distro/capsule/example-setup/resource.yaml @@ -2,20 +2,45 @@ apiVersion: capsule.clastix.io/v1beta2 kind: GlobalTenantResource metadata: - name: custom-cm - namespace: solar-system + name: gitops-owners spec: resyncPeriod: 60s + dependsOn: + - name: custom-cm-2 resources: - additionalMetadata: labels: "replicated-by": "capsule" rawItems: - - apiVersion: v1 - kind: ConfigMap + - apiVersion: capsule.clastix.io/v1beta2 + kind: TenantOwner metadata: - name: game-demo - data: - # property-like keys; each key maps to a simple value - player_initial_lives: "3" - ui_properties_file_name: "user-interface.properties" + name: "{{tenant.name}}-{{namespace}}" + spec: + clusterRoles: + - capsule-namespace-deleter + - admin + kind: ServiceAccount + name: "system:serviceaccount:{{namespace}}:gitops-reconciler" +--- +apiVersion: capsule.clastix.io/v1beta2 +kind: GlobalTenantResource +metadata: + name: cluster-replication +spec: + resyncPeriod: 60s + scope: None + resources: + - generators: + - template: | + {{ $$key := generateAgeKey }} + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: "cluster-replication" + namespace: "default" + data: + data: | + identity: {{ $$key.Identity | quote }} + recipient: {{ $$key.Recipient | quote }} diff --git a/hack/distro/capsule/example-setup/tenants.yaml b/hack/distro/capsule/example-setup/tenants.yaml index 1f70e480..2ccc29d1 100644 --- a/hack/distro/capsule/example-setup/tenants.yaml +++ b/hack/distro/capsule/example-setup/tenants.yaml @@ -9,6 +9,31 @@ spec: owners: - name: alice kind: User + rules: + - permissions: + rules: + - clusterRoles: + - "configmap-replicator" + - namespaceSelector: + matchExpressions: + - key: env + operator: In + values: + - "test" + permissions: + rules: + - clusterRoles: + - "secret-replicator" + - namespaceSelector: + matchExpressions: + - key: env + operator: In + values: + - "prod" + permissions: + rules: + - clusterRoles: + - "sade-boi" permissions: matchOwners: - matchLabels: @@ -16,9 +41,6 @@ spec: - matchLabels: tenant: solar namespaceOptions: - requiredMetadata: - labels: - env: "prod|test|dev" additionalMetadata: labels: team: platform @@ -33,6 +55,16 @@ spec: - apiGroup: rbac.authorization.k8s.io kind: User name: joe + resourceQuotas: + scope: Tenant + items: + - hard: + limits.cpu: "8" + limits.memory: 16Gi + requests.cpu: "8" + requests.memory: 16Gi + - hard: + pods: "10" --- apiVersion: capsule.clastix.io/v1beta2 kind: Tenant @@ -41,10 +73,6 @@ metadata: labels: customer: a spec: - namespaceOptions: - requiredMetadata: - labels: - env: "prod|test|dev" permissions: matchOwners: - matchLabels: @@ -66,6 +94,12 @@ spec: - url: "harbor/.*" policy: - "Never" + - enforce: + registries: + - url: "custom/.*" + policy: + - "Never" + - namespaceSelector: matchExpressions: - key: env diff --git a/hack/distro/fluxcd/kustomization.yaml b/hack/distro/fluxcd/kustomization.yaml index 8a8c2444..8b6ab086 100644 --- a/hack/distro/fluxcd/kustomization.yaml +++ b/hack/distro/fluxcd/kustomization.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - https://github.com/fluxcd/flux2/releases/download/v2.4.0/install.yaml + - https://github.com/fluxcd/flux2/releases/download/v2.7.5/install.yaml patches: - patch: | - op: add diff --git a/hack/kubeconfig-for-sa.sh b/hack/kubeconfig-for-sa.sh new file mode 100644 index 00000000..f8e75169 --- /dev/null +++ b/hack/kubeconfig-for-sa.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail + +CLUSTER_NAME="${1:-capsule}" +NAMESPACE="${2:-capsule-system}" +SA_NAME="${3:-capsule}" +TARGET="${4:-kubeconfig-${SA_NAME}.yaml}" + +SECRET_NAME="${SA_NAME}-static-token" + +echo "👉 Using cluster: $CLUSTER_NAME" +echo "👉 Namespace: $NAMESPACE" +echo "👉 ServiceAccount: $SA_NAME" +echo "👉 Target: $TARGET" + +echo "📄 Exporting kubeconfig..." +TMP_KUBECONFIG=$(mktemp) +kind get kubeconfig --name "$CLUSTER_NAME" > "$TMP_KUBECONFIG" + +echo "🔐 Creating static token secret..." + +kubectl -n "$NAMESPACE" apply -f - < "$TARGET" < 0 { + return c.gvrs, nil + } + + resourceLists, err := disco.ServerPreferredNamespacedResources() + if err != nil && len(resourceLists) == 0 { + return nil, fmt.Errorf("discover namespaced resources: %w", err) + } + + gvrs, err := gvk.NamespacedListableResources(resourceLists) + if err != nil { + return nil, err + } + + c.gvrs = append(c.gvrs[:0], gvrs...) + c.expiresAt = time.Now().Add(c.ttl) + + return c.gvrs, nil +} + +func (c *DiscoveryNamespacedResourceCache) Invalidate() { + c.mu.Lock() + defer c.mu.Unlock() + + c.expiresAt = time.Time{} + c.gvrs = nil +} diff --git a/internal/cache/impersonation_clients.go b/internal/cache/impersonation_clients.go new file mode 100644 index 00000000..c1f67f79 --- /dev/null +++ b/internal/cache/impersonation_clients.go @@ -0,0 +1,119 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "context" + "sync" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/users" +) + +type Key struct { + Namespace string + Name string +} + +type ImpersonationCache struct { + mu sync.RWMutex + clients map[Key]client.Client +} + +func NewImpersonationCache() *ImpersonationCache { + return &ImpersonationCache{ + clients: make(map[Key]client.Client), + } +} + +// Get returns a cached client if present. +func (c *ImpersonationCache) Get(ns, name string) (client.Client, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + + cl, ok := c.clients[Key{Namespace: ns, Name: name}] + + return cl, ok +} + +// Set stores a client explicitly (rarely needed). +func (c *ImpersonationCache) Set(namespace, name string, cl client.Client) { + c.mu.Lock() + defer c.mu.Unlock() + + if cl == nil { + return + } + + c.clients[Key{Namespace: namespace, Name: name}] = cl +} + +// Invalidate removes one entry. +func (c *ImpersonationCache) Invalidate(namespace, name string) { + c.mu.Lock() + defer c.mu.Unlock() + + delete(c.clients, Key{Namespace: namespace, Name: name}) +} + +// Clear drops all cached clients. +func (c *ImpersonationCache) Reset() { + c.mu.Lock() + defer c.mu.Unlock() + + c.clients = make(map[Key]client.Client) +} + +// Stats helps you log cache state. +func (c *ImpersonationCache) Stats() (entries int) { + c.mu.RLock() + defer c.mu.RUnlock() + + return len(c.clients) +} + +// LoadOrCreate returns a cached impersonated client for the given service account, +// creating and caching it if missing. +func (c *ImpersonationCache) LoadOrCreate( + ctx context.Context, + log logr.Logger, + baseREST *rest.Config, + scheme *runtime.Scheme, + sa meta.NamespacedRFC1123ObjectReferenceWithNamespace, +) (client.Client, error) { + key := Key{Namespace: string(sa.Namespace), Name: string(sa.Name)} + + // Fast path + if cl, ok := c.Get(key.Namespace, key.Name); ok { + return cl, nil + } + + cl, err := users.ImpersonatedKubernetesClientForServiceAccount( + baseREST, + scheme, + sa, + ) + if err != nil { + log.Error(err, "failed to create impersonated client", "namespace", key.Namespace, "name", key.Name) + + return nil, err + } + + // Store (double-check to avoid duplicate creation races) + c.mu.Lock() + defer c.mu.Unlock() + + if existing := c.clients[key]; existing != nil { + return existing, nil + } + + c.clients[key] = cl + + return cl, nil +} diff --git a/internal/cache/impersonation_clients_test.go b/internal/cache/impersonation_clients_test.go new file mode 100644 index 00000000..940086ad --- /dev/null +++ b/internal/cache/impersonation_clients_test.go @@ -0,0 +1,307 @@ +// Copyright 2020-2025 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package cache_test + +import ( + "context" + "reflect" + "sync" + "testing" + "time" + + "github.com/go-logr/logr" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/projectcapsule/capsule/internal/cache" + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +// makeSA builds meta.NamespacedRFC1123ObjectReferenceWithNamespace without depending +// on the concrete field types (they're string aliases in Capsule). +func makeSA(ns, name string) meta.NamespacedRFC1123ObjectReferenceWithNamespace { + var sa meta.NamespacedRFC1123ObjectReferenceWithNamespace + + v := reflect.ValueOf(&sa).Elem() + + nsField := v.FieldByName("Namespace") + if !nsField.IsValid() || nsField.Kind() != reflect.String || !nsField.CanSet() { + panic("meta.NamespacedRFC1123ObjectReferenceWithNamespace.Namespace is not a settable string-kind field") + } + nsField.SetString(ns) + + nameField := v.FieldByName("Name") + if !nameField.IsValid() || nameField.Kind() != reflect.String || !nameField.CanSet() { + panic("meta.NamespacedRFC1123ObjectReferenceWithNamespace.Name is not a settable string-kind field") + } + nameField.SetString(name) + + return sa +} + +func TestImpersonationCache_Basics(t *testing.T) { + c := cache.NewImpersonationCache() + + t.Run("Get on empty cache returns false", func(t *testing.T) { + _, ok := c.Get("ns", "sa") + if ok { + t.Fatalf("expected ok=false on empty cache") + } + }) + + t.Run("Set(nil) stores entry but Get returns false", func(t *testing.T) { + c.Reset() + + c.Set("ns", "sa", nil) + + if entries := c.Stats(); entries != 0 { + t.Fatalf("expected Stats()=0 after Set(nil), got %d", entries) + } + + _, ok := c.Get("ns", "sa") + if ok { + t.Fatalf("expected ok=false because stored client is nil") + } + }) + + t.Run("Invalidate removes entry", func(t *testing.T) { + c.Reset() + + cl1 := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + + c.Set("ns", "sa", cl1) + c.Invalidate("ns", "sa") + if entries := c.Stats(); entries != 0 { + t.Fatalf("expected Stats()=0 after Invalidate, got %d", entries) + } + }) + + t.Run("Clear drops all entries", func(t *testing.T) { + c.Reset() + + cl1 := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + cl2 := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + + c.Set("a", "x", cl1) + c.Set("b", "y", cl2) + + if entries := c.Stats(); entries != 2 { + t.Fatalf("expected Stats()=2 before Clear, got %d", entries) + } + + c.Reset() + if entries := c.Stats(); entries != 0 { + t.Fatalf("expected Stats()=0 after Clear, got %d", entries) + } + }) +} + +func TestImpersonationCache_LoadOrCreate_SuccessCachesAndReturnsSameInstance(t *testing.T) { + t.Parallel() + + cache := cache.NewImpersonationCache() + ctx := context.Background() + log := logr.Discard() + sa := makeSA("monitoring", "alertmanager-sa") + + // Provide a REST config that is syntactically valid so controller-runtime client can be constructed + // without needing a live apiserver. + validREST := &rest.Config{ + Host: "https://127.0.0.1", // no connectivity required for client object creation + TLSClientConfig: rest.TLSClientConfig{ + Insecure: true, + }, + } + + sch := scheme.Scheme + + cl1, err := cache.LoadOrCreate(ctx, log, validREST, sch, sa) + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if cl1 == nil { + t.Fatalf("expected non-nil client") + } + + // Should be cached now. + if entries := cache.Stats(); entries != 1 { + t.Fatalf("expected 1 cache entry, got %d", entries) + } + + cl2, err := cache.LoadOrCreate(ctx, log, validREST, sch, sa) + if err != nil { + t.Fatalf("expected nil error on second LoadOrCreate, got %v", err) + } + if cl2 == nil { + t.Fatalf("expected non-nil client on second LoadOrCreate") + } + + // Must return the cached instance. + if cl1 != cl2 { + t.Fatalf("expected same client instance from cache; got different pointers") + } + + // Get should return it and ok=true. + got, ok := cache.Get("monitoring", "alertmanager-sa") + if !ok { + t.Fatalf("expected ok=true from Get after LoadOrCreate") + } + if got != cl1 { + t.Fatalf("expected Get to return cached client instance") + } +} + +func TestImpersonationCache_LoadOrCreate_ConcurrentOnlyCachesOne(t *testing.T) { + t.Parallel() + + cache := cache.NewImpersonationCache() + ctx := context.Background() + log := logr.Discard() + sa := makeSA("monitoring", "alertmanager-sa") + + validREST := &rest.Config{ + Host: "https://127.0.0.1", + TLSClientConfig: rest.TLSClientConfig{ + Insecure: true, + }, + } + sch := scheme.Scheme + + const goroutines = 25 + start := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(goroutines) + + results := make([]client.Client, goroutines) + errs := make([]error, goroutines) + + for i := 0; i < goroutines; i++ { + i := i + go func() { + defer wg.Done() + <-start + cl, err := cache.LoadOrCreate(ctx, log, validREST, sch, sa) + results[i] = cl + errs[i] = err + }() + } + + close(start) + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatalf("timeout waiting for goroutines") + } + + // All should succeed and return a non-nil client. + var first client.Client + for i := 0; i < goroutines; i++ { + if errs[i] != nil { + t.Fatalf("expected nil error for goroutine %d, got %v", i, errs[i]) + } + if results[i] == nil { + t.Fatalf("expected non-nil client for goroutine %d", i) + } + if i == 0 { + first = results[i] + continue + } + // Even with races (double-check store), function returns the stored instance. + if results[i] != first { + t.Fatalf("expected all goroutines to get same cached client instance; got mismatch at %d", i) + } + } + + // Cache should have exactly one entry. + if entries := cache.Stats(); entries != 1 { + t.Fatalf("expected exactly 1 cache entry after concurrent LoadOrCreate, got %d", entries) + } +} + +func TestImpersonationCache_LoadOrCreate_DifferentKeysCreateDifferentEntries(t *testing.T) { + t.Parallel() + + cache := cache.NewImpersonationCache() + ctx := context.Background() + log := logr.Discard() + + validREST := &rest.Config{ + Host: "https://127.0.0.1", + TLSClientConfig: rest.TLSClientConfig{ + Insecure: true, + }, + } + sch := scheme.Scheme + + sa1 := makeSA("monitoring", "sa-one") + sa2 := makeSA("monitoring", "sa-two") + + cl1, err := cache.LoadOrCreate(ctx, log, validREST, sch, sa1) + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + cl2, err := cache.LoadOrCreate(ctx, log, validREST, sch, sa2) + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + + if cl1 == nil || cl2 == nil { + t.Fatalf("expected both clients to be non-nil") + } + if cl1 == cl2 { + t.Fatalf("expected different clients for different keys") + } + if entries := cache.Stats(); entries != 2 { + t.Fatalf("expected 2 cache entries, got %d", entries) + } +} + +func TestImpersonationCache_InvalidateThenLoadOrCreateCreatesNewInstance(t *testing.T) { + t.Parallel() + + cache := cache.NewImpersonationCache() + ctx := context.Background() + log := logr.Discard() + sa := makeSA("monitoring", "alertmanager-sa") + + validREST := &rest.Config{ + Host: "https://127.0.0.1", + TLSClientConfig: rest.TLSClientConfig{ + Insecure: true, + }, + } + sch := scheme.Scheme + + cl1, err := cache.LoadOrCreate(ctx, log, validREST, sch, sa) + if err != nil || cl1 == nil { + t.Fatalf("expected first LoadOrCreate success, err=%v cl=%v", err, cl1) + } + + cache.Invalidate("monitoring", "alertmanager-sa") + if entries := cache.Stats(); entries != 0 { + t.Fatalf("expected 0 entries after Invalidate, got %d", entries) + } + + cl2, err := cache.LoadOrCreate(ctx, log, validREST, sch, sa) + if err != nil || cl2 == nil { + t.Fatalf("expected second LoadOrCreate success, err=%v cl=%v", err, cl2) + } + + // After invalidation, a new client instance should be created and cached. + if cl1 == cl2 { + t.Fatalf("expected new client instance after Invalidate, got same pointer") + } + if entries := cache.Stats(); entries != 1 { + t.Fatalf("expected 1 entry after re-LoadOrCreate, got %d", entries) + } +} diff --git a/internal/cache/invalidation_test.go b/internal/cache/invalidation_test.go new file mode 100644 index 00000000..154957f3 --- /dev/null +++ b/internal/cache/invalidation_test.go @@ -0,0 +1,63 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("ShouldInvalidate", func() { + var now time.Time + + BeforeEach(func() { + now = time.Date(2026, 4, 28, 12, 0, 0, 0, time.UTC) + }) + + It("returns false when interval is zero", func() { + Expect(ShouldInvalidate(nil, now, 0)).To(BeFalse()) + }) + + It("returns false when interval is negative", func() { + Expect(ShouldInvalidate(nil, now, -time.Second)).To(BeFalse()) + }) + + It("returns true when last is nil and interval is positive", func() { + Expect(ShouldInvalidate(nil, now, time.Minute)).To(BeTrue()) + }) + + It("returns true when last is zero and interval is positive", func() { + last := &metav1.Time{} + + Expect(ShouldInvalidate(last, now, time.Minute)).To(BeTrue()) + }) + + It("returns false when last is after now", func() { + last := metav1.NewTime(now.Add(time.Minute)) + + Expect(ShouldInvalidate(&last, now, time.Minute)).To(BeFalse()) + }) + + It("returns false when elapsed time is below interval", func() { + last := metav1.NewTime(now.Add(-30 * time.Second)) + + Expect(ShouldInvalidate(&last, now, time.Minute)).To(BeFalse()) + }) + + It("returns true when elapsed time equals interval", func() { + last := metav1.NewTime(now.Add(-time.Minute)) + + Expect(ShouldInvalidate(&last, now, time.Minute)).To(BeTrue()) + }) + + It("returns true when elapsed time is greater than interval", func() { + last := metav1.NewTime(now.Add(-2 * time.Minute)) + + Expect(ShouldInvalidate(&last, now, time.Minute)).To(BeTrue()) + }) +}) diff --git a/internal/cache/jsonpaths.go b/internal/cache/jsonpaths.go new file mode 100644 index 00000000..457f35ac --- /dev/null +++ b/internal/cache/jsonpaths.go @@ -0,0 +1,123 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "sync" + + "github.com/projectcapsule/capsule/pkg/runtime/jsonpath" +) + +type JSONPathCache struct { + mu sync.RWMutex + data map[string]*jsonpath.CompiledJSONPath +} + +func NewJSONPathCache() *JSONPathCache { + return &JSONPathCache{ + data: make(map[string]*jsonpath.CompiledJSONPath), + } +} + +func (c *JSONPathCache) Get(path string) (*jsonpath.CompiledJSONPath, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + + v, ok := c.data[path] + + return v, ok +} + +func (c *JSONPathCache) GetOrCompile(path string) (*jsonpath.CompiledJSONPath, error) { + c.mu.RLock() + compiled, ok := c.data[path] + c.mu.RUnlock() + + if ok { + return compiled, nil + } + + c.mu.Lock() + defer c.mu.Unlock() + + if compiled, ok = c.data[path]; ok { + return compiled, nil + } + + var err error + + compiled, err = jsonpath.CompileJSONPath(path) + if err != nil { + return nil, err + } + + c.data[path] = compiled + + return compiled, nil +} + +func (c *JSONPathCache) Delete(path string) bool { + c.mu.Lock() + defer c.mu.Unlock() + + if _, ok := c.data[path]; !ok { + return false + } + + delete(c.data, path) + + return true +} + +func (c *JSONPathCache) DeleteMany(expressions ...string) int { + c.mu.Lock() + defer c.mu.Unlock() + + deleted := 0 + + for _, expr := range expressions { + if expr == "" { + continue + } + + if _, ok := c.data[expr]; ok { + delete(c.data, expr) + + deleted++ + } + } + + return deleted +} + +func (c *JSONPathCache) Reset() { + c.mu.Lock() + defer c.mu.Unlock() + + c.data = make(map[string]*jsonpath.CompiledJSONPath) +} + +func (c *JSONPathCache) Stats() int { + c.mu.RLock() + defer c.mu.RUnlock() + + return len(c.data) +} + +func (c *JSONPathCache) PruneActive(active map[string]struct{}) int { + c.mu.Lock() + defer c.mu.Unlock() + + pruned := 0 + + for path := range c.data { + if _, ok := active[path]; !ok { + delete(c.data, path) + + pruned++ + } + } + + return pruned +} diff --git a/internal/cache/jsonpaths_test.go b/internal/cache/jsonpaths_test.go new file mode 100644 index 00000000..4c3a60d9 --- /dev/null +++ b/internal/cache/jsonpaths_test.go @@ -0,0 +1,185 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "sync" + "sync/atomic" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("JSONPathCache", func() { + var c *JSONPathCache + + BeforeEach(func() { + c = NewJSONPathCache() + }) + + It("returns false when path is missing", func() { + value, ok := c.Get(".spec.missing") + + Expect(ok).To(BeFalse()) + Expect(value).To(BeNil()) + }) + + It("compiles and stores a JSONPath", func() { + compiled, err := c.GetOrCompile(".spec.containers[*].resources.requests.cpu") + + Expect(err).NotTo(HaveOccurred()) + Expect(compiled).NotTo(BeNil()) + Expect(c.Stats()).To(Equal(1)) + + cached, ok := c.Get(".spec.containers[*].resources.requests.cpu") + Expect(ok).To(BeTrue()) + Expect(cached).To(BeIdenticalTo(compiled)) + }) + + It("returns the cached instance on repeated GetOrCompile calls", func() { + first, err := c.GetOrCompile(".spec.resources.requests.storage") + Expect(err).NotTo(HaveOccurred()) + + second, err := c.GetOrCompile(".spec.resources.requests.storage") + Expect(err).NotTo(HaveOccurred()) + + Expect(second).To(BeIdenticalTo(first)) + Expect(c.Stats()).To(Equal(1)) + }) + + It("does not store invalid JSONPath expressions", func() { + compiled, err := c.GetOrCompile(".spec.containers[?(") + + Expect(err).To(HaveOccurred()) + Expect(compiled).To(BeNil()) + Expect(c.Stats()).To(Equal(0)) + + _, ok := c.Get(".spec.containers[?(") + Expect(ok).To(BeFalse()) + }) + + It("deletes an existing path", func() { + _, err := c.GetOrCompile(".spec.replicas") + Expect(err).NotTo(HaveOccurred()) + + Expect(c.Delete(".spec.replicas")).To(BeTrue()) + Expect(c.Delete(".spec.replicas")).To(BeFalse()) + + _, ok := c.Get(".spec.replicas") + Expect(ok).To(BeFalse()) + Expect(c.Stats()).To(Equal(0)) + }) + + It("deletes many paths and ignores missing or empty expressions", func() { + _, err := c.GetOrCompile(".spec.a") + Expect(err).NotTo(HaveOccurred()) + + _, err = c.GetOrCompile(".spec.b") + Expect(err).NotTo(HaveOccurred()) + + _, err = c.GetOrCompile(".spec.c") + Expect(err).NotTo(HaveOccurred()) + + deleted := c.DeleteMany(".spec.a", "", ".spec.missing", ".spec.c") + + Expect(deleted).To(Equal(2)) + Expect(c.Stats()).To(Equal(1)) + + _, ok := c.Get(".spec.a") + Expect(ok).To(BeFalse()) + + _, ok = c.Get(".spec.b") + Expect(ok).To(BeTrue()) + + _, ok = c.Get(".spec.c") + Expect(ok).To(BeFalse()) + }) + + It("resets all compiled paths", func() { + _, err := c.GetOrCompile(".spec.a") + Expect(err).NotTo(HaveOccurred()) + + _, err = c.GetOrCompile(".spec.b") + Expect(err).NotTo(HaveOccurred()) + + Expect(c.Stats()).To(Equal(2)) + + c.Reset() + + Expect(c.Stats()).To(Equal(0)) + + _, ok := c.Get(".spec.a") + Expect(ok).To(BeFalse()) + }) + + It("prunes inactive paths", func() { + _, err := c.GetOrCompile(".spec.a") + Expect(err).NotTo(HaveOccurred()) + + _, err = c.GetOrCompile(".spec.b") + Expect(err).NotTo(HaveOccurred()) + + _, err = c.GetOrCompile(".spec.c") + Expect(err).NotTo(HaveOccurred()) + + pruned := c.PruneActive(map[string]struct{}{ + ".spec.a": {}, + ".spec.c": {}, + }) + + Expect(pruned).To(Equal(1)) + Expect(c.Stats()).To(Equal(2)) + + _, ok := c.Get(".spec.a") + Expect(ok).To(BeTrue()) + + _, ok = c.Get(".spec.b") + Expect(ok).To(BeFalse()) + + _, ok = c.Get(".spec.c") + Expect(ok).To(BeTrue()) + }) + + It("prunes all paths when active set is empty", func() { + _, err := c.GetOrCompile(".spec.a") + Expect(err).NotTo(HaveOccurred()) + + _, err = c.GetOrCompile(".spec.b") + Expect(err).NotTo(HaveOccurred()) + + Expect(c.PruneActive(map[string]struct{}{})).To(Equal(2)) + Expect(c.Stats()).To(Equal(0)) + }) + + It("is safe under concurrent GetOrCompile calls for the same path", func() { + const workers = 32 + + var successes atomic.Int32 + start := make(chan struct{}) + + var wg sync.WaitGroup + wg.Add(workers) + + for i := 0; i < workers; i++ { + go func() { + defer GinkgoRecover() + defer wg.Done() + + <-start + + compiled, err := c.GetOrCompile(".spec.containers[*].resources.requests.cpu") + Expect(err).NotTo(HaveOccurred()) + Expect(compiled).NotTo(BeNil()) + + successes.Add(1) + }() + } + + close(start) + wg.Wait() + + Expect(successes.Load()).To(Equal(int32(workers))) + Expect(c.Stats()).To(Equal(1)) + }) +}) diff --git a/internal/cache/quantities.go b/internal/cache/quantities.go new file mode 100644 index 00000000..a62680f5 --- /dev/null +++ b/internal/cache/quantities.go @@ -0,0 +1,437 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "slices" + "sync" + "time" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" +) + +type PendingDeleteHint struct { + UID types.UID + CreatedAt time.Time +} + +type Reservation struct { + ID string + Usage resource.Quantity + UID types.UID + Group string + Version string + Kind string + Namespace string + Name string + CreatedAt time.Time + UpdatedAt time.Time +} + +type QuantityEntry struct { + Reserved resource.Quantity + Reservations map[string]Reservation + PendingDeletes []PendingDeleteHint + CreatedAt time.Time + UpdatedAt time.Time + LastAccess time.Time +} + +type QuantityCache[K comparable] struct { + mu sync.RWMutex + data map[K]QuantityEntry + clock func() time.Time +} + +func NewQuantityCache[K comparable]() *QuantityCache[K] { + return &QuantityCache[K]{ + data: make(map[K]QuantityEntry), + clock: time.Now, + } +} + +func (c *QuantityCache[K]) Get(key K) (QuantityEntry, bool) { + c.mu.RLock() + entry, ok := c.data[key] + c.mu.RUnlock() + + if !ok { + return QuantityEntry{}, false + } + + return copyEntry(entry), true +} + +func (c *QuantityCache[K]) Delete(key K) bool { + c.mu.Lock() + defer c.mu.Unlock() + + _, ok := c.data[key] + if ok { + delete(c.data, key) + } + + return ok +} + +func (c *QuantityCache[K]) Len() int { + c.mu.RLock() + defer c.mu.RUnlock() + + return len(c.data) +} + +func (c *QuantityCache[K]) Snapshot() map[K]QuantityEntry { + c.mu.RLock() + defer c.mu.RUnlock() + + out := make(map[K]QuantityEntry, len(c.data)) + for k, v := range c.data { + out[k] = copyEntry(v) + } + + return out +} + +// UpsertReservation ensures a reservation is idempotent per (key, reservationID). +// It validates: +// +// persistedUsed + sum(all reservations including this one) <= limit +func (c *QuantityCache[K]) UpsertReservation( + key K, + reservation Reservation, + persistedUsed resource.Quantity, + limit resource.Quantity, +) (allowed bool, effectiveUsed resource.Quantity, entry QuantityEntry) { + now := c.clock() + + c.mu.Lock() + defer c.mu.Unlock() + + current, exists := c.data[key] + if !exists { + current = QuantityEntry{ + Reserved: resource.MustParse("0"), + Reservations: make(map[string]Reservation), + CreatedAt: now, + UpdatedAt: now, + LastAccess: now, + } + } else if current.Reservations == nil { + current.Reservations = make(map[string]Reservation) + } + + previous, hadPrevious := current.Reservations[reservation.ID] + + // Fast path: same reservation, same usage and identity. + if hadPrevious && + previous.Usage.Cmp(reservation.Usage) == 0 && + previous.UID == reservation.UID && + previous.Group == reservation.Group && + previous.Version == reservation.Version && + previous.Kind == reservation.Kind && + previous.Namespace == reservation.Namespace && + previous.Name == reservation.Name { + current.LastAccess = now + c.data[key] = current + + effectiveUsed = persistedUsed.DeepCopy() + effectiveUsed.Add(current.Reserved) + + if effectiveUsed.Sign() < 0 { + effectiveUsed = resource.MustParse("0") + } + + return true, effectiveUsed, copyEntry(current) + } + + candidateReservations := make(map[string]Reservation, len(current.Reservations)+1) + for id, r := range current.Reservations { + candidateReservations[id] = Reservation{ + ID: r.ID, + Usage: r.Usage.DeepCopy(), + UID: r.UID, + Group: r.Group, + Version: r.Version, + Kind: r.Kind, + Namespace: r.Namespace, + Name: r.Name, + CreatedAt: r.CreatedAt, + UpdatedAt: r.UpdatedAt, + } + } + + reservation.CreatedAt = func() time.Time { + if hadPrevious { + return previous.CreatedAt + } + + return now + }() + reservation.UpdatedAt = now + + candidateReservations[reservation.ID] = reservation + + newReserved := sumReservations(candidateReservations) + + newUsed := persistedUsed.DeepCopy() + newUsed.Add(newReserved) + + if newUsed.Sign() < 0 { + newUsed = resource.MustParse("0") + } + + if newUsed.Cmp(limit) > 0 { + effectiveUsed = persistedUsed.DeepCopy() + effectiveUsed.Add(current.Reserved) + + if effectiveUsed.Sign() < 0 { + effectiveUsed = resource.MustParse("0") + } + + return false, effectiveUsed, copyEntry(current) + } + + current.Reservations = candidateReservations + current.Reserved = newReserved + current.UpdatedAt = now + current.LastAccess = now + + if current.Reserved.IsZero() && len(current.PendingDeletes) == 0 { + delete(c.data, key) + + return true, newUsed, QuantityEntry{} + } + + c.data[key] = current + + return true, newUsed, copyEntry(current) +} + +func (c *QuantityCache[K]) DeleteReservation(key K, reservationID string) bool { + now := c.clock() + + c.mu.Lock() + defer c.mu.Unlock() + + current, ok := c.data[key] + if !ok || current.Reservations == nil { + return false + } + + if _, exists := current.Reservations[reservationID]; !exists { + return false + } + + delete(current.Reservations, reservationID) + current.Reserved = sumReservations(current.Reservations) + current.UpdatedAt = now + current.LastAccess = now + + if current.Reserved.IsZero() && len(current.PendingDeletes) == 0 { + delete(c.data, key) + + return true + } + + c.data[key] = current + + return true +} + +// PurgeReservationsForKey removes reservations for which shouldDelete returns true. +// Returns number of removed reservations. +func (c *QuantityCache[K]) PurgeReservationsForKey( + key K, + shouldDelete func(Reservation) bool, +) int { + c.mu.Lock() + defer c.mu.Unlock() + + entry, ok := c.data[key] + if !ok || len(entry.Reservations) == 0 { + return 0 + } + + deleted := 0 + + for id, r := range entry.Reservations { + if shouldDelete(r) { + delete(entry.Reservations, id) + + deleted++ + } + } + + if deleted == 0 { + return 0 + } + + now := c.clock() + entry.Reserved = sumReservations(entry.Reservations) + entry.UpdatedAt = now + entry.LastAccess = now + + if entry.Reserved.IsZero() && len(entry.PendingDeletes) == 0 { + delete(c.data, key) + + return deleted + } + + c.data[key] = entry + + return deleted +} + +func (c *QuantityCache[K]) AddPendingDelete(key K, uid types.UID) { + if uid == "" { + return + } + + now := c.clock() + + c.mu.Lock() + defer c.mu.Unlock() + + current, exists := c.data[key] + if !exists { + current = QuantityEntry{ + Reserved: resource.MustParse("0"), + Reservations: make(map[string]Reservation), + CreatedAt: now, + } + } else if current.Reservations == nil { + current.Reservations = make(map[string]Reservation) + } + + if !containsPendingDelete(current.PendingDeletes, uid) { + current.PendingDeletes = append(current.PendingDeletes, PendingDeleteHint{ + UID: uid, + CreatedAt: now, + }) + } + + current.UpdatedAt = now + current.LastAccess = now + c.data[key] = current +} + +func (c *QuantityCache[K]) RemovePendingDelete(key K, uid types.UID) bool { + now := c.clock() + + c.mu.Lock() + defer c.mu.Unlock() + + current, ok := c.data[key] + if !ok { + return false + } + + idx := indexPendingDelete(current.PendingDeletes, uid) + if idx == -1 { + return false + } + + current.PendingDeletes = slices.Delete(current.PendingDeletes, idx, idx+1) + current.UpdatedAt = now + current.LastAccess = now + + if current.Reserved.IsZero() && len(current.PendingDeletes) == 0 { + delete(c.data, key) + + return true + } + + c.data[key] = current + + return true +} + +func (c *QuantityCache[K]) PendingDeletes(key K) []types.UID { + c.mu.RLock() + defer c.mu.RUnlock() + + current, ok := c.data[key] + if !ok || len(current.PendingDeletes) == 0 { + return nil + } + + out := make([]types.UID, 0, len(current.PendingDeletes)) + for _, hint := range current.PendingDeletes { + out = append(out, hint.UID) + } + + return out +} + +func (c *QuantityCache[K]) HasPendingDeletes(key K) bool { + c.mu.RLock() + defer c.mu.RUnlock() + + current, ok := c.data[key] + + return ok && len(current.PendingDeletes) > 0 +} + +func copyEntry(in QuantityEntry) QuantityEntry { + out := QuantityEntry{ + Reserved: in.Reserved.DeepCopy(), + CreatedAt: in.CreatedAt, + UpdatedAt: in.UpdatedAt, + LastAccess: in.LastAccess, + } + + if len(in.Reservations) > 0 { + out.Reservations = make(map[string]Reservation, len(in.Reservations)) + for id, r := range in.Reservations { + out.Reservations[id] = Reservation{ + ID: r.ID, + Usage: r.Usage.DeepCopy(), + UID: r.UID, + Group: r.Group, + Version: r.Version, + Kind: r.Kind, + Namespace: r.Namespace, + Name: r.Name, + CreatedAt: r.CreatedAt, + UpdatedAt: r.UpdatedAt, + } + } + } + + if len(in.PendingDeletes) > 0 { + out.PendingDeletes = make([]PendingDeleteHint, len(in.PendingDeletes)) + copy(out.PendingDeletes, in.PendingDeletes) + } + + return out +} + +func containsPendingDelete(in []PendingDeleteHint, uid types.UID) bool { + return indexPendingDelete(in, uid) != -1 +} + +func indexPendingDelete(in []PendingDeleteHint, uid types.UID) int { + for i := range in { + if in[i].UID == uid { + return i + } + } + + return -1 +} + +func sumReservations(in map[string]Reservation) resource.Quantity { + total := resource.MustParse("0") + for _, r := range in { + total.Add(r.Usage) + } + + if total.Sign() < 0 { + total = resource.MustParse("0") + } + + return total +} diff --git a/internal/cache/quantities_test.go b/internal/cache/quantities_test.go new file mode 100644 index 00000000..a05cf1f4 --- /dev/null +++ b/internal/cache/quantities_test.go @@ -0,0 +1,438 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "sync" + "sync/atomic" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("QuantityCache", func() { + var ( + c *QuantityCache[string] + now time.Time + key string + limit resource.Quantity + used resource.Quantity + ) + + reservation := func(id string, qty string) Reservation { + return Reservation{ + ID: id, + Usage: resource.MustParse(qty), + UID: types.UID("uid-" + id), + Group: "", + Version: "v1", + Kind: "Pod", + Namespace: "default", + Name: "pod-" + id, + } + } + + BeforeEach(func() { + now = time.Date(2026, 4, 28, 12, 0, 0, 0, time.UTC) + c = NewQuantityCache[string]() + c.clock = func() time.Time { return now } + + key = "quota-a" + limit = resource.MustParse("5") + used = resource.MustParse("0") + }) + + It("returns false for missing entries", func() { + entry, ok := c.Get(key) + + Expect(ok).To(BeFalse()) + Expect(entry.Reservations).To(BeNil()) + Expect(entry.PendingDeletes).To(BeNil()) + Expect(entry.Reserved.IsZero()).To(BeTrue()) + }) + + It("creates a reservation when allowed", func() { + allowed, effectiveUsed, entry := c.UpsertReservation(key, reservation("a", "1"), used, limit) + + Expect(allowed).To(BeTrue()) + Expect(effectiveUsed.Cmp(resource.MustParse("1"))).To(Equal(0)) + Expect(entry.Reserved.Cmp(resource.MustParse("1"))).To(Equal(0)) + Expect(entry.Reservations).To(HaveLen(1)) + Expect(entry.Reservations["a"].CreatedAt).To(Equal(now)) + Expect(entry.Reservations["a"].UpdatedAt).To(Equal(now)) + Expect(c.Len()).To(Equal(1)) + }) + + It("rejects a reservation when it would exceed the limit", func() { + allowed, effectiveUsed, entry := c.UpsertReservation( + key, + reservation("a", "2"), + resource.MustParse("4"), + resource.MustParse("5"), + ) + + Expect(allowed).To(BeFalse()) + Expect(effectiveUsed.Cmp(resource.MustParse("4"))).To(Equal(0)) + Expect(entry.Reserved.IsZero()).To(BeTrue()) + Expect(c.Len()).To(Equal(0)) + }) + + It("keeps existing reservations unchanged when a new reservation is rejected", func() { + allowed, _, _ := c.UpsertReservation(key, reservation("a", "2"), used, limit) + Expect(allowed).To(BeTrue()) + + allowed, effectiveUsed, entry := c.UpsertReservation( + key, + reservation("b", "4"), + used, + limit, + ) + + Expect(allowed).To(BeFalse()) + Expect(effectiveUsed.Cmp(resource.MustParse("2"))).To(Equal(0)) + Expect(entry.Reserved.Cmp(resource.MustParse("2"))).To(Equal(0)) + Expect(entry.Reservations).To(HaveLen(1)) + Expect(entry.Reservations).To(HaveKey("a")) + + stored, ok := c.Get(key) + Expect(ok).To(BeTrue()) + Expect(stored.Reserved.Cmp(resource.MustParse("2"))).To(Equal(0)) + Expect(stored.Reservations).To(HaveLen(1)) + }) + + It("is idempotent for the same reservation identity and usage", func() { + allowed, _, first := c.UpsertReservation(key, reservation("a", "1"), used, limit) + Expect(allowed).To(BeTrue()) + + now = now.Add(time.Minute) + + allowed, effectiveUsed, second := c.UpsertReservation(key, reservation("a", "1"), used, limit) + + Expect(allowed).To(BeTrue()) + Expect(effectiveUsed.Cmp(resource.MustParse("1"))).To(Equal(0)) + Expect(second.Reserved.Cmp(resource.MustParse("1"))).To(Equal(0)) + Expect(second.Reservations).To(HaveLen(1)) + Expect(second.Reservations["a"].CreatedAt).To(Equal(first.Reservations["a"].CreatedAt)) + Expect(second.Reservations["a"].UpdatedAt).To(Equal(first.Reservations["a"].UpdatedAt)) + Expect(second.LastAccess).To(Equal(now)) + }) + + It("updates an existing reservation when usage changes", func() { + allowed, _, first := c.UpsertReservation(key, reservation("a", "1"), used, limit) + Expect(allowed).To(BeTrue()) + + now = now.Add(time.Minute) + + allowed, effectiveUsed, second := c.UpsertReservation(key, reservation("a", "3"), used, limit) + + Expect(allowed).To(BeTrue()) + Expect(effectiveUsed.Cmp(resource.MustParse("3"))).To(Equal(0)) + Expect(second.Reserved.Cmp(resource.MustParse("3"))).To(Equal(0)) + Expect(second.Reservations).To(HaveLen(1)) + Expect(second.Reservations["a"].CreatedAt).To(Equal(first.Reservations["a"].CreatedAt)) + Expect(second.Reservations["a"].UpdatedAt).To(Equal(now)) + }) + + It("updates an existing reservation when object identity changes", func() { + res := reservation("a", "1") + + allowed, _, first := c.UpsertReservation(key, res, used, limit) + Expect(allowed).To(BeTrue()) + + now = now.Add(time.Minute) + res.Name = "renamed-pod" + + allowed, _, second := c.UpsertReservation(key, res, used, limit) + + Expect(allowed).To(BeTrue()) + Expect(second.Reservations["a"].Name).To(Equal("renamed-pod")) + Expect(second.Reservations["a"].CreatedAt).To(Equal(first.Reservations["a"].CreatedAt)) + Expect(second.Reservations["a"].UpdatedAt).To(Equal(now)) + }) + + It("returns defensive copies from Get", func() { + allowed, _, _ := c.UpsertReservation(key, reservation("a", "1"), used, limit) + Expect(allowed).To(BeTrue()) + + entry, ok := c.Get(key) + Expect(ok).To(BeTrue()) + + entry.Reserved = resource.MustParse("999") + entry.Reservations["a"] = reservation("mutated", "999") + entry.PendingDeletes = append(entry.PendingDeletes, PendingDeleteHint{UID: "fake"}) + + again, ok := c.Get(key) + Expect(ok).To(BeTrue()) + Expect(again.Reserved.Cmp(resource.MustParse("1"))).To(Equal(0)) + Expect(again.Reservations).To(HaveKey("a")) + Expect(again.Reservations).NotTo(HaveKey("mutated")) + Expect(again.PendingDeletes).To(BeEmpty()) + }) + + It("returns defensive copies from Snapshot", func() { + allowed, _, _ := c.UpsertReservation(key, reservation("a", "1"), used, limit) + Expect(allowed).To(BeTrue()) + + snap := c.Snapshot() + snap[key] = QuantityEntry{ + Reserved: resource.MustParse("999"), + } + + entry, ok := c.Get(key) + Expect(ok).To(BeTrue()) + Expect(entry.Reserved.Cmp(resource.MustParse("1"))).To(Equal(0)) + }) + + It("deletes entries", func() { + allowed, _, _ := c.UpsertReservation(key, reservation("a", "1"), used, limit) + Expect(allowed).To(BeTrue()) + + Expect(c.Delete(key)).To(BeTrue()) + Expect(c.Delete(key)).To(BeFalse()) + Expect(c.Len()).To(Equal(0)) + }) + + It("deletes reservations and removes empty entries", func() { + allowed, _, _ := c.UpsertReservation(key, reservation("a", "1"), used, limit) + Expect(allowed).To(BeTrue()) + + Expect(c.DeleteReservation(key, "a")).To(BeTrue()) + Expect(c.Len()).To(Equal(0)) + + _, ok := c.Get(key) + Expect(ok).To(BeFalse()) + }) + + It("deletes only the selected reservation and recomputes reserved", func() { + Expect(c.UpsertReservation(key, reservation("a", "1"), used, limit)).To(ReceiveAllowed()) + Expect(c.UpsertReservation(key, reservation("b", "2"), used, limit)).To(ReceiveAllowed()) + + Expect(c.DeleteReservation(key, "a")).To(BeTrue()) + + entry, ok := c.Get(key) + Expect(ok).To(BeTrue()) + Expect(entry.Reserved.Cmp(resource.MustParse("2"))).To(Equal(0)) + Expect(entry.Reservations).To(HaveKey("b")) + Expect(entry.Reservations).NotTo(HaveKey("a")) + }) + + It("returns false when deleting a missing reservation", func() { + Expect(c.DeleteReservation(key, "missing")).To(BeFalse()) + + allowed, _, _ := c.UpsertReservation(key, reservation("a", "1"), used, limit) + Expect(allowed).To(BeTrue()) + + Expect(c.DeleteReservation(key, "missing")).To(BeFalse()) + }) + + It("purges reservations for a key", func() { + Expect(c.UpsertReservation(key, reservation("a", "1"), used, limit)).To(ReceiveAllowed()) + Expect(c.UpsertReservation(key, reservation("b", "2"), used, limit)).To(ReceiveAllowed()) + Expect(c.UpsertReservation(key, reservation("c", "1"), used, limit)).To(ReceiveAllowed()) + + deleted := c.PurgeReservationsForKey(key, func(r Reservation) bool { + return r.ID == "a" || r.ID == "c" + }) + + Expect(deleted).To(Equal(2)) + + entry, ok := c.Get(key) + Expect(ok).To(BeTrue()) + Expect(entry.Reserved.Cmp(resource.MustParse("2"))).To(Equal(0)) + Expect(entry.Reservations).To(HaveLen(1)) + Expect(entry.Reservations).To(HaveKey("b")) + }) + + It("purges all reservations and removes entry when no pending deletes remain", func() { + Expect(c.UpsertReservation(key, reservation("a", "1"), used, limit)).To(ReceiveAllowed()) + + deleted := c.PurgeReservationsForKey(key, func(r Reservation) bool { + return true + }) + + Expect(deleted).To(Equal(1)) + Expect(c.Len()).To(Equal(0)) + }) + + It("does not remove entry after purging all reservations if pending deletes remain", func() { + Expect(c.UpsertReservation(key, reservation("a", "1"), used, limit)).To(ReceiveAllowed()) + c.AddPendingDelete(key, "uid-a") + + deleted := c.PurgeReservationsForKey(key, func(r Reservation) bool { + return true + }) + + Expect(deleted).To(Equal(1)) + + entry, ok := c.Get(key) + Expect(ok).To(BeTrue()) + Expect(entry.Reserved.IsZero()).To(BeTrue()) + Expect(entry.PendingDeletes).To(HaveLen(1)) + }) + + It("returns zero when purging a missing key or no reservation matches", func() { + Expect(c.PurgeReservationsForKey("missing", func(r Reservation) bool { return true })).To(Equal(0)) + + Expect(c.UpsertReservation(key, reservation("a", "1"), used, limit)).To(ReceiveAllowed()) + + Expect(c.PurgeReservationsForKey(key, func(r Reservation) bool { return false })).To(Equal(0)) + }) + + It("adds pending deletes idempotently", func() { + c.AddPendingDelete(key, "uid-a") + c.AddPendingDelete(key, "uid-a") + + Expect(c.HasPendingDeletes(key)).To(BeTrue()) + Expect(c.PendingDeletes(key)).To(Equal([]types.UID{"uid-a"})) + + entry, ok := c.Get(key) + Expect(ok).To(BeTrue()) + Expect(entry.PendingDeletes[0].CreatedAt).To(Equal(now)) + }) + + It("ignores empty pending delete UIDs", func() { + c.AddPendingDelete(key, "") + + Expect(c.Len()).To(Equal(0)) + Expect(c.PendingDeletes(key)).To(BeNil()) + }) + + It("removes pending deletes and removes empty entry", func() { + c.AddPendingDelete(key, "uid-a") + + Expect(c.RemovePendingDelete(key, "uid-a")).To(BeTrue()) + Expect(c.RemovePendingDelete(key, "uid-a")).To(BeFalse()) + Expect(c.Len()).To(Equal(0)) + }) + + It("removes pending delete but keeps entry when reservations remain", func() { + Expect(c.UpsertReservation(key, reservation("a", "1"), used, limit)).To(ReceiveAllowed()) + c.AddPendingDelete(key, "uid-a") + + Expect(c.RemovePendingDelete(key, "uid-a")).To(BeTrue()) + + entry, ok := c.Get(key) + Expect(ok).To(BeTrue()) + Expect(entry.Reserved.Cmp(resource.MustParse("1"))).To(Equal(0)) + Expect(entry.PendingDeletes).To(BeEmpty()) + }) + + It("returns nil pending deletes for missing keys", func() { + Expect(c.PendingDeletes("missing")).To(BeNil()) + Expect(c.HasPendingDeletes("missing")).To(BeFalse()) + }) + + It("clamps negative reserved totals to zero", func() { + Expect(c.UpsertReservation(key, reservation("a", "-5"), used, limit)).To(ReceiveAllowed()) + + entry, ok := c.Get(key) + Expect(ok).To(BeTrue()) + Expect(entry.Reserved.IsZero()).To(BeTrue()) + }) + + It("is safe under concurrent reservation upserts", func() { + const workers = 32 + + c.clock = time.Now + limit := resource.MustParse("100") + var allowedCount atomic.Int32 + + start := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(workers) + + for i := 0; i < workers; i++ { + i := i + go func() { + defer GinkgoRecover() + defer wg.Done() + + <-start + + allowed, _, _ := c.UpsertReservation( + key, + reservation(string(rune('a'+i)), "1"), + resource.MustParse("0"), + limit, + ) + if allowed { + allowedCount.Add(1) + } + }() + } + + close(start) + wg.Wait() + + Expect(allowedCount.Load()).To(Equal(int32(workers))) + + entry, ok := c.Get(key) + Expect(ok).To(BeTrue()) + Expect(entry.Reservations).To(HaveLen(workers)) + Expect(entry.Reserved.Cmp(resource.MustParse("32"))).To(Equal(0)) + }) + + It("enforces limit under concurrent reservation upserts", func() { + const workers = 16 + + c.clock = time.Now + limit := resource.MustParse("5") + var allowedCount atomic.Int32 + + start := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(workers) + + for i := 0; i < workers; i++ { + i := i + go func() { + defer GinkgoRecover() + defer wg.Done() + + <-start + + allowed, _, _ := c.UpsertReservation( + key, + reservation(string(rune('a'+i)), "1"), + resource.MustParse("0"), + limit, + ) + if allowed { + allowedCount.Add(1) + } + }() + } + + close(start) + wg.Wait() + + Expect(allowedCount.Load()).To(Equal(int32(5))) + + entry, ok := c.Get(key) + Expect(ok).To(BeTrue()) + Expect(entry.Reservations).To(HaveLen(5)) + Expect(entry.Reserved.Cmp(resource.MustParse("5"))).To(Equal(0)) + }) +}) + +type quantityResult struct { + allowed bool + effectiveUsed resource.Quantity + entry QuantityEntry +} + +func ReceiveAllowed() OmegaMatcher { + return WithTransform(func(in any) bool { + switch v := in.(type) { + case quantityResult: + return v.allowed + default: + return false + } + }, BeTrue()) +} diff --git a/internal/cache/registries.go b/internal/cache/registries.go index 0dc021b5..035b872a 100644 --- a/internal/cache/registries.go +++ b/internal/cache/registries.go @@ -176,6 +176,13 @@ func (c *RegistryRuleSetCache) Has(id string) bool { return ok } +func (c *RegistryRuleSetCache) Reset() { + c.mu.Lock() + defer c.mu.Unlock() + + c.rs = make(map[string]*RuleSet) +} + // InsertForTest can be behind a build tag if you prefer, but it's fine to keep simple. // //nolint:unused diff --git a/internal/controllers/admission/mutating.go b/internal/controllers/admission/mutating.go index 5b4aa80a..79e96ec8 100644 --- a/internal/controllers/admission/mutating.go +++ b/internal/controllers/admission/mutating.go @@ -13,17 +13,21 @@ import ( admissionv1 "k8s.io/api/admissionregistration/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/controllers/utils" "github.com/projectcapsule/capsule/pkg/api/meta" - clt "github.com/projectcapsule/capsule/pkg/runtime/client" + "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/predicates" ) @@ -39,9 +43,28 @@ func (r *mutatingReconciler) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils return ctrl.NewControllerManagedBy(mgr). Named("capsule/admission/mutating"). For( - &capsulev1beta2.CapsuleConfiguration{}, + &admissionv1.MutatingWebhookConfiguration{}, builder.WithPredicates( predicate.GenerationChangedPredicate{}, + predicates.NamesMatchingPredicate{Names: []string{string(r.configuration.Admission().Mutating.Name)}}, + ), + ). + Watches( + &capsulev1beta2.CapsuleConfiguration{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { + return []reconcile.Request{{ + NamespacedName: types.NamespacedName{Name: string(r.configuration.Admission().Mutating.Name)}, + }} + }), + builder.WithPredicates( + predicate.Or( + predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return e.Object.GetName() == ctrlConfig.ConfigurationName + }, + }, + predicates.CapsuleConfigSpecAdmissionChangedPredicate{}, + ), predicates.NamesMatchingPredicate{Names: []string{ctrlConfig.ConfigurationName}}, ), ). @@ -57,16 +80,16 @@ func (r *mutatingReconciler) Reconcile(ctx context.Context, request reconcile.Re func (r *mutatingReconciler) reconcileConfiguration( ctx context.Context, - cfg capsulev1beta2.DynamicAdmissionConfig, + cfg *capsulev1beta2.DynamicMutatingAdmissionConfig, ) error { desiredName := string(cfg.Name) - hooks, err := r.webhooks(ctx, cfg) + desiredHooks, err := r.webhooks(ctx, cfg) if err != nil { return err } - if len(hooks) == 0 { + if len(desiredHooks) == 0 { managed, err := r.listManagedWebhookConfigs(ctx) if err != nil { return err @@ -81,33 +104,55 @@ func (r *mutatingReconciler) reconcileConfiguration( return nil } + sort.Slice(desiredHooks, func(i, j int) bool { return desiredHooks[i].Name < desiredHooks[j].Name }) + obj := &admissionv1.MutatingWebhookConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: admissionv1.SchemeGroupVersion.String(), + Kind: "MutatingWebhookConfiguration", + }, ObjectMeta: metav1.ObjectMeta{Name: string(cfg.Name)}, } - sort.Slice(hooks, func(i, j int) bool { return hooks[i].Name < hooks[j].Name }) + _, err = controllerutil.CreateOrUpdate(ctx, r.client, obj, func() error { + if err := controllerutil.SetOwnerReference( + r.configuration.GetConfigObject(), + obj, + r.client.Scheme(), + ); err != nil { + return err + } - labels := obj.GetLabels() - if labels == nil { - labels = make(map[string]string) - } + labels := obj.GetLabels() + if labels == nil { + labels = make(map[string]string) + } - maps.Copy(labels, cfg.Labels) + maps.Copy(labels, cfg.Labels) - labels[meta.CreatedByCapsuleLabel] = meta.ValueController + labels[meta.CreatedByCapsuleLabel] = meta.ValueController - obj.SetLabels(labels) + obj.SetLabels(labels) - annotations := obj.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } - maps.Copy(annotations, cfg.Annotations) + maps.Copy(annotations, cfg.Annotations) - obj.SetAnnotations(annotations) + obj.SetAnnotations(annotations) - if err := clt.CreateOrPatch(ctx, r.client, obj, meta.FieldManagerCapsuleController, true); err != nil { + // Do not overwrite caBundle. cert-manager or the legacy TLS reconciler owns it. + if cfg.Client.CABundle == nil { + obj.Webhooks = preserveMutatingWebhookCABundles(obj.Webhooks, desiredHooks) + } else { + obj.Webhooks = desiredHooks + } + + return err + }) + if err != nil { return err } @@ -160,7 +205,39 @@ func (r *mutatingReconciler) deleteWebhookConfig(ctx context.Context, name strin func (r *mutatingReconciler) webhooks( ctx context.Context, - cfg capsulev1beta2.DynamicAdmissionConfig, + cfg *capsulev1beta2.DynamicMutatingAdmissionConfig, ) (hooks []admissionv1.MutatingWebhook, err error) { - return + for _, hook := range cfg.Webhooks { + h, err := admission.NewMutatingWebhook(hook, cfg.Client, r.configuration.Users(), r.configuration.Administrators()) + if err != nil { + return nil, err + } + + hooks = append(hooks, h) + } + + return hooks, nil +} + +func preserveMutatingWebhookCABundles( + existing []admissionv1.MutatingWebhook, + desired []admissionv1.MutatingWebhook, +) []admissionv1.MutatingWebhook { + existingByName := make(map[string][]byte, len(existing)) + + for _, hook := range existing { + if len(hook.ClientConfig.CABundle) == 0 { + continue + } + + existingByName[hook.Name] = append([]byte(nil), hook.ClientConfig.CABundle...) + } + + for i := range desired { + if caBundle, ok := existingByName[desired[i].Name]; ok { + desired[i].ClientConfig.CABundle = append([]byte(nil), caBundle...) + } + } + + return desired } diff --git a/internal/controllers/admission/validating.go b/internal/controllers/admission/validating.go index 5ba7f90b..ebff47a2 100644 --- a/internal/controllers/admission/validating.go +++ b/internal/controllers/admission/validating.go @@ -13,16 +13,23 @@ import ( admissionv1 "k8s.io/api/admissionregistration/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/controllers/utils" "github.com/projectcapsule/capsule/pkg/api/meta" - clt "github.com/projectcapsule/capsule/pkg/runtime/client" + "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/predicates" ) type validatingReconciler struct { @@ -35,7 +42,32 @@ type validatingReconciler struct { func (r *validatingReconciler) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.ControllerOptions) error { return ctrl.NewControllerManagedBy(mgr). Named("capsule/admission/validating"). - For(&capsulev1beta2.CapsuleConfiguration{}, utils.NamesMatchingPredicate(ctrlConfig.ConfigurationName)). + For( + &admissionv1.ValidatingWebhookConfiguration{}, + builder.WithPredicates( + predicate.GenerationChangedPredicate{}, + predicates.NamesMatchingPredicate{Names: []string{string(r.configuration.Admission().Validating.Name)}}, + ), + ). + Watches( + &capsulev1beta2.CapsuleConfiguration{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { + return []reconcile.Request{{ + NamespacedName: types.NamespacedName{Name: string(r.configuration.Admission().Validating.Name)}, + }} + }), + builder.WithPredicates( + predicate.Or( + predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return e.Object.GetName() == ctrlConfig.ConfigurationName + }, + }, + predicates.CapsuleConfigSpecAdmissionChangedPredicate{}, + ), + predicates.NamesMatchingPredicate{Names: []string{ctrlConfig.ConfigurationName}}, + ), + ). WithOptions(controller.Options{MaxConcurrentReconciles: ctrlConfig.MaxConcurrentReconciles}). Complete(r) } @@ -48,16 +80,16 @@ func (r *validatingReconciler) Reconcile(ctx context.Context, request reconcile. func (r *validatingReconciler) reconcileValidatingConfiguration( ctx context.Context, - cfg capsulev1beta2.DynamicAdmissionConfig, + cfg *capsulev1beta2.DynamicValidatingAdmissionConfig, ) error { desiredName := string(cfg.Name) - hooks, err := r.validatingWebhooks(ctx, cfg) + desiredHooks, err := r.validatingWebhooks(ctx, cfg) if err != nil { return err } - if len(hooks) == 0 { + if len(desiredHooks) == 0 { managed, err := r.listManagedValidatingWebhookConfigs(ctx) if err != nil { return err @@ -72,33 +104,57 @@ func (r *validatingReconciler) reconcileValidatingConfiguration( return nil } + sort.Slice(desiredHooks, func(i, j int) bool { return desiredHooks[i].Name < desiredHooks[j].Name }) + obj := &admissionv1.ValidatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{Name: string(cfg.Name)}, + TypeMeta: metav1.TypeMeta{ + APIVersion: admissionv1.SchemeGroupVersion.String(), + Kind: "ValidatingWebhookConfiguration", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: string(cfg.Name), + }, } - sort.Slice(hooks, func(i, j int) bool { return hooks[i].Name < hooks[j].Name }) + _, err = controllerutil.CreateOrUpdate(ctx, r.client, obj, func() error { + if err := controllerutil.SetOwnerReference( + r.configuration.GetConfigObject(), + obj, + r.client.Scheme(), + ); err != nil { + return err + } - labels := obj.GetLabels() - if labels == nil { - labels = make(map[string]string) - } + labels := obj.GetLabels() + if labels == nil { + labels = make(map[string]string) + } - maps.Copy(labels, cfg.Labels) + maps.Copy(labels, cfg.Labels) - labels[meta.CreatedByCapsuleLabel] = meta.ValueController + labels[meta.CreatedByCapsuleLabel] = meta.ValueController - obj.SetLabels(labels) + obj.SetLabels(labels) - annotations := obj.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } - maps.Copy(annotations, cfg.Annotations) + maps.Copy(annotations, cfg.Annotations) - obj.SetAnnotations(annotations) + obj.SetAnnotations(annotations) - if err := clt.CreateOrPatch(ctx, r.client, obj, meta.FieldManagerCapsuleController, true); err != nil { + // Do not overwrite caBundle. cert-manager or the legacy TLS reconciler owns it. + if cfg.Client.CABundle == nil { + obj.Webhooks = preserveValidatingWebhookCABundles(obj.Webhooks, desiredHooks) + } else { + obj.Webhooks = desiredHooks + } + + return err + }) + if err != nil { return err } @@ -151,7 +207,43 @@ func (r *validatingReconciler) deleteValidatingWebhookConfig(ctx context.Context func (r *validatingReconciler) validatingWebhooks( ctx context.Context, - cfg capsulev1beta2.DynamicAdmissionConfig, + cfg *capsulev1beta2.DynamicValidatingAdmissionConfig, ) (hooks []admissionv1.ValidatingWebhook, err error) { - return + for _, hook := range cfg.Webhooks { + h, err := admission.NewValidatingWebhook(hook, cfg.Client, r.configuration.Users(), r.configuration.Administrators()) + if err != nil { + return nil, err + } + + hooks = append(hooks, h) + } + + return hooks, nil +} + +func preserveValidatingWebhookCABundles( + existing []admissionv1.ValidatingWebhook, + desired []admissionv1.ValidatingWebhook, +) []admissionv1.ValidatingWebhook { + existingByName := make(map[string][]byte, len(existing)) + + for _, hook := range existing { + if len(hook.ClientConfig.CABundle) == 0 { + continue + } + + existingByName[hook.Name] = append([]byte(nil), hook.ClientConfig.CABundle...) + } + + for i := range desired { + if len(desired[i].ClientConfig.CABundle) > 0 { + continue + } + + if caBundle, ok := existingByName[desired[i].Name]; ok { + desired[i].ClientConfig.CABundle = append([]byte(nil), caBundle...) + } + } + + return desired } diff --git a/internal/controllers/cfg/cache_registries.go b/internal/controllers/cfg/cache_registries.go deleted file mode 100644 index 5a354213..00000000 --- a/internal/controllers/cfg/cache_registries.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package config - -import ( - "context" - - "github.com/go-logr/logr" - "sigs.k8s.io/controller-runtime/pkg/client" - - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api/meta" -) - -func (r *Manager) getItemsForStatusRegistryCache(ctx context.Context) ([]capsulev1beta2.RuleStatus, error) { - rsList := &capsulev1beta2.RuleStatusList{} - if err := r.List(ctx, rsList, - client.MatchingLabels{ - meta.NewManagedByCapsuleLabel: meta.ValueController, - meta.CapsuleNameLabel: meta.NameForManagedRuleStatus(), - }, - ); err != nil { - return nil, err - } - - return rsList.Items, nil -} - -func (r *Manager) warmupRuleStatusRegistryCache(ctx context.Context, log logr.Logger, items []capsulev1beta2.RuleStatus) error { - for _, item := range items { - regs := item.Status.Rule.Enforce.Registries - if len(regs) == 0 { - continue - } - - if _, _, err := r.RegistryCache.GetOrBuild(regs); err != nil { - return err - } - } - - log.V(5).Info("warmed up cache based on existing rules", "rules", len(items), "cache_rules", r.RegistryCache.Stats()) - - return nil -} - -func (r *Manager) invalidateRuleStatusRegistryCache(ctx context.Context, log logr.Logger) error { - items, err := r.getItemsForStatusRegistryCache(ctx) - if err != nil { - return err - } - - log.V(5).Info("cached before invalidation", "cache_rules", r.RegistryCache.Stats()) - - active := make(map[string]struct{}, len(items)) - - for _, item := range items { - regs := item.Status.Rule.Enforce.Registries - if len(regs) == 0 { - continue - } - - id := r.RegistryCache.HashRules(regs) - active[id] = struct{}{} - } - - _ = r.RegistryCache.PruneActive(active) - - log.V(5).Info("cached after invalidation", "rules", len(items), "cache_rules", r.RegistryCache.Stats()) - - return nil -} diff --git a/internal/controllers/cfg/caches.go b/internal/controllers/cfg/caches.go deleted file mode 100644 index 71abe6ff..00000000 --- a/internal/controllers/cfg/caches.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package config - -import ( - "context" - - "github.com/go-logr/logr" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/util/retry" - "sigs.k8s.io/controller-runtime/pkg/client" - - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" -) - -// invalidateCaches invokes for all caches their invalidation functions. -func (r *Manager) invalidateCaches(ctx context.Context, log logr.Logger) error { - err := r.invalidateRuleStatusRegistryCache(ctx, log) - if err != nil { - return err - } - - now := metav1.Now() - - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - cfg := &capsulev1beta2.CapsuleConfiguration{} - if err := r.Get(ctx, client.ObjectKey{Name: r.configName}, cfg); err != nil { - return err - } - - cfg.Status.LastCacheInvalidation = now - - return r.Status().Update(ctx, cfg) - }) -} - -// populateCaches warms up all custom caches. -func (r *Manager) populateCaches(ctx context.Context, log logr.Logger) error { - items, err := r.getItemsForStatusRegistryCache(ctx) - if err != nil { - return err - } - - err = r.warmupRuleStatusRegistryCache(ctx, log, items) - if err != nil { - return err - } - - return nil -} diff --git a/internal/controllers/cfg/invalidator/clients.go b/internal/controllers/cfg/invalidator/clients.go new file mode 100644 index 00000000..6ae7d72d --- /dev/null +++ b/internal/controllers/cfg/invalidator/clients.go @@ -0,0 +1,189 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package invalidator + +import ( + "context" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/indexers/tenantresource" +) + +func (r *CacheInvalidator) rebuildImpersonationCache( + ctx context.Context, + log logr.Logger, +) error { + var referencedServiceAccounts []meta.NamespacedRFC1123ObjectReferenceWithNamespace + + seen := make(map[string]struct{}) + + var gtr capsulev1beta2.GlobalTenantResourceList + if err := r.List(ctx, >r); err != nil { + return err + } + + for _, item := range gtr.Items { + if item.Status.ServiceAccount == nil { + continue + } + + saName := item.Status.ServiceAccount.Name + saNamespace := item.Status.ServiceAccount.Namespace + + key := saNamespace.String() + "/" + saName.String() + if _, ok := seen[key]; ok { + continue + } + + sa := &corev1.ServiceAccount{} + if err := r.Get(ctx, types.NamespacedName{ + Namespace: saNamespace.String(), + Name: saName.String(), + }, sa); err != nil { + if apierrors.IsNotFound(err) { + continue + } + + return err + } + + seen[key] = struct{}{} + + referencedServiceAccounts = append(referencedServiceAccounts, meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: meta.RFC1123Name(sa.Name), + Namespace: meta.RFC1123SubdomainName(sa.Namespace), + }) + } + + var ntr capsulev1beta2.TenantResourceList + if err := r.List(ctx, &ntr); err != nil { + return err + } + + for _, item := range ntr.Items { + if item.Status.ServiceAccount == nil { + continue + } + + saName := item.Status.ServiceAccount.Name + saNamespace := item.Status.ServiceAccount.Namespace + + key := string(saNamespace) + "/" + string(saName) + if _, ok := seen[key]; ok { + continue + } + + sa := &corev1.ServiceAccount{} + if err := r.Get(ctx, types.NamespacedName{ + Namespace: saNamespace.String(), + Name: saName.String(), + }, sa); err != nil { + if apierrors.IsNotFound(err) { + continue + } + + return err + } + + seen[key] = struct{}{} + + referencedServiceAccounts = append(referencedServiceAccounts, meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: meta.RFC1123Name(sa.Name), + Namespace: meta.RFC1123SubdomainName(sa.Namespace), + }) + } + + log.V(5).Info("rebuilding impersonation cache", + "serviceAccounts", len(referencedServiceAccounts), + "cacheBefore", r.ImpersonationCache.Stats(), + ) + + r.ImpersonationCache.Reset() + + re, err := r.Configuration.ServiceAccountClient(ctx) + if err != nil { + log.Error(err, "failed to load impersonated rest client") + + return err + } + + for _, sa := range referencedServiceAccounts { + if _, err := r.ImpersonationCache.LoadOrCreate( + ctx, + log, + re, + r.Scheme(), + sa, + ); err != nil { + return err + } + } + + log.V(5).Info("rebuilt impersonation cache", + "serviceAccounts", len(referencedServiceAccounts), + "cacheAfter", r.ImpersonationCache.Stats(), + ) + + return nil +} + +func (r *CacheInvalidator) invalidateServiceAccount( + ctx context.Context, + sa *corev1.ServiceAccount, +) error { + hasReference, err := r.checkServiceAccountReferences(ctx, sa) + if err != nil { + return err + } + + if !hasReference { + r.Log.V(4).Info("invalidating cache for serviceaccount cache", "name", sa.GetNamespace(), "namespace", sa.GetName()) + + r.ImpersonationCache.Invalidate(sa.GetNamespace(), sa.GetName()) + } + + return nil +} + +func (r *CacheInvalidator) checkServiceAccountReferences( + ctx context.Context, + sa *corev1.ServiceAccount, +) (ref bool, err error) { + key := sa.GetNamespace() + "/" + sa.GetName() + + var gtr capsulev1beta2.GlobalTenantResourceList + if err := r.List( + ctx, + >r, + client.MatchingFields{tenantresource.ServiceAccountIndexerFieldName: key}, + ); err != nil { + return false, err + } + + if len(gtr.Items) > 0 { + return true, nil + } + + var ntr capsulev1beta2.TenantResourceList + if err := r.List( + ctx, + &ntr, + client.MatchingFields{tenantresource.ServiceAccountIndexerFieldName: key}, + ); err != nil { + return false, err + } + + if len(ntr.Items) > 0 { + return true, nil + } + + return false, nil +} diff --git a/internal/controllers/cfg/invalidator/jsonpath.go b/internal/controllers/cfg/invalidator/jsonpath.go new file mode 100644 index 00000000..40b8b1d7 --- /dev/null +++ b/internal/controllers/cfg/invalidator/jsonpath.go @@ -0,0 +1,79 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package invalidator + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +func (r *CacheInvalidator) rebuildJSONPathCache(ctx context.Context, log logr.Logger) error { + customQuotas := &capsulev1beta2.CustomQuotaList{} + if err := r.List(ctx, customQuotas); err != nil { + return err + } + + globalCustomQuotas := &capsulev1beta2.GlobalCustomQuotaList{} + if err := r.List(ctx, globalCustomQuotas); err != nil { + return err + } + + log.V(5).Info("rebuilding custom quota jsonpath cache", + "jsonPathsBefore", r.JSONPathCache.Stats(), + "customQuotas", len(customQuotas.Items), + "globalCustomQuotas", len(globalCustomQuotas.Items), + ) + + r.JSONPathCache.Reset() + + expressions := make(map[string]struct{}) + + for _, cq := range customQuotas.Items { + collectJSONPaths(expressions, cq.Spec.Sources) + } + + for _, gcq := range globalCustomQuotas.Items { + collectJSONPaths(expressions, gcq.Spec.Sources) + } + + for expr := range expressions { + if expr == "" { + continue + } + + if _, err := r.JSONPathCache.GetOrCompile(expr); err != nil { + return fmt.Errorf("build JSONPath cache entry %q: %w", expr, err) + } + } + + log.V(5).Info("rebuilt custom quota jsonpath cache", + "uniqueExpressions", len(expressions), + "jsonPathsAfter", r.JSONPathCache.Stats(), + ) + + return nil +} + +func collectJSONPaths( + set map[string]struct{}, + sources []capsulev1beta2.CustomQuotaSpecSource, +) { + for _, source := range sources { + if source.Path != "" { + set[source.Path] = struct{}{} + } + + for _, sel := range source.Selectors { + for _, fs := range sel.FieldSelectors { + if fs != "" { + set[fs] = struct{}{} + } + } + } + } +} diff --git a/internal/controllers/cfg/invalidator/manager.go b/internal/controllers/cfg/invalidator/manager.go new file mode 100644 index 00000000..53558e83 --- /dev/null +++ b/internal/controllers/cfg/invalidator/manager.go @@ -0,0 +1,195 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package invalidator + +import ( + "context" + "errors" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/rest" + "k8s.io/client-go/util/workqueue" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" + "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/predicates" +) + +type CacheInvalidator struct { + client.Client + + Rest *rest.Config + Log logr.Logger + + configName string + Configuration configuration.Configuration + + RegistryCache *cache.RegistryRuleSetCache + TargetsCache *cache.CompiledTargetsCache[string] + JSONPathCache *cache.JSONPathCache + ImpersonationCache *cache.ImpersonationCache +} + +func (r *CacheInvalidator) NeedLeaderElection() bool { + return false +} + +// Start is the Runnable function triggered upon Manager start-up to perform cache population. +func (r *CacheInvalidator) Start(ctx context.Context) error { + if err := r.rebuildCaches(ctx, r.Log); err != nil { + r.Log.Error(err, "cache population failed") + + return nil + } + + <-ctx.Done() + + return nil +} + +func (r *CacheInvalidator) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.ControllerOptions) (err error) { + r.configName = ctrlConfig.ConfigurationName + + err = ctrl.NewControllerManagedBy(mgr). + Named("config/caches"). + For( + &capsulev1beta2.CapsuleConfiguration{}, + builder.WithPredicates( + predicate.GenerationChangedPredicate{}, + predicates.NamesMatchingPredicate{Names: []string{ctrlConfig.ConfigurationName}}, + ), + ). + Watches( + &capsulev1beta2.CapsuleConfiguration{}, + handler.Funcs{ + UpdateFunc: func(ctx context.Context, updateEvent event.TypedUpdateEvent[client.Object], limitingInterface workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if err := r.rebuildImpersonationCache(ctx, r.Log); err != nil { + r.Log.Error(err, "unable to invalidate impersonation cache") + } + }, + }, + builder.WithPredicates( + predicates.CapsuleConfigSpecImpersonationChangedPredicate{}, + predicates.NamesMatchingPredicate{Names: []string{ctrlConfig.ConfigurationName}}, + ), + ). + Watches( + &corev1.ServiceAccount{}, + handler.Funcs{ + DeleteFunc: func( + ctx context.Context, + e event.TypedDeleteEvent[client.Object], + q workqueue.TypedRateLimitingInterface[reconcile.Request], + ) { + sa, ok := e.Object.(*corev1.ServiceAccount) + if !ok { + return + } + + if err := r.invalidateServiceAccount(ctx, sa); err != nil { + r.Log.Error(err, "unable to invalidate serviceaccount cache", + "namespace", sa.GetNamespace(), + "name", sa.GetName(), + ) + } + }, + }, + builder.WithPredicates(predicate.Funcs{ + DeleteFunc: func(e event.DeleteEvent) bool { + return true + }, + CreateFunc: func(e event.CreateEvent) bool { + return false + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return false + }, + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + }, + ), + ). + Complete(r) + if err != nil { + return err + } + + // register Start(ctx) as a manager runnable. + return mgr.Add(r) +} + +func (r *CacheInvalidator) Reconcile(ctx context.Context, request reconcile.Request) (res reconcile.Result, err error) { + log := r.Log.WithValues("configuration", request.Name) + + log.V(5).Info("invalidating and rebuilding caches") + + cfg := configuration.NewCapsuleConfiguration(ctx, r.Client, r.Rest, request.Name) + + instance := &capsulev1beta2.CapsuleConfiguration{} + if err = r.Get(ctx, request.NamespacedName, instance); err != nil { + if apierrors.IsNotFound(err) { + log.V(5).Info("requested object not found, could have been deleted after reconcile request") + + return reconcile.Result{}, nil + } + + log.Error(err, "error reading the object") + + return res, err + } + + if err := r.rebuildCaches(ctx, log); err != nil { + return res, err + } + + interval := cfg.CacheInvalidation() + + return reconcile.Result{ + Requeue: true, + RequeueAfter: interval.Duration, + }, err +} + +// invalidateCaches invokes for all caches their invalidation functions. +func (r *CacheInvalidator) rebuildCaches( + ctx context.Context, + log logr.Logger, +) error { + var errs []error + + if err := r.rebuildJSONPathCache(ctx, log); err != nil { + errs = append(errs, fmt.Errorf("rebuild JSONPath cache: %w", err)) + } + + if err := r.rebuildTargetsCache(ctx, log); err != nil { + errs = append(errs, fmt.Errorf("rebuild targets cache: %w", err)) + } + + if err := r.rebuildRuleStatusRegistryCache(ctx, log); err != nil { + errs = append(errs, fmt.Errorf("rebuild registry cache: %w", err)) + } + + if err := r.rebuildImpersonationCache(ctx, log); err != nil { + errs = append(errs, fmt.Errorf("rebuild impersonation cache: %w", err)) + } + + if len(errs) > 0 { + return errors.Join(errs...) + } + + return nil +} diff --git a/internal/controllers/cfg/invalidator/registries.go b/internal/controllers/cfg/invalidator/registries.go new file mode 100644 index 00000000..95304d54 --- /dev/null +++ b/internal/controllers/cfg/invalidator/registries.go @@ -0,0 +1,51 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package invalidator + +import ( + "context" + + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +func (r *CacheInvalidator) rebuildRuleStatusRegistryCache(ctx context.Context, log logr.Logger) error { + rsList := &capsulev1beta2.RuleStatusList{} + if err := r.List(ctx, rsList, + client.MatchingLabels{ + meta.NewManagedByCapsuleLabel: meta.ValueController, + meta.CapsuleNameLabel: meta.NameForManagedRuleStatus(), + }, + ); err != nil { + return err + } + + log.V(5).Info("rebuilding registry cache from existing rules", + "rules", len(rsList.Items), + "cache_rules_before", r.RegistryCache.Stats(), + ) + + r.RegistryCache.Reset() + + for _, item := range rsList.Items { + regs := item.Status.Rule.Enforce.Registries + if len(regs) == 0 { + continue + } + + if _, _, err := r.RegistryCache.GetOrBuild(regs); err != nil { + return err + } + } + + log.V(5).Info("rebuilt registry cache from existing rules", + "rules", len(rsList.Items), + "cache_rules_after", r.RegistryCache.Stats(), + ) + + return nil +} diff --git a/internal/controllers/cfg/invalidator/targets.go b/internal/controllers/cfg/invalidator/targets.go new file mode 100644 index 00000000..69dc3dd7 --- /dev/null +++ b/internal/controllers/cfg/invalidator/targets.go @@ -0,0 +1,78 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package invalidator + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/controllers/customquotas" +) + +func (r *CacheInvalidator) rebuildTargetsCache(ctx context.Context, log logr.Logger) error { + customQuotas := &capsulev1beta2.CustomQuotaList{} + if err := r.List(ctx, customQuotas); err != nil { + return err + } + + globalCustomQuotas := &capsulev1beta2.GlobalCustomQuotaList{} + if err := r.List(ctx, globalCustomQuotas); err != nil { + return err + } + + log.V(5).Info("rebuilding custom quota targets cache", + "targetsBefore", r.TargetsCache.Stats(), + "customQuotas", len(customQuotas.Items), + "globalCustomQuotas", len(globalCustomQuotas.Items), + ) + + r.TargetsCache.Reset() + + targetsByKey := make(map[string][]capsulev1beta2.CustomQuotaStatusTarget, len(customQuotas.Items)+len(globalCustomQuotas.Items)) + + for _, cq := range customQuotas.Items { + key := customquotas.MakeCustomQuotaCacheKey(cq.GetNamespace(), cq.GetName()) + targetsByKey[key] = customQuotaStatusTargetsFromSources(cq.Spec.Sources) + } + + for _, gcq := range globalCustomQuotas.Items { + key := customquotas.MakeGlobalCustomQuotaCacheKey(gcq.GetName()) + targetsByKey[key] = customQuotaStatusTargetsFromSources(gcq.Spec.Sources) + } + + for key, targets := range targetsByKey { + compiled, err := customquotas.CompileTargets(r.JSONPathCache, targets) + if err != nil { + return fmt.Errorf("compile targets for cache key %q: %w", key, err) + } + + r.TargetsCache.Set(key, compiled) + } + + log.V(5).Info("rebuilt custom quota targets cache", + "targets", len(targetsByKey), + "targetsAfter", r.TargetsCache.Stats(), + ) + + return nil +} + +func customQuotaStatusTargetsFromSources( + sources []capsulev1beta2.CustomQuotaSpecSource, +) []capsulev1beta2.CustomQuotaStatusTarget { + targets := make([]capsulev1beta2.CustomQuotaStatusTarget, 0, len(sources)) + + for _, source := range sources { + targets = append(targets, capsulev1beta2.CustomQuotaStatusTarget{ + GroupVersionKind: metav1.GroupVersionKind(source.GroupVersionKind()), + CustomQuotaSpecSourceConfig: source.CustomQuotaSpecSourceConfig, + }) + } + + return targets +} diff --git a/internal/controllers/cfg/manager.go b/internal/controllers/cfg/status/manager.go similarity index 78% rename from internal/controllers/cfg/manager.go rename to internal/controllers/cfg/status/manager.go index dcb5871f..5bd1a289 100644 --- a/internal/controllers/cfg/manager.go +++ b/internal/controllers/cfg/status/manager.go @@ -6,14 +6,13 @@ package config import ( "context" "fmt" - "time" "github.com/go-logr/logr" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" - "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -23,9 +22,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/cache" "github.com/projectcapsule/capsule/internal/controllers/utils" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/predicates" ) @@ -33,16 +31,16 @@ import ( type Manager struct { client.Client - configName string + Rest *rest.Config - RegistryCache *cache.RegistryRuleSetCache - Log logr.Logger + configName string + Log logr.Logger } func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.ControllerOptions) (err error) { r.configName = ctrlConfig.ConfigurationName - err = ctrl.NewControllerManagedBy(mgr). + return ctrl.NewControllerManagedBy(mgr). Named("capsule/configuration"). For( &capsulev1beta2.CapsuleConfiguration{}, @@ -51,6 +49,21 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller predicates.NamesMatchingPredicate{Names: []string{ctrlConfig.ConfigurationName}}, ), ). + Watches( + &capsulev1beta2.Tenant{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Name: ctrlConfig.ConfigurationName, + }, + }, + } + }), + builder.WithPredicates( + predicates.TenantStatusOwnersChangedPredicate{}, + ), + ). Watches( &capsulev1beta2.TenantOwner{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { @@ -98,36 +111,17 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller }), ). Complete(r) - if err != nil { - return err - } - - // register Start(ctx) as a manager runnable. - return mgr.Add(r) -} - -// Start is the Runnable function triggered upon Manager start-up to perform cache population. -func (r *Manager) Start(ctx context.Context) error { - if err := r.populateCaches(ctx, r.Log); err != nil { - r.Log.Error(err, "cache population failed") - - return nil - } - - r.Log.Info("caches populated") - - return nil } func (r *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res reconcile.Result, err error) { log := r.Log.WithValues("configuration", request.Name) - cfg := configuration.NewCapsuleConfiguration(ctx, r.Client, request.Name) + cfg := configuration.NewCapsuleConfiguration(ctx, r.Client, r.Rest, request.Name) instance := &capsulev1beta2.CapsuleConfiguration{} if err = r.Get(ctx, request.NamespacedName, instance); err != nil { if apierrors.IsNotFound(err) { - log.V(3).Info("requested object not found, could have been deleted after reconcile request") + log.V(5).Info("requested object not found, could have been deleted after reconcile request") return reconcile.Result{}, nil } @@ -156,19 +150,7 @@ func (r *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res log.V(5).Info("gathering capsule users", "users", len(instance.Status.Users)) - interval := cfg.CacheInvalidation() - if cache.ShouldInvalidate(ptr.To(instance.Status.LastCacheInvalidation), time.Now(), interval.Duration) { - log.V(3).Info("invalidating caches") - - if err := r.invalidateCaches(ctx, log); err != nil { - return res, err - } - } - - return reconcile.Result{ - Requeue: true, - RequeueAfter: interval.Duration, - }, err + return reconcile.Result{}, err } func (r *Manager) gatherCapsuleUsers( @@ -190,7 +172,7 @@ func (r *Manager) gatherCapsuleUsers( continue } - users.Upsert(api.UserSpec{ + users.Upsert(rbac.UserSpec{ Kind: to.Spec.Kind, Name: to.Spec.Name, }) @@ -213,6 +195,6 @@ func (r *Manager) updateConfigStatus( latest.Status = instance.Status - return r.Status().Update(ctx, latest) + return r.Client.Status().Update(ctx, latest) }) } diff --git a/internal/controllers/customquotas/calculation.go b/internal/controllers/customquotas/calculation.go new file mode 100644 index 00000000..fe655d90 --- /dev/null +++ b/internal/controllers/customquotas/calculation.go @@ -0,0 +1,291 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquotas + +import ( + "context" + "errors" + "fmt" + "sort" + + "github.com/go-logr/logr" + k8smeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/quota" +) + +type quotaUsageReconcileInput struct { + Log logr.Logger + + Client client.Client + Mapper k8smeta.RESTMapper + + JSONPathCache *cache.JSONPathCache + + Sources []capsulev1beta2.CustomQuotaSpecSource + ScopeSelectors []metav1.LabelSelector + + // For namespaced CustomQuota: + // []string{instance.Namespace} + // + // For GlobalCustomQuota: + // resolved namespaces or []string{"*"} + Namespaces []string + + // Namespaced CustomQuota should only accept namespaced targets. + RequireNamespacedTargets bool + + // Used for compiled target cache. + CacheKey string + TargetsCache *cache.CompiledTargetsCache[string] +} + +type quotaUsageReconcileResult struct { + Targets []capsulev1beta2.CustomQuotaStatusTarget + Usage capsulev1beta2.CustomQuotaStatusUsage + Claims []capsulev1beta2.CustomQuotaClaimItem +} + +type quotaClaimKey struct { + UID types.UID + Group string + Version string + Kind string + Namespace string + Name string +} + +func reconcileQuotaUsage( + ctx context.Context, + in quotaUsageReconcileInput, + limit resource.Quantity, +) (quotaUsageReconcileResult, error) { + out := quotaUsageReconcileResult{ + Targets: []capsulev1beta2.CustomQuotaStatusTarget{}, + Usage: capsulev1beta2.CustomQuotaStatusUsage{}, + Claims: nil, + } + + for _, src := range in.Sources { + kind := src.GroupVersionKind() + + mapping, err := in.Mapper.RESTMapping(kind.GroupKind(), kind.Version) + if err != nil { + return out, fmt.Errorf("failed to resolve REST mapping for %s: %w", kind.String(), err) + } + + if in.RequireNamespacedTargets && mapping.Scope.Name() != k8smeta.RESTScopeNameNamespace { + return out, fmt.Errorf("GVK %s is not namespaced", kind.String()) + } + + out.Targets = append(out.Targets, capsulev1beta2.CustomQuotaStatusTarget{ + GroupVersionKind: metav1.GroupVersionKind(kind), + CustomQuotaSpecSourceConfig: src.CustomQuotaSpecSourceConfig, + Scope: mapping.Scope.Name(), + }) + } + + targets, err := CompileTargets(in.JSONPathCache, out.Targets) + if err != nil { + return out, err + } + + if in.TargetsCache != nil && in.CacheKey != "" { + in.TargetsCache.Set(in.CacheKey, targets) + } + + var errs []error + + itemsByGVK := make(map[schema.GroupVersionKind][]unstructured.Unstructured, len(out.Targets)) + claimsByKey := make(map[quotaClaimKey]capsulev1beta2.CustomQuotaClaimItem) + + for _, target := range targets { + gvk := schema.GroupVersionKind{ + Group: target.Group, + Version: target.Version, + Kind: target.Kind, + } + + items, ok := itemsByGVK[gvk] + if !ok { + items, err = getResourcesByGVK(ctx, gvk, in.Client, in.ScopeSelectors, in.Namespaces...) + if err != nil { + errs = append(errs, fmt.Errorf("list resources for %s: %w", gvk.String(), err)) + + continue + } + + itemsByGVK[gvk] = items + } + + in.Log.V(5).Info("listed resources for target", + "gvk", gvk.String(), + "count", len(items), + "namespaces", in.Namespaces, + "scopeSelectors", in.ScopeSelectors, + ) + + for _, item := range items { + matches, err := MatchesCompiledSelectorsWithFields(item, target.CompiledSelectors) + if err != nil { + errs = append(errs, fmt.Errorf( + "evaluate selectors for %s/%s (%s): %w", + item.GetNamespace(), + item.GetName(), + item.GetObjectKind().GroupVersionKind().String(), + err, + )) + + continue + } + + if !matches { + continue + } + + rawUsage, err := usageForTarget(item, target) + if err != nil { + errs = append(errs, err) + + continue + } + + accountingUsage := rawUsage.DeepCopy() + + switch target.Operation { + case quota.OpSub: + accountingUsage.Neg() + out.Usage.Used.Add(accountingUsage) + quota.ClampQuantityToZero(&out.Usage.Used) + + case quota.OpAdd, quota.OpCount: + out.Usage.Used.Add(accountingUsage) + + default: + errs = append(errs, fmt.Errorf( + "unsupported operation %q for %s/%s (%s)", + target.Operation, + item.GetNamespace(), + item.GetName(), + item.GetObjectKind().GroupVersionKind().String(), + )) + + continue + } + + key := quotaClaimKey{ + UID: item.GetUID(), + Group: target.Group, + Version: target.Version, + Kind: target.Kind, + Namespace: item.GetNamespace(), + Name: item.GetName(), + } + + claim, exists := claimsByKey[key] + if !exists { + claim = capsulev1beta2.CustomQuotaClaimItem{ + GroupVersionKind: metav1.GroupVersionKind{ + Group: target.Group, + Version: target.Version, + Kind: target.Kind, + }, + NamespacedObjectWithUIDReference: meta.NamespacedObjectWithUIDReference{ + Name: item.GetName(), + Namespace: meta.RFC1123SubdomainName(item.GetNamespace()), + UID: item.GetUID(), + }, + Usage: resource.MustParse("0"), + } + } + + // Claims must mirror the same net per-object contribution used by admission reservations. + // This is required so reservationMaterializedLedger(res, claims) can clear reservations. + // + // For pure subtraction sources, the claim remains present but clamps to zero. + // Example: + // claim usage = max(0 - 2Gi, 0) = 0 + // + // For mixed sources, e.g. count + sub cpu: + // claim usage = max(1 - 500m, 0) = 500m + claim.Usage.Add(accountingUsage) + quota.ClampQuantityToZero(&claim.Usage) + + claimsByKey[key] = claim + } + } + + quota.ClampQuantityToZero(&out.Usage.Used) + + out.Usage.Available = limit.DeepCopy() + out.Usage.Available.Sub(out.Usage.Used) + quota.ClampQuantityToZero(&out.Usage.Available) + + out.Claims = make([]capsulev1beta2.CustomQuotaClaimItem, 0, len(claimsByKey)) + for _, claim := range claimsByKey { + out.Claims = append(out.Claims, claim) + } + + sort.SliceStable(out.Claims, func(i, j int) bool { + if out.Claims[i].Namespace != out.Claims[j].Namespace { + return out.Claims[i].Namespace < out.Claims[j].Namespace + } + + if out.Claims[i].Kind != out.Claims[j].Kind { + return out.Claims[i].Kind < out.Claims[j].Kind + } + + return out.Claims[i].Name < out.Claims[j].Name + }) + + if len(errs) > 0 { + return out, errors.Join(errs...) + } + + return out, nil +} + +func usageForTarget( + item unstructured.Unstructured, + target cache.CompiledTarget, +) (resource.Quantity, error) { + switch target.Operation { + case quota.OpCount: + return *resource.NewQuantity(1, resource.DecimalSI), nil + + case quota.OpAdd, quota.OpSub: + usage, err := quota.ParseQuantityFromUnstructured(item, target.CompiledPath) + if err != nil { + return resource.Quantity{}, fmt.Errorf( + "get usage from %s/%s (%s) path %q op %q: %w", + item.GetNamespace(), + item.GetName(), + item.GetObjectKind().GroupVersionKind().String(), + target.Path, + target.Operation, + err, + ) + } + + return usage, nil + + default: + return resource.Quantity{}, fmt.Errorf( + "unsupported operation %q for %s/%s (%s)", + target.Operation, + item.GetNamespace(), + item.GetName(), + item.GetObjectKind().GroupVersionKind().String(), + ) + } +} diff --git a/internal/controllers/customquotas/custom_quota_controller.go b/internal/controllers/customquotas/custom_quota_controller.go new file mode 100644 index 00000000..4363135c --- /dev/null +++ b/internal/controllers/customquotas/custom_quota_controller.go @@ -0,0 +1,321 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquotas + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/go-logr/logr" + apierrors "k8s.io/apimachinery/pkg/api/errors" + k8smeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/cluster-api/util/patch" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" + cutils "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/internal/metrics" + caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/predicates" +) + +type customQuotaClaimController struct { + client.Client + + reader client.Reader + + log logr.Logger + recorder events.EventRecorder + metrics *metrics.CustomQuotaRecorder + mapper k8smeta.RESTMapper + + jsonPathCache *cache.JSONPathCache + targetsCache *cache.CompiledTargetsCache[string] +} + +func (r *customQuotaClaimController) SetupWithManager(mgr ctrl.Manager, cfg cutils.ControllerOptions) error { + r.mapper = mgr.GetRESTMapper() + r.reader = mgr.GetAPIReader() + + return ctrl.NewControllerManagedBy(mgr). + For( + &capsulev1beta2.CustomQuota{}, + builder.WithPredicates( + predicate.Or( + predicate.GenerationChangedPredicate{}, + predicates.ReconcileRequestedPredicate{}, + ), + ), + ). + Watches( + &capsulev1beta2.QuantityLedger{}, + handler.EnqueueRequestForOwner( + mgr.GetScheme(), + mgr.GetRESTMapper(), + &capsulev1beta2.CustomQuota{}, + ), + ). + WithOptions(controller.Options{MaxConcurrentReconciles: cfg.MaxConcurrentReconciles}). + Complete(r) +} + +func (r *customQuotaClaimController) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + log := r.log.WithValues("Request.Name", request.Name, "Request.Namespace", request.Namespace) + + instance := &capsulev1beta2.CustomQuota{} + if err := r.Get(ctx, request.NamespacedName, instance); err != nil { + if apierrors.IsNotFound(err) { + log.V(5).Info("Request object not found, could have been deleted after reconcile request") + r.metrics.DeleteAllMetricsForCustomQuota(request.Name, request.Namespace) + + return reconcile.Result{}, nil + } + + log.Error(err, "Error reading the object") + + return reconcile.Result{}, err + } + + patchHelper, err := patch.NewHelper(instance, r.Client) + if err != nil { + return reconcile.Result{}, err + } + + if err := r.ensureQuotaLedger(ctx, instance); err != nil { + if instance.DeletionTimestamp != nil || shouldIgnoreLedgerEnsureError(err) { + log.V(4).Info("skipping QuantityLedger ensure because CustomQuota or namespace is terminating", + "customQuota", request.String(), + "error", err, + ) + + return reconcile.Result{}, nil + } + + return reconcile.Result{}, err + } + + reconcileErr := r.reconcile(ctx, log, instance) + + requeueAfter, ledgerErr := r.reconcileLedger(ctx, log, instance) + + statusErr := errors.Join(reconcileErr, ledgerErr) + + if err := r.updateStatus(ctx, instance, statusErr); err != nil { + if caperrors.IgnoreGone(err) { + return reconcile.Result{}, nil + } + + return reconcile.Result{}, fmt.Errorf("cannot update status: %w", err) + } + + r.emitMetrics(instance) + + if err := patchHelper.Patch(ctx, instance); err != nil { + if caperrors.IgnoreGone(err) { + return reconcile.Result{}, nil + } + + return reconcile.Result{}, fmt.Errorf("cannot patch: %w", err) + } + + if requeueAfter != nil { + log.V(5).Info("ledger still has pending work, requeueing", + "customQuota", instance.Name, + "namespace", instance.Namespace, + "after", requeueAfter.String(), + ) + + return ctrl.Result{RequeueAfter: *requeueAfter}, nil + } + + return ctrl.Result{}, nil +} + +func (r *customQuotaClaimController) reconcile( + ctx context.Context, + log logr.Logger, + instance *capsulev1beta2.CustomQuota, +) error { + result, err := reconcileQuotaUsage(ctx, quotaUsageReconcileInput{ + Log: log, + + Client: r.Client, + Mapper: r.mapper, + + JSONPathCache: r.jsonPathCache, + + Sources: instance.Spec.Sources, + ScopeSelectors: instance.Spec.ScopeSelectors, + + Namespaces: []string{instance.Namespace}, + + RequireNamespacedTargets: true, + + CacheKey: MakeCustomQuotaCacheKey(instance.GetNamespace(), instance.GetName()), + TargetsCache: r.targetsCache, + }, instance.Spec.Limit) + + instance.Status.Targets = result.Targets + instance.Status.Usage = result.Usage + instance.Status.Claims = result.Claims + + return err +} + +func (r *customQuotaClaimController) reconcileLedger( + ctx context.Context, + log logr.Logger, + instance *capsulev1beta2.CustomQuota, +) (*time.Duration, error) { + key := types.NamespacedName{ + Name: instance.GetName(), + Namespace: instance.GetNamespace(), + } + + return reconcileQuantityLedgerAllocation( + ctx, + r.Client, + log, + key, + instance.Status.Usage.Used.DeepCopy(), + instance.Status.Claims, + ) +} + +func (r *customQuotaClaimController) ensureQuotaLedger( + ctx context.Context, + instance *capsulev1beta2.CustomQuota, +) error { + ledger := &capsulev1beta2.QuantityLedger{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.GetName(), + Namespace: instance.GetNamespace(), + }, + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, ledger, func() error { + if ledger.Labels == nil { + ledger.Labels = map[string]string{} + } + + ledger.Labels[meta.ManagedByCapsuleLabel] = meta.ValueController + + ledger.Spec.TargetRef = capsulev1beta2.QuantityLedgerTargetRef{ + APIGroup: capsulev1beta2.GroupVersion.Group, + Kind: "CustomQuota", + Namespace: instance.GetNamespace(), + Name: instance.GetName(), + UID: instance.GetUID(), + } + + return controllerutil.SetControllerReference(instance, ledger, r.Scheme()) + }) + if err != nil { + return fmt.Errorf("create or update QuantityLedger %s/%s for CustomQuota %s/%s: %w", + ledger.Namespace, ledger.Name, instance.Namespace, instance.Name, err) + } + + return nil +} + +func (r *customQuotaClaimController) emitMetrics( + instance *capsulev1beta2.CustomQuota, +) { + // Condition Metrics + for _, status := range []string{meta.ReadyCondition} { + var value float64 + + cond := instance.Status.Conditions.GetConditionByType(status) + if cond == nil { + r.metrics.DeleteConditionMetricByType(instance.GetName(), instance.GetNamespace(), status) + + continue + } + + if cond.Status == metav1.ConditionTrue { + value = 1 + } + + r.metrics.ConditionGauge.WithLabelValues(instance.GetName(), instance.GetNamespace(), status).Set(value) + } + + // Usage Metrics + r.metrics.ResourceUsageGauge.WithLabelValues(instance.GetName(), instance.GetNamespace()).Set(float64(instance.Status.Usage.Used.MilliValue()) / 1000) + r.metrics.ResourceAvailableGauge.WithLabelValues(instance.GetName(), instance.GetNamespace()).Set(float64(instance.Status.Usage.Available.MilliValue()) / 1000) + r.metrics.ResourceLimitGauge.WithLabelValues(instance.GetName(), instance.GetNamespace()).Set(float64(instance.Spec.Limit.MilliValue()) / 1000) + + // Emit for Claims + r.metrics.ResourceItemUsageGauge.DeletePartialMatch(map[string]string{ + "custom_quota": instance.GetName(), + "target_namespace": instance.GetNamespace(), + }) + + // Skip emitting metrics on claim basis + if !instance.Spec.Options.EmitPerClaimMetrics { + return + } + + for _, claim := range instance.Status.Claims { + r.metrics.ResourceItemUsageGauge.WithLabelValues( + instance.GetName(), + instance.GetNamespace(), + claim.Name, + claim.Kind, + claim.Group, + ).Set(float64(claim.Usage.MilliValue()) / 1000) + } +} + +func (r *customQuotaClaimController) updateStatus( + ctx context.Context, + instance *capsulev1beta2.CustomQuota, + reconcileError error, +) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + latest := &capsulev1beta2.CustomQuota{} + if err = r.reader.Get(ctx, types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}, latest); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + latest.Status = instance.Status + + // Set Ready Condition + readyCondition := meta.NewReadyCondition(instance) + if reconcileError != nil { + readyCondition.Message = reconcileError.Error() + readyCondition.Status = metav1.ConditionFalse + readyCondition.Reason = meta.FailedReason + } + + latest.Status.Conditions.UpdateConditionByType(readyCondition) + + if err := r.Client.Status().Update(ctx, latest); err != nil { + return err + } + + // Keep the in-memory object aligned with what we just wrote. + instance.Status = latest.Status + + return nil + }) +} diff --git a/internal/controllers/customquotas/global_custom_quota_controller.go b/internal/controllers/customquotas/global_custom_quota_controller.go new file mode 100644 index 00000000..d3bf46b5 --- /dev/null +++ b/internal/controllers/customquotas/global_custom_quota_controller.go @@ -0,0 +1,413 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquotas + +import ( + "context" + "errors" + "fmt" + "reflect" + "slices" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + k8smeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/cluster-api/util/patch" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" + cutils "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/internal/metrics" + caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/predicates" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +type clusterCustomQuotaClaimController struct { + client.Client + + reader client.Reader + + log logr.Logger + recorder events.EventRecorder + metrics *metrics.GlobalCustomQuotaRecorder + mapper k8smeta.RESTMapper + + jsonPathCache *cache.JSONPathCache + targetsCache *cache.CompiledTargetsCache[string] +} + +func (r *clusterCustomQuotaClaimController) SetupWithManager(mgr ctrl.Manager, cfg cutils.ControllerOptions) error { + r.mapper = mgr.GetRESTMapper() + r.reader = mgr.GetAPIReader() + + return ctrl.NewControllerManagedBy(mgr). + For( + &capsulev1beta2.GlobalCustomQuota{}, + builder.WithPredicates( + predicate.Or( + predicate.GenerationChangedPredicate{}, + predicates.ReconcileRequestedPredicate{}, + ), + ), + ). + Watches( + &capsulev1beta2.QuantityLedger{}, + handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &capsulev1beta2.GlobalCustomQuota{}), + ). + Watches( + &corev1.Namespace{}, + handler.EnqueueRequestsFromMapFunc(r.mapNamespaceToGlobalCustomQuotas), + builder.WithPredicates(predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return true + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return true + }, + UpdateFunc: func(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + + oldNs, okOld := e.ObjectOld.(*corev1.Namespace) + + newNs, okNew := e.ObjectNew.(*corev1.Namespace) + + if !okOld || !okNew { + return false + } + + return !reflect.DeepEqual(oldNs.Labels, newNs.Labels) || + !reflect.DeepEqual(oldNs.Annotations, newNs.Annotations) + }, + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + }), + ). + WithOptions(controller.Options{MaxConcurrentReconciles: cfg.MaxConcurrentReconciles}). + Complete(r) +} + +func (r *clusterCustomQuotaClaimController) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + log := r.log.WithValues("Request.Name", request.Name) + + instance := &capsulev1beta2.GlobalCustomQuota{} + if err := r.Get(ctx, request.NamespacedName, instance); err != nil { + if apierrors.IsNotFound(err) { + log.V(5).Info("Request object not found, could have been deleted after reconcile request") + r.metrics.DeleteAllMetricsForGlobalCustomQuota(request.Name) + + return reconcile.Result{}, nil + } + + return reconcile.Result{}, err + } + + patchHelper, err := patch.NewHelper(instance, r.Client) + if err != nil { + return reconcile.Result{}, err + } + + if err := r.ensureQuotaLedger(ctx, instance); err != nil { + if instance.DeletionTimestamp != nil || shouldIgnoreLedgerEnsureError(err) { + log.V(4).Info("skipping QuantityLedger ensure because CustomQuota or namespace is terminating", + "customQuota", request.String(), + "error", err, + ) + + return reconcile.Result{}, nil + } + + return reconcile.Result{}, err + } + + reconcileErr := r.reconcile(ctx, log, instance) + + requeueAfter, ledgerErr := r.reconcileLedger(ctx, log, instance) + + statusErr := errors.Join(reconcileErr, ledgerErr) + + if err := r.updateStatus(ctx, instance, statusErr); err != nil { + if caperrors.IgnoreGone(err) { + return reconcile.Result{}, nil + } + + return reconcile.Result{}, fmt.Errorf("cannot update status: %w", err) + } + + r.emitMetrics(instance) + + if err := patchHelper.Patch(ctx, instance); err != nil { + if caperrors.IgnoreGone(err) { + return reconcile.Result{}, nil + } + + return reconcile.Result{}, fmt.Errorf("failed to patch: %w", err) + } + + if requeueAfter != nil { + log.V(5).Info("ledger still has pending work, requeueing", + "customQuota", instance.Name, + "namespace", instance.Namespace, + "after", requeueAfter.String(), + ) + + return ctrl.Result{RequeueAfter: *requeueAfter}, nil + } + + return ctrl.Result{}, nil +} + +func (r *clusterCustomQuotaClaimController) mapNamespaceToGlobalCustomQuotas( + ctx context.Context, + obj client.Object, +) []reconcile.Request { + ns, ok := obj.(*corev1.Namespace) + if !ok { + return nil + } + + var quotaList capsulev1beta2.GlobalCustomQuotaList + if err := r.List(ctx, "aList); err != nil { + r.log.Error(err, "cannot list GlobalCustomQuota objects for namespace event", "namespace", ns.Name) + + return nil + } + + requests := make([]reconcile.Request, 0, len(quotaList.Items)) + + for i := range quotaList.Items { + gcq := "aList.Items[i] + + if shouldReconcileForNamespaceEvent(gcq, ns.Name) { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: gcq.Name, + }, + }) + } + } + + return requests +} + +func shouldReconcileForNamespaceEvent( + instance *capsulev1beta2.GlobalCustomQuota, + namespace string, +) bool { + if len(instance.Spec.NamespaceSelectors) > 0 { + return true + } + + return slices.Contains(instance.Status.Namespaces, namespace) +} + +func (r *clusterCustomQuotaClaimController) reconcile( + ctx context.Context, + log logr.Logger, + instance *capsulev1beta2.GlobalCustomQuota, +) error { + var namespaces []string + + var err error + + if len(instance.Spec.NamespaceSelectors) > 0 { + namespaces, err = selectors.GetNamespacesMatchingSelectorsStrings( + ctx, + r.Client, + instance.Spec.NamespaceSelectors, + ) + if err != nil { + return err + } + } else { + namespaces = []string{"*"} + } + + instance.Status.Namespaces = namespaces + + result, err := reconcileQuotaUsage(ctx, quotaUsageReconcileInput{ + Log: log, + + Client: r.Client, + Mapper: r.mapper, + + JSONPathCache: r.jsonPathCache, + + Sources: instance.Spec.Sources, + ScopeSelectors: instance.Spec.ScopeSelectors, + + Namespaces: namespaces, + + RequireNamespacedTargets: false, + + CacheKey: MakeGlobalCustomQuotaCacheKey(instance.GetName()), + TargetsCache: r.targetsCache, + }, instance.Spec.Limit) + + instance.Status.Targets = result.Targets + instance.Status.Usage = result.Usage + instance.Status.Claims = result.Claims + + return err +} + +func (r *clusterCustomQuotaClaimController) reconcileLedger( + ctx context.Context, + log logr.Logger, + instance *capsulev1beta2.GlobalCustomQuota, +) (*time.Duration, error) { + key := types.NamespacedName{ + Name: instance.GetName(), + Namespace: configuration.ControllerNamespace(), + } + + return reconcileQuantityLedgerAllocation( + ctx, + r.Client, + log, + key, + instance.Status.Usage.Used.DeepCopy(), + instance.Status.Claims, + ) +} + +func (r *clusterCustomQuotaClaimController) ensureQuotaLedger( + ctx context.Context, + instance *capsulev1beta2.GlobalCustomQuota, +) error { + ledger := &capsulev1beta2.QuantityLedger{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.GetName(), + Namespace: configuration.ControllerNamespace(), + }, + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, ledger, func() error { + if ledger.Labels == nil { + ledger.Labels = map[string]string{} + } + + ledger.Labels[meta.ManagedByCapsuleLabel] = meta.ValueController + + ledger.Spec.TargetRef = capsulev1beta2.QuantityLedgerTargetRef{ + APIGroup: capsulev1beta2.GroupVersion.Group, + Kind: "GlobalCustomQuota", + Name: instance.GetName(), + UID: instance.GetUID(), + } + + return controllerutil.SetControllerReference(instance, ledger, r.Scheme()) + }) + if err != nil { + return fmt.Errorf("create or update QuantityLedger %s/%s for GlobalCustomQuota %s: %w", + ledger.Namespace, ledger.Name, instance.GetName(), err) + } + + return nil +} + +func (r *clusterCustomQuotaClaimController) emitMetrics( + instance *capsulev1beta2.GlobalCustomQuota, +) { + // Condition Metrics + for _, status := range []string{meta.ReadyCondition} { + var value float64 + + cond := instance.Status.Conditions.GetConditionByType(status) + if cond == nil { + r.metrics.DeleteConditionMetricByType(instance.GetName(), status) + + continue + } + + if cond.Status == metav1.ConditionTrue { + value = 1 + } + + r.metrics.ConditionGauge.WithLabelValues(instance.GetName(), status).Set(value) + } + + // Usage Metrics + r.metrics.ResourceUsageGauge.WithLabelValues(instance.GetName()).Set(float64(instance.Status.Usage.Used.MilliValue()) / 1000) + r.metrics.ResourceAvailableGauge.WithLabelValues(instance.GetName()).Set(float64(instance.Status.Usage.Available.MilliValue()) / 1000) + r.metrics.ResourceLimitGauge.WithLabelValues(instance.GetName()).Set(float64(instance.Spec.Limit.MilliValue()) / 1000) + + // Emit for Claims + r.metrics.ResourceItemUsageGauge.DeletePartialMatch(map[string]string{ + "custom_quota": instance.GetName(), + }) + + // Skip emitting metrics on claim basis + if !instance.Spec.Options.EmitPerClaimMetrics { + return + } + + for _, claim := range instance.Status.Claims { + r.metrics.ResourceItemUsageGauge.WithLabelValues( + instance.GetName(), + claim.Name, + string(claim.Namespace), + claim.Kind, + claim.Group, + ).Set(float64(claim.Usage.MilliValue()) / 1000) + } +} + +func (r *clusterCustomQuotaClaimController) updateStatus( + ctx context.Context, + instance *capsulev1beta2.GlobalCustomQuota, + reconcileError error, +) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + latest := &capsulev1beta2.GlobalCustomQuota{} + if err = r.reader.Get(ctx, types.NamespacedName{Name: instance.GetName()}, latest); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + latest.Status = instance.Status + + // Set Ready Condition + readyCondition := meta.NewReadyCondition(instance) + if reconcileError != nil { + readyCondition.Message = reconcileError.Error() + readyCondition.Status = metav1.ConditionFalse + readyCondition.Reason = meta.FailedReason + } + + latest.Status.Conditions.UpdateConditionByType(readyCondition) + + if err := r.Client.Status().Update(ctx, latest); err != nil { + return err + } + + // Keep the in-memory object aligned with what we just wrote. + instance.Status = latest.Status + + return nil + }) +} diff --git a/internal/controllers/customquotas/manager.go b/internal/controllers/customquotas/manager.go new file mode 100644 index 00000000..f1c2dd6a --- /dev/null +++ b/internal/controllers/customquotas/manager.go @@ -0,0 +1,54 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquotas + +import ( + "fmt" + + "github.com/go-logr/logr" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" + "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/internal/metrics" +) + +func Add( + log logr.Logger, + mgr manager.Manager, + recorder events.EventRecorder, + cfg utils.ControllerOptions, + quantityCache *cache.QuantityCache[string], + jsonPathCache *cache.JSONPathCache, + targetsCache *cache.CompiledTargetsCache[string], + namespaceNotifier chan event.TypedGenericEvent[*capsulev1beta2.CustomQuota], + globalNotifier chan event.TypedGenericEvent[*capsulev1beta2.GlobalCustomQuota], +) (err error) { + if err = (&customQuotaClaimController{ + Client: mgr.GetClient(), + log: log.WithName("CustomQuota"), + recorder: recorder, + metrics: metrics.MustMakeCustomQuotaRecorder(), + jsonPathCache: jsonPathCache, + targetsCache: targetsCache, + }).SetupWithManager(mgr, cfg); err != nil { + return fmt.Errorf("unable to create custom quota controller: %w", err) + } + + if err = (&clusterCustomQuotaClaimController{ + Client: mgr.GetClient(), + log: log.WithName("ClusterCustomQuota"), + recorder: recorder, + metrics: metrics.MustMakeGlobalCustomQuotaRecorder(), + jsonPathCache: jsonPathCache, + targetsCache: targetsCache, + }).SetupWithManager(mgr, cfg); err != nil { + return fmt.Errorf("unable to create cluster custom quota controller: %w", err) + } + + return nil +} diff --git a/internal/controllers/customquotas/utils.go b/internal/controllers/customquotas/utils.go new file mode 100644 index 00000000..3a985846 --- /dev/null +++ b/internal/controllers/customquotas/utils.go @@ -0,0 +1,514 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquotas + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" + "github.com/projectcapsule/capsule/pkg/runtime/jsonpath" + "github.com/projectcapsule/capsule/pkg/runtime/quota" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +const immediatePendingDeleteRequeue = 500 * time.Millisecond + +type GroupedTarget struct { + GVK schema.GroupVersionKind + Targets []capsulev1beta2.CustomQuotaStatusTarget +} + +type CompiledTarget struct { + capsulev1beta2.CustomQuotaStatusTarget + + CompiledPath *jsonpath.CompiledJSONPath + CompiledSelectors []selectors.CompiledSelectorWithFields +} + +func CompileTargets( + jcache *cache.JSONPathCache, + targets []capsulev1beta2.CustomQuotaStatusTarget, +) ([]cache.CompiledTarget, error) { + out := make([]cache.CompiledTarget, 0, len(targets)) + + for _, target := range targets { + pt := cache.CompiledTarget{ + CustomQuotaStatusTarget: target, + } + + switch target.Operation { + case quota.OpCount: + // no usage path needed + + case quota.OpAdd, quota.OpSub: + compiledPath, err := jcache.GetOrCompile(target.Path) + if err != nil { + return nil, fmt.Errorf( + "compile usage path %q for %s %q: %w", + target.Path, + target.String(), + target.Operation, + err, + ) + } + + pt.CompiledPath = compiledPath + + default: + return nil, fmt.Errorf("unsupported operation %q for %s", target.Operation, target.String()) + } + + compiledSelectors, err := CompileSelectorsWithFields(jcache, target.Selectors) + if err != nil { + return nil, fmt.Errorf( + "compile selectors for %s: %w", + target.String(), + err, + ) + } + + pt.CompiledSelectors = compiledSelectors + + out = append(out, pt) + } + + return out, nil +} + +func MatchesCompiledSelectorsWithFields( + u unstructured.Unstructured, + selectors []selectors.CompiledSelectorWithFields, +) (bool, error) { + if len(selectors) == 0 { + return true, nil + } + + itemLabels := labels.Set(u.GetLabels()) + + for _, sel := range selectors { + if !sel.LabelSelector.Matches(itemLabels) { + continue + } + + allFieldsMatch := true + + for _, matcher := range sel.FieldMatchers { + ok, err := jsonpath.EvaluateTruthyFromCompiled(u, matcher) + if err != nil { + return false, err + } + + if !ok { + allFieldsMatch = false + + break + } + } + + if allFieldsMatch { + return true, nil + } + } + + return false, nil +} + +func MakeCustomQuotaCacheKey(namespace, name string) string { + return namespace + "/" + name +} + +func MakeGlobalCustomQuotaCacheKey(name string) string { + return "C/" + name +} + +func CompileSelectorsWithFields( + cache *cache.JSONPathCache, + in []selectors.SelectorWithFields, +) ([]selectors.CompiledSelectorWithFields, error) { + if len(in) == 0 { + return nil, nil + } + + out := make([]selectors.CompiledSelectorWithFields, 0, len(in)) + + for _, selector := range in { + lblSel := labels.Everything() + + if selector.LabelSelector != nil { + compiled, err := metav1.LabelSelectorAsSelector(selector.LabelSelector) + if err != nil { + return nil, fmt.Errorf("compile label selector with fields: %w", err) + } + + lblSel = compiled + } + + fieldMatchers := make([]*jsonpath.CompiledJSONPath, 0, len(selector.FieldSelectors)) + + for _, path := range selector.FieldSelectors { + compiledPath, err := cache.GetOrCompile(path) + if err != nil { + return nil, fmt.Errorf("compile field selector path %q: %w", path, err) + } + + fieldMatchers = append(fieldMatchers, compiledPath) + } + + out = append(out, selectors.CompiledSelectorWithFields{ + LabelSelector: lblSel, + FieldMatchers: fieldMatchers, + }) + } + + return out, nil +} + +func shouldIgnoreLedgerEnsureError(err error) bool { + if err == nil { + return false + } + + if apierrors.IsNotFound(err) { + return true + } + + if apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) { + return true + } + + var statusErr *apierrors.StatusError + if errors.As(err, &statusErr) { + if statusErr.ErrStatus.Reason == metav1.StatusReasonForbidden && + strings.Contains(statusErr.ErrStatus.Message, "because it is being terminated") { + return true + } + } + + return false +} + +func getResourcesByGVK( + ctx context.Context, + gvk schema.GroupVersionKind, + kubeClient client.Reader, + scopeSelectors []metav1.LabelSelector, + namespaces ...string, +) ([]unstructured.Unstructured, error) { + compiledSelectors := make([]labels.Selector, 0, len(scopeSelectors)) + + for _, selector := range scopeSelectors { + sel, err := metav1.LabelSelectorAsSelector(&selector) + if err != nil { + return nil, err + } + + compiledSelectors = append(compiledSelectors, sel) + } + + filterByNamespace := true + namespaceSet := make(map[string]struct{}, len(namespaces)) + + for _, ns := range namespaces { + if ns == "*" { + filterByNamespace = false + namespaceSet = nil + + break + } + + namespaceSet[ns] = struct{}{} + } + + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(schema.GroupVersionKind{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind + "List", + }) + + if err := kubeClient.List(ctx, list); err != nil { + return nil, err + } + + items := make([]unstructured.Unstructured, 0, len(list.Items)) + seen := make(map[string]struct{}, len(list.Items)) + + for i := range list.Items { + item := list.Items[i] + + // Skip objects that are already definitely deleting: + // deletionTimestamp is set and there are no finalizers left. + if item.GetDeletionTimestamp() != nil && len(item.GetFinalizers()) == 0 { + continue + } + + // Namespace filter + if filterByNamespace { + if _, ok := namespaceSet[item.GetNamespace()]; !ok { + continue + } + } + + // Label selector filter (OR semantics) + if len(compiledSelectors) > 0 { + itemLabels := labels.Set(item.GetLabels()) + + matched := false + + for _, sel := range compiledSelectors { + if sel.Matches(itemLabels) { + matched = true + + break + } + } + + if !matched { + continue + } + } + + key := item.GetNamespace() + "/" + item.GetName() + if item.GetNamespace() == "" { + key = item.GetName() + } + + if _, exists := seen[key]; exists { + continue + } + + seen[key] = struct{}{} + + items = append(items, item) + } + + // Sort by oldest first + sort.Slice(items, func(i, j int) bool { + return items[i].GetCreationTimestamp().Time.Before(items[j].GetCreationTimestamp().Time) + }) + + return items, nil +} + +func minDurationPtr(cur *time.Duration, cand time.Duration) *time.Duration { + if cand < 0 { + cand = 0 + } + + if cur == nil || cand < *cur { + return &cand + } + + return cur +} + +func pendingDeleteStillPresent( + pd capsulev1beta2.QuantityLedgerPendingDelete, + claims []capsulev1beta2.CustomQuotaClaimItem, +) bool { + for _, claim := range claims { + if pd.ObjectRef.UID != "" && claim.UID != "" && pd.ObjectRef.UID == claim.UID { + return true + } + + if pd.ObjectRef.APIGroup == claim.Group && + pd.ObjectRef.APIVersion == claim.Version && + pd.ObjectRef.Kind == claim.Kind && + pd.ObjectRef.Namespace == string(claim.Namespace) && + pd.ObjectRef.Name == claim.Name { + return true + } + } + + return false +} + +const unresolvedReservationRequeue = 250 * time.Millisecond + +func nextReservationMaterializationRequeue( + now metav1.Time, + res capsulev1beta2.QuantityLedgerReservation, +) time.Duration { + if res.ExpiresAt == nil { + return unresolvedReservationRequeue + } + + untilExpiry := time.Until(res.ExpiresAt.Time) + if untilExpiry <= 0 { + return 0 + } + + if untilExpiry < unresolvedReservationRequeue { + return untilExpiry + } + + return unresolvedReservationRequeue +} + +func reconcileQuantityLedgerAllocation( + ctx context.Context, + c client.Client, + log logr.Logger, + key types.NamespacedName, + observedUsed resource.Quantity, + claims []capsulev1beta2.CustomQuotaClaimItem, +) (*time.Duration, error) { + var requeueAfter *time.Duration + + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + ledger := &capsulev1beta2.QuantityLedger{} + if err := c.Get(ctx, key, ledger); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + now := metav1.Now() + pendingDeleteTTL := 30 * time.Second + + activeReservations := make([]capsulev1beta2.QuantityLedgerReservation, 0, len(ledger.Status.Reservations)) + + for _, res := range ledger.Status.Reservations { + materialized := reservationMaterializedLedger(res, claims) + expired := res.ExpiresAt != nil && res.ExpiresAt.Before(&now) + + log.V(5).Info("evaluating ledger reservation", + "ledger", key.String(), + "reservationID", res.ID, + "usage", res.Usage.String(), + "uid", string(res.ObjectRef.UID), + "group", res.ObjectRef.APIGroup, + "version", res.ObjectRef.APIVersion, + "kind", res.ObjectRef.Kind, + "namespace", res.ObjectRef.Namespace, + "name", res.ObjectRef.Name, + "materialized", materialized, + "expired", expired, + ) + + switch { + case materialized: + continue + + case expired: + continue + + default: + activeReservations = append(activeReservations, res) + + requeueAfter = minDurationPtr( + requeueAfter, + nextReservationMaterializationRequeue(now, res), + ) + } + } + + activeDeletes := make([]capsulev1beta2.QuantityLedgerPendingDelete, 0, len(ledger.Status.PendingDeletes)) + + for _, pd := range ledger.Status.PendingDeletes { + stillPresent := pendingDeleteStillPresent(pd, claims) + expired := now.Sub(pd.CreatedAt.Time) >= pendingDeleteTTL + + log.V(5).Info("evaluating pending delete", + "ledger", key.String(), + "uid", string(pd.ObjectRef.UID), + "group", pd.ObjectRef.APIGroup, + "version", pd.ObjectRef.APIVersion, + "kind", pd.ObjectRef.Kind, + "namespace", pd.ObjectRef.Namespace, + "name", pd.ObjectRef.Name, + "stillPresent", stillPresent, + "expired", expired, + ) + + if stillPresent { + activeDeletes = append(activeDeletes, pd) + requeueAfter = minDurationPtr(requeueAfter, immediatePendingDeleteRequeue) + } + } + + reserved := resource.MustParse("0") + for _, res := range activeReservations { + reserved.Add(res.Usage) + } + + allocated := observedUsed.DeepCopy() + allocated.Add(reserved) + quota.ClampQuantityToZero(&allocated) + + ledger.Status.Reservations = activeReservations + ledger.Status.PendingDeletes = activeDeletes + ledger.Status.Reserved = reserved + ledger.Status.Allocated = allocated + + return c.Status().Update(ctx, ledger) + }) + if err != nil { + return nil, err + } + + return requeueAfter, nil +} + +func reservationMaterializedLedger( + res capsulev1beta2.QuantityLedgerReservation, + claims []capsulev1beta2.CustomQuotaClaimItem, +) bool { + for _, claim := range claims { + if !sameLedgerObject(res.ObjectRef, claim) { + continue + } + + // Important for updates: + // UID/name match alone is not enough. The controller must have observed + // the same usage that the webhook reserved. + if claim.Usage.Cmp(res.Usage) != 0 { + continue + } + + return true + } + + return false +} + +func sameLedgerObject( + ref capsulev1beta2.QuantityLedgerObjectRef, + claim capsulev1beta2.CustomQuotaClaimItem, +) bool { + if ref.APIGroup != claim.Group || + ref.APIVersion != claim.Version || + ref.Kind != claim.Kind || + ref.Namespace != string(claim.Namespace) || + ref.Name != claim.Name { + return false + } + + // CREATE admissions often do not have a UID yet. + if ref.UID != "" && claim.UID != "" { + return ref.UID == claim.UID + } + + return true +} diff --git a/internal/controllers/pv/controller.go b/internal/controllers/pv/controller.go index 4ef5d437..25e1fb45 100644 --- a/internal/controllers/pv/controller.go +++ b/internal/controllers/pv/controller.go @@ -64,7 +64,7 @@ func (c *Controller) Reconcile(ctx context.Context, request reconcile.Request) ( persistentVolume := corev1.PersistentVolume{} if err := c.client.Get(ctx, request.NamespacedName, &persistentVolume); err != nil { if errors.IsNotFound(err) { - log.V(3).Info("skipping reconciliation, resource may have been deleted") + log.V(5).Info("skipping reconciliation, resource may have been deleted") return reconcile.Result{}, nil } diff --git a/internal/controllers/rbac/manager.go b/internal/controllers/rbac/manager.go index 5e9db525..a28a62ba 100644 --- a/internal/controllers/rbac/manager.go +++ b/internal/controllers/rbac/manager.go @@ -24,15 +24,13 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/controllers/utils" - "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/predicates" ) -const ( - controllerManager = "rbac-controller" -) +const controllerManager = "rbac-controller" type Manager struct { Log logr.Logger @@ -109,17 +107,17 @@ func (r *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res } func (r *Manager) EnsureClusterRoleBindingsProvisioner(ctx context.Context) error { - rbac := r.Configuration.RBAC() + cfg := r.Configuration.RBAC() crb := &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{Name: rbac.ProvisionerClusterRole}, + ObjectMeta: metav1.ObjectMeta{Name: cfg.ProvisionerClusterRole}, } return retry.RetryOnConflict(retry.DefaultRetry, func() error { _, err := controllerutil.CreateOrUpdate(ctx, r.Client, crb, func() error { crb.RoleRef = rbacv1.RoleRef{ Kind: "ClusterRole", - Name: rbac.ProvisionerClusterRole, + Name: cfg.ProvisionerClusterRole, APIGroup: rbacv1.GroupName, } @@ -141,17 +139,17 @@ func (r *Manager) EnsureClusterRoleBindingsProvisioner(ctx context.Context) erro for _, entity := range users { switch entity.Kind { - case api.UserOwner: + case rbac.UserOwner: crb.Subjects = append(crb.Subjects, rbacv1.Subject{ Kind: rbacv1.UserKind, Name: entity.Name, }) - case api.GroupOwner: + case rbac.GroupOwner: crb.Subjects = append(crb.Subjects, rbacv1.Subject{ Kind: rbacv1.GroupKind, Name: entity.Name, }) - case api.ServiceAccountOwner: + case rbac.ServiceAccountOwner: namespace, name, err := serviceaccount.SplitUsername(entity.Name) if err != nil { return err diff --git a/internal/controllers/resourcepools/claim_controller.go b/internal/controllers/resourcepools/claim_controller.go index 004d296c..e7134b9e 100644 --- a/internal/controllers/resourcepools/claim_controller.go +++ b/internal/controllers/resourcepools/claim_controller.go @@ -26,18 +26,23 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/controllers/utils" "github.com/projectcapsule/capsule/internal/metrics" + caperrors "github.com/projectcapsule/capsule/pkg/api/errors" "github.com/projectcapsule/capsule/pkg/api/meta" ) type resourceClaimController struct { client.Client + reader client.Reader + metrics *metrics.ClaimRecorder log logr.Logger recorder events.EventRecorder } func (r *resourceClaimController) SetupWithManager(mgr ctrl.Manager, cfg utils.ControllerOptions) error { + r.reader = mgr.GetAPIReader() + return ctrl.NewControllerManagedBy(mgr). Named("capsule/resourcepools/claims"). For( @@ -59,9 +64,9 @@ func (r resourceClaimController) Reconcile(ctx context.Context, request ctrl.Req log := r.log.WithValues("Request.Name", request.Name) instance := &capsulev1beta2.ResourcePoolClaim{} - if err = r.Get(ctx, request.NamespacedName, instance); err != nil { + if err = r.reader.Get(ctx, request.NamespacedName, instance); err != nil { if apierrors.IsNotFound(err) { - log.V(3).Info("Request object not found, could have been deleted after reconcile request") + log.V(5).Info("Request object not found, could have been deleted after reconcile request") r.metrics.DeleteClaimMetric(request.Name, request.Namespace) @@ -88,7 +93,13 @@ func (r resourceClaimController) Reconcile(ctx context.Context, request ctrl.Req r.metrics.RecordClaimCondition(instance) if e := patchHelper.Patch(ctx, instance); e != nil { - err = e + if caperrors.IgnoreGone(e) { + err = nil + + return + } + + err = gherrors.Wrap(e, "failed to patch") return } @@ -113,7 +124,7 @@ func (r *resourceClaimController) claimsWithoutPoolFromNamespaces(ctx context.Co for _, ns := range pool.Status.Namespaces { claimList := &capsulev1beta2.ResourcePoolClaimList{} - if err := r.List(ctx, claimList, client.InNamespace(ns)); err != nil { + if err := r.reader.List(ctx, claimList, client.InNamespace(ns)); err != nil { r.log.Error(err, "Failed to list claims in namespace", "namespace", ns) continue @@ -261,14 +272,17 @@ func (r *resourceClaimController) updateStatus( ) error { return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { latest := &capsulev1beta2.ResourcePoolClaim{} - if err = r.Get(ctx, types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}, latest); err != nil { + if err = r.reader.Get(ctx, types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}, latest); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err } latest.Status = instance.Status - // Set Ready Condition - readyCondition := meta.NewReadyCondition(instance) + readyCondition := meta.NewReadyCondition(latest) if reconcileError != nil { readyCondition.Message = reconcileError.Error() readyCondition.Status = metav1.ConditionFalse @@ -281,6 +295,13 @@ func (r *resourceClaimController) updateStatus( //nolint:staticcheck latest.Status.Condition = metav1.Condition{} - return r.Client.Status().Update(ctx, latest) + if err := r.Client.Status().Update(ctx, latest); err != nil { + return err + } + + // Keep the in-memory object aligned with what we just wrote. + instance.Status = latest.Status + + return nil }) } diff --git a/internal/controllers/resourcepools/pool_controller.go b/internal/controllers/resourcepools/pool_controller.go index a6fbf5ca..1d52d04d 100644 --- a/internal/controllers/resourcepools/pool_controller.go +++ b/internal/controllers/resourcepools/pool_controller.go @@ -31,6 +31,7 @@ import ( ctrlutils "github.com/projectcapsule/capsule/internal/controllers/utils" "github.com/projectcapsule/capsule/internal/metrics" "github.com/projectcapsule/capsule/pkg/api" + caperrors "github.com/projectcapsule/capsule/pkg/api/errors" "github.com/projectcapsule/capsule/pkg/api/meta" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/utils" @@ -39,12 +40,16 @@ import ( type resourcePoolController struct { client.Client + reader client.Reader + metrics *metrics.ResourcePoolRecorder log logr.Logger recorder events.EventRecorder } func (r *resourcePoolController) SetupWithManager(mgr ctrl.Manager, cfg ctrlutils.ControllerOptions) error { + r.reader = mgr.GetAPIReader() + return ctrl.NewControllerManagedBy(mgr). Named("capsule/resourcepools/pools"). For(&capsulev1beta2.ResourcePool{}). @@ -56,7 +61,7 @@ func (r *resourcePoolController) SetupWithManager(mgr ctrl.Manager, cfg ctrlutil handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, _ client.Object) []reconcile.Request { // Fetch all GlobalResourceQuota objects grqList := &capsulev1beta2.ResourcePoolList{} - if err := mgr.GetClient().List(ctx, grqList); err != nil { + if err := r.reader.List(ctx, grqList); err != nil { r.log.Error(err, "Failed to list ResourcePools objects") return nil @@ -80,11 +85,10 @@ func (r *resourcePoolController) SetupWithManager(mgr ctrl.Manager, cfg ctrlutil func (r resourcePoolController) Reconcile(ctx context.Context, request ctrl.Request) (result ctrl.Result, err error) { log := r.log.WithValues("Request.Name", request.Name) - // Fetch the Tenant instance instance := &capsulev1beta2.ResourcePool{} if err = r.Get(ctx, request.NamespacedName, instance); err != nil { if apierrors.IsNotFound(err) { - log.V(3).Info("Request object not found, could have been deleted after reconcile request") + log.V(5).Info("Request object not found, could have been deleted after reconcile request") r.metrics.DeleteResourcePoolMetric(request.Name) @@ -105,6 +109,12 @@ func (r resourcePoolController) Reconcile(ctx context.Context, request ctrl.Requ r.finalize(ctx, instance) if uerr := r.updateStatus(ctx, instance, err); uerr != nil { + if caperrors.IgnoreGone(uerr) { + err = nil + + return + } + err = fmt.Errorf("cannot update pool status: %w", uerr) return @@ -113,13 +123,20 @@ func (r resourcePoolController) Reconcile(ctx context.Context, request ctrl.Requ r.metrics.ResourceUsageMetrics(instance) if e := patchHelper.Patch(ctx, instance); e != nil { - err = e + if caperrors.IgnoreGone(e) { + err = nil + + return + } + + err = gherrors.Wrap(e, "failed to patch") return } + + err = nil }() - // ResourceQuota Reconciliation err = r.reconcile(ctx, log, instance) return ctrl.Result{}, err @@ -209,7 +226,7 @@ func (r *resourcePoolController) reconcile( pool.CalculateClaimedResources() pool.AssignClaims() - if err := r.syncResourceQuotas(ctx, r.Client, pool, namespaces); err != nil { + if err := r.syncResourceQuotas(ctx, r.Client, r.reader, pool, namespaces); err != nil { return fmt.Errorf("sync resourcequotas: %w", err) } @@ -230,7 +247,7 @@ func (r *resourcePoolController) reconcile( cond.Reason = meta.FailedReason cond.Message = "claim causes exhaustions" - if err := updateStatusAndEmitEvent(ctx, r.Client, r.recorder, cl, cond); err != nil { + if err := updateStatusAndEmitEvent(ctx, r.Client, r.recorder, cl, pool, cond); err != nil { errs = append(errs, fmt.Errorf("update exhausted claim condition %s/%s: %w", cl.Namespace, cl.Name, err)) } } @@ -272,7 +289,7 @@ func (r *resourcePoolController) reconcileClaimsInUseForNamespace( ) error { // Fetch the quota we manage for this namespace rq := &corev1.ResourceQuota{} - if err := r.Get(ctx, types.NamespacedName{ + if err := r.reader.Get(ctx, types.NamespacedName{ Name: pool.GetQuotaName(), Namespace: namespace, }, rq); err != nil { if apierrors.IsNotFound(err) { @@ -312,7 +329,7 @@ func (r *resourcePoolController) reconcileClaimsInUseForNamespace( err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { current := &capsulev1beta2.ResourcePoolClaim{} - if err := r.Get(ctx, client.ObjectKeyFromObject(cl), current); err != nil { + if err := r.reader.Get(ctx, client.ObjectKeyFromObject(cl), current); err != nil { return fmt.Errorf("failed to refetch instance before update: %w", err) } @@ -348,6 +365,7 @@ func (r *resourcePoolController) reconcileResourceClaim( queued, err = r.handleClaimOrderedExhaustion( ctx, + pool, claim, exhaustion, ) @@ -369,6 +387,7 @@ func (r *resourcePoolController) reconcileResourceClaim( return r.handleClaimResourceExhaustion( ctx, + pool, claim, exhaustions, exhaustion, @@ -412,6 +431,7 @@ func (r *resourcePoolController) canClaimWithinNamespace( // Handles exhaustions when a exhaustion was already declared in the given map. func (r *resourcePoolController) handleClaimOrderedExhaustion( ctx context.Context, + pool *capsulev1beta2.ResourcePool, claim *capsulev1beta2.ResourcePoolClaim, exhaustions map[string]api.PoolExhaustionResource, ) (queued bool, err error) { @@ -441,7 +461,7 @@ func (r *resourcePoolController) handleClaimOrderedExhaustion( cond.Reason = meta.QueueExhaustedReason cond.Message = strings.Join(status, "; ") - return queued, updateStatusAndEmitEvent(ctx, r.Client, r.recorder, claim, cond) + return queued, updateStatusAndEmitEvent(ctx, r.Client, r.recorder, claim, pool, cond) } return queued, err @@ -449,6 +469,7 @@ func (r *resourcePoolController) handleClaimOrderedExhaustion( func (r *resourcePoolController) handleClaimResourceExhaustion( ctx context.Context, + pool *capsulev1beta2.ResourcePool, claim *capsulev1beta2.ResourcePoolClaim, currentExhaustions map[string]api.PoolExhaustionResource, exhaustions map[string]api.PoolExhaustionResource, @@ -490,7 +511,7 @@ func (r *resourcePoolController) handleClaimResourceExhaustion( cond.Reason = meta.PoolExhaustedReason cond.Message = strings.Join(status, "; ") - return updateStatusAndEmitEvent(ctx, r.Client, r.recorder, claim, cond) + return updateStatusAndEmitEvent(ctx, r.Client, r.recorder, claim, pool, cond) } return err @@ -506,7 +527,7 @@ func (r *resourcePoolController) handleClaimToPoolBinding( cond.Reason = meta.NoExhaustionsReason cond.Message = "resource claimable from pool" - if err = updateStatusAndEmitEvent(ctx, r.Client, r.recorder, claim, cond); err != nil { + if err = updateStatusAndEmitEvent(ctx, r.Client, r.recorder, claim, pool, cond); err != nil { return err } @@ -531,7 +552,7 @@ func (r *resourcePoolController) handleClaimDisassociation( } err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if err := r.Get(ctx, types.NamespacedName{ + if err := r.reader.Get(ctx, types.NamespacedName{ Name: claim.Name.String(), Namespace: claim.Namespace.String(), }, current); err != nil { @@ -562,8 +583,8 @@ func (r *resourcePoolController) handleClaimDisassociation( } r.recorder.Eventf( - pool, current, + pool, corev1.EventTypeNormal, evt.ReasonDisassociated, evt.ActionDisassociating, @@ -587,6 +608,7 @@ func (r *resourcePoolController) handleClaimDisassociation( func (r *resourcePoolController) syncResourceQuotas( ctx context.Context, c client.Client, + reader client.Reader, quota *capsulev1beta2.ResourcePool, namespaces []corev1.Namespace, ) (err error) { @@ -596,7 +618,7 @@ func (r *resourcePoolController) syncResourceQuotas( namespace := ns group.Go(func() error { - return r.syncResourceQuota(ctx, c, quota, namespace) + return r.syncResourceQuota(ctx, c, reader, quota, namespace) }) } @@ -607,6 +629,7 @@ func (r *resourcePoolController) syncResourceQuotas( func (r *resourcePoolController) syncResourceQuota( ctx context.Context, c client.Client, + reader client.Reader, pool *capsulev1beta2.ResourcePool, namespace corev1.Namespace, ) (err error) { @@ -624,7 +647,7 @@ func (r *resourcePoolController) syncResourceQuota( }, } - if err := c.Get(ctx, types.NamespacedName{Name: target.Name, Namespace: target.Namespace}, target); err != nil && !apierrors.IsNotFound(err) { + if err := reader.Get(ctx, types.NamespacedName{Name: target.Name, Namespace: target.Namespace}, target); err != nil && !apierrors.IsNotFound(err) { return err } @@ -686,7 +709,7 @@ func (r *resourcePoolController) gatherMatchingNamespaces( } for _, selector := range pool.Spec.Selectors { - selected, serr := selector.GetMatchingNamespaces(ctx, r.Client) + selected, serr := selector.GetMatchingNamespaces(ctx, r.reader) if serr != nil { log.Error(err, "Cannot get matching namespaces") @@ -838,7 +861,7 @@ func (r *resourcePoolController) garbageCollectNamespace( // Check if the namespace still exists ns := &corev1.Namespace{} - if err := r.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil { + if err := r.reader.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil { if apierrors.IsNotFound(err) { r.log.V(5).Info("Namespace does not exist, skipping garbage collection", "namespace", namespace) @@ -858,7 +881,7 @@ func (r *resourcePoolController) garbageCollectNamespace( }, } - err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: target.GetName()}, target) + err := r.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: target.GetName()}, target) if err != nil { if apierrors.IsNotFound(err) { r.log.V(5).Info("ResourceQuota already deleted", "namespace", namespace, "name", name) @@ -877,17 +900,21 @@ func (r *resourcePoolController) garbageCollectNamespace( return nil } -func (r *resourcePoolController) updateStatus(ctx context.Context, pool *capsulev1beta2.ResourcePool, reconcileError error) error { +func (r *resourcePoolController) updateStatus(ctx context.Context, instance *capsulev1beta2.ResourcePool, reconcileError error) error { return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { latest := &capsulev1beta2.ResourcePool{} - if err = r.Get(ctx, types.NamespacedName{Name: pool.GetName()}, latest); err != nil { + if err = r.reader.Get(ctx, types.NamespacedName{Name: instance.GetName()}, latest); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err } - latest.Status = pool.Status + latest.Status = instance.Status // Set Ready Condition - readyCondition := meta.NewReadyCondition(pool) + readyCondition := meta.NewReadyCondition(instance) if reconcileError != nil { readyCondition.Message = reconcileError.Error() readyCondition.Status = metav1.ConditionFalse @@ -897,7 +924,7 @@ func (r *resourcePoolController) updateStatus(ctx context.Context, pool *capsule latest.Status.Conditions.UpdateConditionByType(readyCondition) // Set Exhaustion Condition - exCondition := meta.NewExhaustedCondition(pool) + exCondition := meta.NewExhaustedCondition(instance) if len(latest.Status.Exhaustions) != 0 { exCondition.Message = "Pool has exhaustions" exCondition.Status = metav1.ConditionTrue @@ -906,6 +933,13 @@ func (r *resourcePoolController) updateStatus(ctx context.Context, pool *capsule latest.Status.Conditions.UpdateConditionByType(exCondition) - return r.Client.Status().Update(ctx, latest) + if err := r.Client.Status().Update(ctx, latest); err != nil { + return err + } + + // Keep the in-memory object aligned with what we just wrote. + instance.Status = latest.Status + + return nil }) } diff --git a/internal/controllers/resourcepools/utils.go b/internal/controllers/resourcepools/utils.go index e6bdce5c..dcdf9f90 100644 --- a/internal/controllers/resourcepools/utils.go +++ b/internal/controllers/resourcepools/utils.go @@ -25,6 +25,7 @@ func updateStatusAndEmitEvent( c client.Client, recorder events.EventRecorder, claim *capsulev1beta2.ResourcePoolClaim, + pool *capsulev1beta2.ResourcePool, condition meta.Condition, ) (err error) { updated := claim.Status.Conditions.UpdateConditionByTypeWithStatus(condition) @@ -54,7 +55,7 @@ func updateStatusAndEmitEvent( recorder.Eventf( claim, - claim, + pool, eventType, condition.Reason, evt.ActionReconciled, diff --git a/internal/controllers/resources/collect.go b/internal/controllers/resources/collect.go new file mode 100644 index 00000000..698df9bc --- /dev/null +++ b/internal/controllers/resources/collect.go @@ -0,0 +1,471 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "errors" + "fmt" + "maps" + "strconv" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + k8smeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/processor" + "github.com/projectcapsule/capsule/pkg/runtime/gvk" + "github.com/projectcapsule/capsule/pkg/runtime/sanitize" + tpl "github.com/projectcapsule/capsule/pkg/template" + "github.com/projectcapsule/capsule/pkg/tenant" + "github.com/projectcapsule/capsule/pkg/utils" +) + +type Collector struct { + gatherClient client.Reader + mapper k8smeta.RESTMapper + contextSanitizeOptions sanitize.SanitizeOptions + objectSanitizeOptions sanitize.SanitizeOptions + reservedLabelSet map[string]struct{} +} + +type CollectorOptions struct { + AllowCrossNamespaceSelection bool + Accumulator processor.Accumulator + Iterator CollectorIteratorOptions + ValidatorNamespaces tpl.NamespaceValidator +} + +type CollectorIteratorOptions struct { + Labels map[string]string + Annotations map[string]string + FastContext map[string]string + FastContextAny map[string]any +} + +func NewCollectorIteratorOptions( + tnt *capsulev1beta2.Tenant, + ns *corev1.Namespace, + spec capsulev1beta2.ResourceSpec, +) CollectorIteratorOptions { + opts := CollectorIteratorOptions{} + + opts.FastContext = tenant.FastContextForTenantAndNamespace(tnt, ns) + + labels, annotations := GatherAdditionalMetadata(spec, opts.FastContext) + opts.Labels = labels + opts.Annotations = annotations + + return opts +} + +func NewCollector(c client.Reader, mapper k8smeta.RESTMapper) Collector { + return Collector{ + gatherClient: c, + mapper: mapper, + contextSanitizeOptions: sanitize.SanitizeOptions{ + StripUID: false, + StripManagedFields: true, + StripLastApplied: true, + StripStatus: false, + }, + objectSanitizeOptions: sanitize.DefaultSanitizeOptions(), + reservedLabelSet: map[string]struct{}{ + meta.ResourcesLabel: {}, + meta.CreatedByCapsuleLabel: {}, + meta.ManagedByCapsuleLabel: {}, + meta.NewManagedByCapsuleLabel: {}, + }, + } +} + +// With this function we are attempting to collect all the unstructured items +// No Interacting is done with the kubernetes regarding applying etc. +// + +func (co *Collector) Collect( + ctx context.Context, + c client.Client, + opts CollectorOptions, + tnt *capsulev1beta2.Tenant, + resourceIndex string, + spec capsulev1beta2.ResourceSpec, + ns *corev1.Namespace, +) (err error) { + log := log.FromContext(ctx) + + var syncErr error + + tplContext := tpl.ReferenceContext{} + + if spec.Context != nil { + namespace := "" + if ns != nil { + namespace = ns.GetName() + } + + tplContext, err = spec.Context.GatherContext( + ctx, + c, + co.mapper, + opts.Iterator.FastContext, + namespace, + nil, + opts.ValidatorNamespaces, + ) + if err != nil { + return err + } + } + + if tnt != nil { + tCtx, err := tenant.NewTenantContext(tnt, c.Scheme(), co.contextSanitizeOptions) + if err != nil { + return err + } + + tplContext["tenant"] = tCtx + } + + if ns != nil { + err = sanitize.SanitizeObject(ns, c.Scheme(), co.contextSanitizeOptions) + if err != nil { + return err + } + + nsMap, err := utils.ToUnstructuredMap(&ns) + if err != nil { + return err + } + + tplContext["namespace"] = nsMap + } + + log.V(7).Info("available context", "context", tplContext) + + // Run Raw Items + for rawIndex, item := range spec.RawItems { + log.V(5).Info("processing raw item", "index", rawIndex) + + p, rawError := co.handleRawItem(ctx, c, opts, item, ns) + if rawError != nil { + syncErr = errors.Join(syncErr, rawError) + + continue + } + + log.V(7).Info("evaluated raw item", "object", p) + + rawError = co.AddToAccumulation(tnt, ns, opts, spec, p, resourceIndex+"/raw-"+strconv.Itoa(rawIndex), true) + if rawError != nil { + syncErr = errors.Join(syncErr, rawError) + + continue + } + } + + // Run Generators + for generatorIndex, item := range spec.Generators { + log.V(5).Info("processing generator item", "index", generatorIndex) + + p, genError := co.handleGeneratorItem(ctx, c, generatorIndex, item, ns, tplContext) + if genError != nil { + syncErr = errors.Join(syncErr, genError) + + continue + } + + log.V(5).Info("loaded resources", "amount", len(p)) + + for i, o := range p { + genError = co.AddToAccumulation(tnt, ns, opts, spec, o, resourceIndex+"/generator-"+strconv.Itoa(generatorIndex)+"-"+strconv.Itoa(i), true) + if genError != nil { + syncErr = errors.Join(syncErr, genError) + + continue + } + } + } + + return syncErr +} + +// Add an item to the accumulator +// Mainly handles conflicts. +func (co *Collector) AddToAccumulation( + tnt *capsulev1beta2.Tenant, + ns *corev1.Namespace, + opts CollectorOptions, + spec capsulev1beta2.ResourceSpec, + obj *unstructured.Unstructured, + origin string, + combine bool, +) (err error) { + if obj == nil { + return err + } + + tntName := "" + if tnt != nil { + tntName = tnt.GetName() + } + + resource := gvk.NewResourceID(obj, tntName, origin) + + if !combine { + if _, k := opts.Accumulator[resource.GetKey("")]; k { + return nil + } + } + + if !opts.AllowCrossNamespaceSelection && ns != nil { + obj.SetNamespace(ns.GetName()) + } + + if len(opts.Iterator.Labels) > 0 { + dst := obj.GetLabels() + if dst == nil { + dst = make(map[string]string, len(opts.Iterator.Labels)) + } + + maps.Copy(dst, opts.Iterator.Labels) + obj.SetLabels(dst) + + meta.SetFilteredLabels(obj, co.reservedLabelSet) + } + + if len(opts.Iterator.Annotations) > 0 { + dst := obj.GetAnnotations() + if dst == nil { + dst = make(map[string]string, len(opts.Iterator.Annotations)) + } + + maps.Copy(dst, opts.Iterator.Annotations) + + obj.SetAnnotations(dst) + } + + sanitize.SanitizeUnstructured(obj, co.objectSanitizeOptions) + + processor.AccumulatorAdd(opts.Accumulator, resource, processor.AccumulatorObject{ + Object: obj, + Origin: gvk.TenantResourceIDWithOrigin{ + TenantResourceID: gvk.TenantResourceID{ + Tenant: tntName, + }, + Origin: origin, + }, + }) + + return nil +} + +func (co *Collector) CollectNamespacedItems( + ctx context.Context, + c client.Client, + opts CollectorOptions, + spec capsulev1beta2.ResourceSpec, + ns *corev1.Namespace, + tnt capsulev1beta2.Tenant, +) (items map[gvk.ResourceKey]*unstructured.Unstructured, err error) { + var totalError error + + seen := make(map[gvk.ResourceKey]*unstructured.Unstructured) + + log := log.FromContext(ctx) + tntNamespaces := sets.NewString(tnt.Status.Namespaces...) + + namespace := "" + if !opts.AllowCrossNamespaceSelection && ns != nil { + namespace = ns.GetName() + } + + // A TenantResource is created by a TenantOwner, and potentially, they could point to a resource in a non-owned + // Namespace: this must be blocked by checking it this is the case. + if !opts.AllowCrossNamespaceSelection && !tntNamespaces.Has(namespace) { + err = fmt.Errorf("cross-namespace selection is not allowed. Referring a Namespace (%s) that is not part of the given Tenant (allowed %s)", namespace, tntNamespaces) + + return nil, err + } + + selector, err := getSelectorForCreatedResourcesExclusion() + if err != nil { + return nil, err + } + + for _, item := range spec.NamespacedItems { + p, err := item.LoadResources(ctx, c, co.mapper, namespace, []labels.Selector{selector}, opts.Iterator.FastContext, false, opts.ValidatorNamespaces) + if err != nil { + totalError = errors.Join(totalError, err) + + continue + } + + // Remove the keys from the result by which they were sourced. + filterKeys := meta.LabelSelectorKeys(item.Selector) + + for _, o := range p { + // Namespaced Items are different. Even if we allow cross namespace loading + // If a target namespace is given it always is used + if ns != nil && ns.GetName() != "" { + o.SetNamespace(ns.GetName()) + } + + k, ok := gvk.KeyFromUnstructured(o) + if ok { + if _, already := seen[k]; already { + log.V(6).Info("skipping duplicate loaded resource", + "gvk", schema.GroupVersionKind{Group: k.Group, Version: k.Version, Kind: k.Kind}.String(), + "namespace", k.Namespace, + "name", k.Name, + ) + + continue + } + + log.V(6).Info("loaded resource", + "gvk", schema.GroupVersionKind{Group: k.Group, Version: k.Version, Kind: k.Kind}.String(), + "namespace", k.Namespace, + "name", k.Name, + ) + + meta.SetFilteredLabels(o, filterKeys) + + seen[k] = o + } else { + log.V(4).Info("resource missing identity; cannot dedupe reliably", + "apiVersion", o.GetAPIVersion(), "kind", o.GetKind(), "namespace", o.GetNamespace(), "name", o.GetName(), + ) + } + } + } + + return seen, totalError +} + +// Allows templating in. +func GatherAdditionalMetadata( + spec capsulev1beta2.ResourceSpec, + fastContext map[string]string, +) (labels map[string]string, annotations map[string]string) { + labels = make(map[string]string) + annotations = make(map[string]string) + + md := spec.AdditionalMetadata + if md == nil { + return labels, annotations + } + + if md.Labels != nil { + labels = tpl.FastTemplateMap(maps.Clone(md.Labels), fastContext) + } + + if md.Annotations != nil { + annotations = tpl.FastTemplateMap(maps.Clone(md.Annotations), fastContext) + } + + return labels, annotations +} + +// Handles a single generator item. +func (co *Collector) handleGeneratorItem( + ctx context.Context, + c client.Client, + index int, + item capsulev1beta2.TemplateItemSpec, + ns *corev1.Namespace, + tmplContext tpl.ReferenceContext, +) (processed []*unstructured.Unstructured, err error) { + objs, err := tpl.RenderUnstructuredItems(tmplContext, item.MissingKey, item.Template) + if err != nil { + return nil, fmt.Errorf("error running generator: %w", err) + } + + for _, obj := range objs { + if ns != nil { + obj.SetNamespace(ns.Name) + } + + processed = append(processed, obj) + } + + return +} + +func (co *Collector) handleRawItem( + ctx context.Context, + c client.Client, + opts CollectorOptions, + item capsulev1beta2.RawExtension, + ns *corev1.Namespace, +) (processed *unstructured.Unstructured, err error) { + tmplString := tpl.FastTemplate(string(item.Raw), opts.Iterator.FastContext) + + obj := &unstructured.Unstructured{} + if _, _, err := unstructured.UnstructuredJSONScheme.Decode([]byte(tmplString), nil, obj); err != nil { + return nil, fmt.Errorf("decode unstructured: %w", err) + } + + if ns != nil { + obj.SetNamespace(ns.Name) + } + + return obj, nil +} + +func (co *Collector) selectedTenantNamespaces( + ctx context.Context, + log logr.Logger, + tnt capsulev1beta2.Tenant, + resource capsulev1beta2.ResourceSpec, +) (ns []*corev1.Namespace, err error) { + // Creating Namespace selector + var selector labels.Selector + + if resource.NamespaceSelector != nil { + selector, err = metav1.LabelSelectorAsSelector(resource.NamespaceSelector) + if err != nil { + log.Error(err, "cannot create Namespace selector for Namespace filtering and resource replication") + + return nil, err + } + } else { + selector = labels.NewSelector() + } + // Resources can be replicated only on Namespaces belonging to the same Global: + // preventing a boundary cross by enforcing the selection. + tntRequirement, err := labels.NewRequirement(meta.TenantLabel, selection.Equals, []string{tnt.GetName()}) + if err != nil { + log.Error(err, "unable to create requirement for Namespace filtering and resource replication") + + return nil, err + } + + selector = selector.Add(*tntRequirement) + // Selecting the targeted Namespace according to the TenantResource specification. + namespaces := corev1.NamespaceList{} + if err = co.gatherClient.List(ctx, &namespaces, client.MatchingLabelsSelector{Selector: selector}); err != nil { + log.Error(err, "cannot retrieve Namespaces for resource") + + return nil, err + } + + log.V(5).Info("retrieved namespaces", "size", len(namespaces.Items)) + + for _, names := range namespaces.Items { + ns = append(ns, &names) + } + + return ns, nil +} diff --git a/internal/controllers/resources/global.go b/internal/controllers/resources/global.go index b22cded0..0b046211 100644 --- a/internal/controllers/resources/global.go +++ b/internal/controllers/resources/global.go @@ -5,56 +5,110 @@ package resources import ( "context" - "errors" + "fmt" + "strconv" + "github.com/go-logr/logr" gherrors "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/retry" "sigs.k8s.io/cluster-api/util/patch" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" ctrllog "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/internal/metrics" + "github.com/projectcapsule/capsule/pkg/api" + caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/processor" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/predicates" + "github.com/projectcapsule/capsule/pkg/runtime/sanitize" ) -type Global struct { - client client.Client - processor Processor +type globalResourceController struct { + client client.Client + reader client.Reader + + log logr.Logger + processor processor.Processor + collector Collector + configuration configuration.Configuration + metrics *metrics.GlobalTenantResourceRecorder + + impersonation *cache.ImpersonationCache } -func (r *Global) SetupWithManager(mgr ctrl.Manager, cfg utils.ControllerOptions) error { +func (r *globalResourceController) SetupWithManager(mgr ctrl.Manager, cfg utils.ControllerOptions) error { r.client = mgr.GetClient() - r.processor = Processor{ - client: mgr.GetClient(), + r.reader = mgr.GetAPIReader() + + r.processor = processor.Processor{ + Configuration: r.configuration, + GatherClient: mgr.GetAPIReader(), + AllowCrossNamespaceSelection: true, + Mapper: mgr.GetRESTMapper(), } + r.collector = NewCollector( + mgr.GetAPIReader(), + mgr.GetRESTMapper(), + ) return ctrl.NewControllerManagedBy(mgr). - For(&capsulev1beta2.GlobalTenantResource{}). - Watches(&capsulev1beta2.Tenant{}, handler.EnqueueRequestsFromMapFunc(r.enqueueRequestFromTenant)). + For( + &capsulev1beta2.GlobalTenantResource{}, + builder.WithPredicates( + predicate.Or( + predicate.GenerationChangedPredicate{}, + predicates.ReconcileRequestedPredicate{}, + ), + ), + ). + Watches( + &capsulev1beta2.CapsuleConfiguration{}, + handler.EnqueueRequestsFromMapFunc(r.enqueueAllResources), + builder.WithPredicates( + predicates.CapsuleConfigSpecImpersonationChangedPredicate{}, + predicates.NamesMatchingPredicate{Names: []string{cfg.ConfigurationName}}, + ), + ). + Watches( + &capsulev1beta2.GlobalTenantResource{}, + handler.EnqueueRequestsFromMapFunc(r.enqueueDependentGlobalTenantResources), + ). + Watches( + &capsulev1beta2.Tenant{}, + handler.EnqueueRequestsFromMapFunc(r.enqueueRequestFromTenant), + ). WithOptions(controller.Options{MaxConcurrentReconciles: cfg.MaxConcurrentReconciles}). Complete(r) } -func (r *Global) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - var err error - +func (r *globalResourceController) Reconcile(ctx context.Context, request reconcile.Request) (res reconcile.Result, err error) { log := ctrllog.FromContext(ctx) - log.V(4).Info("start processing") - // Retrieving the GlobalTenantResource + log.V(5).Info("start processing") + tntResource := &capsulev1beta2.GlobalTenantResource{} if err = r.client.Get(ctx, request.NamespacedName, tntResource); err != nil { if apierrors.IsNotFound(err) { - log.V(3).Info("Request object not found, could have been deleted after reconcile request") + log.V(5).Info("Request object not found, could have been deleted after reconcile request") + + r.metrics.DeleteMetrics(request.Name) return reconcile.Result{}, nil } @@ -62,29 +116,161 @@ func (r *Global) Reconcile(ctx context.Context, request reconcile.Request) (reco return reconcile.Result{}, err } + requeue := reconcile.Result{ + Requeue: true, + RequeueAfter: tntResource.Spec.ResyncPeriod.Duration, + } + patchHelper, err := patch.NewHelper(tntResource, r.client) if err != nil { return reconcile.Result{}, gherrors.Wrap(err, "failed to init patch helper") } + var statusErr error + + //nolint:dupl defer func() { - if e := patchHelper.Patch(ctx, tntResource); e != nil { - if err == nil { - err = gherrors.Wrap(e, "failed to patch GlobalTenantResource") - } + reconcileErr := err + if statusErr != nil { + reconcileErr = statusErr } + + if uerr := r.updateStatus(ctx, tntResource, reconcileErr); uerr != nil { + if caperrors.IgnoreGone(uerr) { + err = nil + + return + } + + err = fmt.Errorf("cannot update globaltenantresource status: %w", uerr) + + return + } + + r.metrics.RecordConditions(tntResource) + + if e := patchHelper.Patch(ctx, tntResource); e != nil { + if caperrors.IgnoreGone(e) { + err = nil + + return + } + + res = reconcile.Result{} + err = gherrors.Wrap(e, "failed to patch GlobalTenantResource") + + return + } + + // Controller-runtime should not receive handled reconciliation errors. + err = nil }() - // Handle deleted GlobalTenantResource - if !tntResource.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, tntResource) + // On Deletion these checks are skipped. + //nolint:nestif + if tntResource.DeletionTimestamp.IsZero() { + if tntResource.Spec.Cordoned != nil && *tntResource.Spec.Cordoned { + log.V(5).Info("global tenant resource cordoned") + + return reconcile.Result{}, nil + } + + for _, dep := range tntResource.Spec.DependsOn { + d := &capsulev1beta2.GlobalTenantResource{} + + if getErr := r.client.Get(ctx, types.NamespacedName{Name: dep.Name.String()}, d); getErr != nil { + if apierrors.IsNotFound(getErr) { + statusErr = fmt.Errorf("dependency %s not found", dep.Name) + } else { + statusErr = getErr + } + + return requeue, nil + } + + stat := d.Status.Conditions.GetConditionByType(meta.ReadyCondition) + if stat == nil || stat.Status != metav1.ConditionTrue { + statusErr = fmt.Errorf("dependency %s not ready", dep.Name) + + return requeue, nil + } + } } - // Handle non-deleted GlobalTenantResource - return r.reconcileNormal(ctx, tntResource) + // Load client must be first since it updates the new serviceaccount used which can then be directly + // posted to the status. + c, loadErr := r.loadClient(ctx, log, tntResource) + if loadErr != nil { + statusErr = gherrors.Wrap(loadErr, "failed to load serviceaccount client") + + return requeue, nil + } + + if updateErr := r.updateReconcilingStatus(ctx, tntResource); updateErr != nil { + if caperrors.IgnoreGone(updateErr) { + return reconcile.Result{}, nil + } + + statusErr = gherrors.Wrap(updateErr, "failed to update status") + + return requeue, nil + } + + if c == nil { + statusErr = fmt.Errorf("received empty client for serviceaccount") + + return requeue, nil + } + + statusErr = r.reconcile(ctx, c, tntResource) + + if len(tntResource.Status.ProcessedItems) > 0 { + controllerutil.AddFinalizer(tntResource, meta.ControllerFinalizer) + } else { + controllerutil.RemoveFinalizer(tntResource, meta.ControllerFinalizer) + } + + controllerutil.RemoveFinalizer(tntResource, meta.LegacyResourceFinalizer) + + return requeue, nil } -func (r *Global) enqueueRequestFromTenant(ctx context.Context, object client.Object) (reqs []reconcile.Request) { +//nolint:dupl +func (r *globalResourceController) enqueueDependentGlobalTenantResources( + ctx context.Context, + obj client.Object, +) []ctrl.Request { + changed, ok := obj.(*capsulev1beta2.GlobalTenantResource) + if !ok { + return nil + } + + var list capsulev1beta2.GlobalTenantResourceList + if err := r.client.List(ctx, &list); err != nil { + return nil + } + + reqs := make([]ctrl.Request, 0) + + for _, gtr := range list.Items { + for _, dep := range gtr.Spec.DependsOn { + if dep.Name.String() == changed.Name { + reqs = append(reqs, ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: gtr.Name, + Namespace: gtr.Namespace, + }, + }) + + break + } + } + } + + return reqs +} + +func (r *globalResourceController) enqueueRequestFromTenant(ctx context.Context, object client.Object) (reqs []reconcile.Request) { tnt := object.(*capsulev1beta2.Tenant) //nolint:forcetypeassert resList := capsulev1beta2.GlobalTenantResourceList{} @@ -118,15 +304,37 @@ func (r *Global) enqueueRequestFromTenant(ctx context.Context, object client.Obj return reqs } -func (r *Global) reconcileNormal(ctx context.Context, tntResource *capsulev1beta2.GlobalTenantResource) (reconcile.Result, error) { - log := ctrllog.FromContext(ctx) +//nolint:dupl +func (r *globalResourceController) enqueueAllResources(ctx context.Context, _ client.Object) []reconcile.Request { + var list capsulev1beta2.GlobalTenantResourceList + if err := r.client.List(ctx, &list); err != nil { + r.log.V(1).Error(err, "unable to list GlobalTenantResourceList for config-triggered reconcile") - if *tntResource.Spec.PruningOnDelete { - controllerutil.AddFinalizer(tntResource, finalizer) + return nil } + reqs := make([]reconcile.Request, 0, len(list.Items)) + for i := range list.Items { + reqs = append(reqs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: list.Items[i].Name, + Namespace: list.Items[i].Namespace, + }, + }) + } + + return reqs +} + +func (r *globalResourceController) reconcile( + ctx context.Context, + c client.Client, + tntResource *capsulev1beta2.GlobalTenantResource, +) (err error) { + log := ctrllog.FromContext(ctx) + if tntResource.Status.ProcessedItems == nil { - tntResource.Status.ProcessedItems = make([]capsulev1beta2.ObjectReferenceStatus, 0) + tntResource.Status.ProcessedItems = make([]meta.ObjectReferenceStatus, 0) } // Retrieving the list of the Tenants up to the selector provided by the GlobalTenantResource resource. @@ -134,78 +342,337 @@ func (r *Global) reconcileNormal(ctx context.Context, tntResource *capsulev1beta if err != nil { log.Error(err, "cannot create MatchingLabelsSelector for Global filtering") - return reconcile.Result{}, err + return err } + // Use Controller Client. tntList := capsulev1beta2.TenantList{} - if err = r.client.List(ctx, &tntList, &client.MatchingLabelsSelector{Selector: tntSelector}); err != nil { + if err = r.reader.List(ctx, &tntList, &client.MatchingLabelsSelector{Selector: tntSelector}); err != nil { log.Error(err, "cannot list Tenants matching the provided selector") - return reconcile.Result{}, err + return err } - // This is the list of newer Tenants that are matching the provided GlobalTenantResource Selector: - // upon replication and pruning, this will be updated in the status of the resource. - tntSet := sets.NewString() - // A TenantResource is made of several Resource sections, each one with specific options: - // the Status can be updated only in case of no errors across all of them to guarantee a valid and coherent status. - processedItems := sets.NewString() + filtered := make([]capsulev1beta2.Tenant, 0, len(tntList.Items)) - for index, resource := range tntResource.Spec.Resources { - tenantLabel, labelErr := capsulev1beta2.GetTypeLabel(&capsulev1beta2.Tenant{}) - if labelErr != nil { - log.Error(labelErr, "expected label for selection") - - return reconcile.Result{}, labelErr + for _, tnt := range tntList.Items { + if tnt.DeletionTimestamp != nil { + continue } - for _, tnt := range tntList.Items { - tntSet.Insert(tnt.GetName()) + filtered = append(filtered, tnt) + } - items, sectionErr := r.processor.HandleSection(ctx, tnt, true, tenantLabel, index, resource) - if sectionErr != nil { - // Upon a process error storing the last error occurred and continuing to iterate, - // avoid to block the whole processing. - err = errors.Join(err, sectionErr) - } else { - processedItems.Insert(items...) + // Always post the processed items, as they allow users to track errors + defer func() { + tntResource.AssignTenants(filtered) + }() + + acc := processor.Accumulator{} + owner := meta.GetLooseOwnerReference(tntResource) + + // Gather Resources + if tntResource.DeletionTimestamp.IsZero() { + err := r.gatherResources( + ctx, + c, + log, + tntResource, + tntList, + acc, + ) + if err != nil { + return err + } + } + + return r.processor.Reconcile( + ctx, + log, + c, + &tntResource.Status.ProcessedItems, + acc, + processor.ProcessorOptions{ + FieldOwnerPrefix: getFieldOwner(tntResource.GetName(), tntResource.GetNamespace()), + Prune: *tntResource.Spec.PruningOnDelete, + Adopt: *tntResource.Spec.Settings.Adopt, + Force: *tntResource.Spec.Settings.Force, + Owner: &owner, + }) +} + +//nolint:gocognit +func (r *globalResourceController) gatherResources( + ctx context.Context, + c client.Client, + log logr.Logger, + tntResource *capsulev1beta2.GlobalTenantResource, + tnts capsulev1beta2.TenantList, + acc processor.Accumulator, +) error { + opts := CollectorOptions{ + Accumulator: acc, + AllowCrossNamespaceSelection: true, + } + + // Collect Available Generated Items + for resourceIndex, resource := range tntResource.Spec.Resources { + switch tntResource.Spec.Scope { + case api.ResourceScopeNone: + ilog := log.WithValues("tenant", "Cluster", "resource", resourceIndex) + ilog.V(5).Info("replicating once for cluster scope") + + clusterTenant := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "None", + }, + } + + opts.Iterator = NewCollectorIteratorOptions(clusterTenant, nil, resource) + + if err := r.collector.Collect( + ctx, + c, + opts, + clusterTenant, + strconv.Itoa(resourceIndex), + resource, + nil, + ); err != nil { + return err + } + + case api.ResourceScopeTenant: + for _, tnt := range tnts.Items { + ilog := log.WithValues("tenant", tnt.GetName(), "resource", resourceIndex) + ilog.V(5).Info("replicating for each tenant") + + opts.Iterator = NewCollectorIteratorOptions(&tnt, nil, resource) + + if err := r.collector.Collect( + ctx, + c, + opts, + &tnt, + strconv.Itoa(resourceIndex), + resource, + nil, + ); err != nil { + return err + } + } + + case api.ResourceScopeNamespace: + for _, tnt := range tnts.Items { + ilog := log.WithValues("tenant", tnt.GetName(), "resource", resourceIndex) + ilog.V(5).Info("replicating for each namespace") + + opts.AllowCrossNamespaceSelection = true + + objs, err := r.collector.CollectNamespacedItems(ctx, c, opts, resource, nil, tnt) + if err != nil { + return err + } + + for g := range objs { + ilog.V(5).Info("found replication source object", "name", g.Name, "namespace", g.Namespace, "kind", g.Kind) + } + + namespaces, err := r.collector.selectedTenantNamespaces(ctx, log, tnt, resource) + if err != nil { + return err + } + + opts.AllowCrossNamespaceSelection = false + + for _, innerNs := range namespaces { + opts.Iterator = NewCollectorIteratorOptions(&tnt, innerNs, resource) + + for _, obj := range objs { + if obj.GetNamespace() == innerNs.GetName() { + continue + } + + target := obj.DeepCopy() + if err := sanitize.SanitizeObject(target, c.Scheme(), r.collector.objectSanitizeOptions); err != nil { + return err + } + + target.SetNamespace(innerNs.GetName()) + + log.V(4).Info( + "adding replication for namespaced item", + "name", target.GetName(), + "namespace", target.GetNamespace(), + "kind", target.GetKind(), + ) + + if err := r.collector.AddToAccumulation(&tnt, innerNs, opts, resource, target, "replica", false); err != nil { + return err + } + } + + if err := r.collector.Collect( + ctx, + c, + opts, + &tnt, + strconv.Itoa(resourceIndex), + resource, + innerNs, + ); err != nil { + return err + } + } } } } + return nil +} + +//nolint:dupl +func (r *globalResourceController) loadClient( + ctx context.Context, + log logr.Logger, + tntResource *capsulev1beta2.GlobalTenantResource, +) (client.Client, error) { + sa := r.impersonatedServiceAccount(ctx, log, tntResource) + if sa == nil { + sa, ns := configuration.ControllerServiceAccount() + + tntResource.Status.ServiceAccount = &meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: meta.RFC1123Name(sa), + Namespace: meta.RFC1123SubdomainName(ns), + } + + return r.client, nil + } + + tntResource.Status.ServiceAccount = &meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: sa.Name, + Namespace: sa.Namespace, + } + + re, err := r.configuration.ServiceAccountClient(ctx) if err != nil { - log.Error(err, "unable to replicate the requested resources") + log.Error(err, "failed to load impersonated rest client") - return reconcile.Result{}, err + return nil, err } - if r.processor.HandlePruning(ctx, tntResource.Status.ProcessedItems.AsSet(), sets.Set[string](processedItems)) { - tntResource.Status.ProcessedItems = make([]capsulev1beta2.ObjectReferenceStatus, 0, len(processedItems)) + log.V(5).Info("using impersonation client", "serviceaccount", sa.Name, "namespace", sa.Namespace) - for _, item := range processedItems.List() { - if or := (capsulev1beta2.ObjectReferenceStatus{}); or.ParseFromString(item) == nil { - tntResource.Status.ProcessedItems = append(tntResource.Status.ProcessedItems, or) - } + return r.impersonation.LoadOrCreate(ctx, log, re, r.client.Scheme(), *sa) +} + +func (r *globalResourceController) impersonatedServiceAccount( + ctx context.Context, + log logr.Logger, + tntResource *capsulev1beta2.GlobalTenantResource, +) *meta.NamespacedRFC1123ObjectReferenceWithNamespace { + if sa := tntResource.Spec.ServiceAccount; sa != nil { + name := sa.Name.String() + ns := sa.Namespace.String() + + if name == "" || ns == "" { + log.V(4).Info("serviceAccount reference is set but incomplete; ignoring", + "name", name, "namespace", ns, + ) + + return nil + } + + return &meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: sa.Name, + Namespace: sa.Namespace, } } - tntResource.Status.SelectedTenants = tntSet.List() + cfg := r.configuration.ServiceAccountClientProperties() - log.V(4).Info("processing completed") + name := cfg.GlobalDefaultServiceAccount.String() + ns := cfg.GlobalDefaultServiceAccountNamespace.String() - return reconcile.Result{Requeue: true, RequeueAfter: tntResource.Spec.ResyncPeriod.Duration}, nil -} + nameSet := name != "" + nsSet := ns != "" -func (r *Global) reconcileDelete(ctx context.Context, tntResource *capsulev1beta2.GlobalTenantResource) (reconcile.Result, error) { - log := ctrllog.FromContext(ctx) + if nameSet != nsSet { + log.V(2).Info("invalid config: global default service account requires both name and namespace", + "name", name, "namespace", ns, + ) - if *tntResource.Spec.PruningOnDelete { - r.processor.HandlePruning(ctx, tntResource.Status.ProcessedItems.AsSet(), nil) - - controllerutil.RemoveFinalizer(tntResource, finalizer) + return nil } - log.V(4).Info("processing completed") + if !nameSet && !nsSet { + return nil + } - return reconcile.Result{Requeue: true, RequeueAfter: tntResource.Spec.ResyncPeriod.Duration}, nil + return &meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: cfg.GlobalDefaultServiceAccount, + Namespace: cfg.GlobalDefaultServiceAccountNamespace, + } +} + +func (r *globalResourceController) updateReconcilingStatus(ctx context.Context, instance *capsulev1beta2.GlobalTenantResource) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + latest := &capsulev1beta2.GlobalTenantResource{} + if err = r.reader.Get(ctx, types.NamespacedName{Name: instance.GetName()}, latest); err != nil { + return err + } + + latest.Status.ServiceAccount = instance.Status.ServiceAccount + + latest.Status.Conditions.UpdateConditionByType(meta.NewReadyConditionReconcilingReason(instance)) + + return r.client.Status().Update(ctx, latest) + }) +} + +func (r *globalResourceController) updateStatus(ctx context.Context, instance *capsulev1beta2.GlobalTenantResource, reconcileError error) error { + instance.Status.UpdateStats() + + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + latest := &capsulev1beta2.GlobalTenantResource{} + if err = r.reader.Get(ctx, types.NamespacedName{Name: instance.GetName()}, latest); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + latest.Status = instance.Status + + // Set Ready Condition + readyCondition := meta.NewReadyCondition(instance) + if reconcileError != nil { + readyCondition.Message = reconcileError.Error() + readyCondition.Status = metav1.ConditionFalse + readyCondition.Reason = meta.FailedReason + } + + latest.Status.Conditions.UpdateConditionByType(readyCondition) + + // Set Cordoned Condition + cordonedCondition := meta.NewCordonedCondition(instance) + + if *instance.Spec.Cordoned { + cordonedCondition.Reason = meta.CordonedReason + cordonedCondition.Message = "is cordoned" //nolint:goconst + cordonedCondition.Status = metav1.ConditionTrue + } + + latest.Status.Conditions.UpdateConditionByType(cordonedCondition) + + if err := r.client.Status().Update(ctx, latest); err != nil { + return err + } + + // Keep the in-memory object aligned with what we just wrote. + instance.Status = latest.Status + + return nil + }) } diff --git a/internal/controllers/resources/manager.go b/internal/controllers/resources/manager.go new file mode 100644 index 00000000..05973867 --- /dev/null +++ b/internal/controllers/resources/manager.go @@ -0,0 +1,46 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "fmt" + + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/projectcapsule/capsule/internal/cache" + "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/internal/metrics" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" +) + +func Add( + log logr.Logger, + mgr manager.Manager, + configuration configuration.Configuration, + opts utils.ControllerOptions, + cache *cache.ImpersonationCache, +) (err error) { + if err = (&globalResourceController{ + log: log.WithName("Global"), + configuration: configuration, + metrics: metrics.MustMakeGlobalTenantResourceRecorder(), + + impersonation: cache, + }).SetupWithManager(mgr, opts); err != nil { + return fmt.Errorf("unable to create global controller: %w", err) + } + + if err = (&namespacedResourceController{ + log: log.WithName("Namespaced"), + configuration: configuration, + metrics: metrics.MustMakeTenantResourceRecorder(), + + impersonation: cache, + }).SetupWithManager(mgr, opts); err != nil { + return fmt.Errorf("unable to create namespaced controller: %w", err) + } + + return nil +} diff --git a/internal/controllers/resources/namespaced.go b/internal/controllers/resources/namespaced.go index 6f87c73a..30bd4dda 100644 --- a/internal/controllers/resources/namespaced.go +++ b/internal/controllers/resources/namespaced.go @@ -5,50 +5,147 @@ package resources import ( "context" - "errors" + "fmt" + "reflect" + "strconv" + "github.com/go-logr/logr" gherrors "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/fields" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/retry" "sigs.k8s.io/cluster-api/util/patch" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" ctrllog "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/internal/cache" + cutils "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/internal/metrics" + caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/processor" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/predicates" + "github.com/projectcapsule/capsule/pkg/runtime/sanitize" + tpl "github.com/projectcapsule/capsule/pkg/template" + "github.com/projectcapsule/capsule/pkg/tenant" ) -type Namespaced struct { - client client.Client - processor Processor +type namespacedResourceController struct { + client client.Client + reader client.Reader + + log logr.Logger + processor processor.Processor + collector Collector + configuration configuration.Configuration + metrics *metrics.TenantResourceRecorder + + impersonation *cache.ImpersonationCache } -func (r *Namespaced) SetupWithManager(mgr ctrl.Manager, cfg utils.ControllerOptions) error { +func (r *namespacedResourceController) SetupWithManager(mgr ctrl.Manager, cfg cutils.ControllerOptions) error { r.client = mgr.GetClient() - r.processor = Processor{ - client: mgr.GetClient(), + r.reader = mgr.GetAPIReader() + + r.processor = processor.Processor{ + Configuration: r.configuration, + AllowCrossNamespaceSelection: false, + GatherClient: mgr.GetAPIReader(), + Mapper: mgr.GetRESTMapper(), } + r.collector = NewCollector( + mgr.GetAPIReader(), + mgr.GetRESTMapper(), + ) return ctrl.NewControllerManagedBy(mgr). - For(&capsulev1beta2.TenantResource{}). + For( + &capsulev1beta2.TenantResource{}, + builder.WithPredicates( + predicate.Or( + predicate.GenerationChangedPredicate{}, + predicates.ReconcileRequestedPredicate{}, + ), + ), + ). + Watches( + &capsulev1beta2.TenantResource{}, + handler.EnqueueRequestsFromMapFunc(r.enqueueDependentTenantResources), + ). + Watches( + &capsulev1beta2.CapsuleConfiguration{}, + handler.EnqueueRequestsFromMapFunc(r.enqueueAllResources), + builder.WithPredicates( + predicates.CapsuleConfigSpecImpersonationChangedPredicate{}, + predicates.NamesMatchingPredicate{Names: []string{cfg.ConfigurationName}}, + ), + ). + Watches( + &corev1.Namespace{}, + handler.EnqueueRequestsFromMapFunc(r.enqueueTenantResourcesForNamespace), + builder.WithPredicates( + predicates.LabelPresentPredicate{Label: meta.TenantLabel}, + ), + ). + Watches( + &capsulev1beta2.Tenant{}, + handler.EnqueueRequestsFromMapFunc(r.enqueueTenantResourcesForTenant), + builder.WithPredicates( + predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return true + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return true + }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldObj, okOld := e.ObjectOld.(*capsulev1beta2.Tenant) + + newObj, okNew := e.ObjectNew.(*capsulev1beta2.Tenant) + if !okOld || !okNew { + return false + } + + if !reflect.DeepEqual(oldObj.Status.Namespaces, newObj.Status.Namespaces) { + return true + } + + return false + }, + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + }, + ), + ). WithOptions(controller.Options{MaxConcurrentReconciles: cfg.MaxConcurrentReconciles}). Complete(r) } -func (r *Namespaced) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { +func (r *namespacedResourceController) Reconcile(ctx context.Context, request reconcile.Request) (res reconcile.Result, err error) { log := ctrllog.FromContext(ctx) - log.V(4).Info("start processing") - // Retrieving the TenantResource + log.V(5).Info("start processing") + tntResource := &capsulev1beta2.TenantResource{} if err := r.client.Get(ctx, request.NamespacedName, tntResource); err != nil { if apierrors.IsNotFound(err) { - log.V(3).Info("Request object not found, could have been deleted after reconcile request") + log.V(5).Info("Request object not found, could have been deleted after reconcile request") + + r.metrics.DeleteMetrics(request.Name, request.Namespace) return reconcile.Result{}, nil } @@ -56,111 +153,606 @@ func (r *Namespaced) Reconcile(ctx context.Context, request reconcile.Request) ( return reconcile.Result{}, err } + requeue := reconcile.Result{ + Requeue: true, + RequeueAfter: tntResource.Spec.ResyncPeriod.Duration, + } + patchHelper, err := patch.NewHelper(tntResource, r.client) if err != nil { return reconcile.Result{}, gherrors.Wrap(err, "failed to init patch helper") } + var statusErr error + + //nolint:dupl defer func() { - if e := patchHelper.Patch(ctx, tntResource); e != nil { - if err == nil { - err = gherrors.Wrap(e, "failed to patch TenantResource") - } + reconcileErr := err + if statusErr != nil { + reconcileErr = statusErr } + + if uerr := r.updateStatus(ctx, tntResource, reconcileErr); uerr != nil { + if caperrors.IgnoreGone(uerr) { + err = nil + + return + } + + err = fmt.Errorf("cannot update tenantresource status: %w", uerr) + + return + } + + r.metrics.RecordConditions(tntResource) + + if e := patchHelper.Patch(ctx, tntResource); e != nil { + if caperrors.IgnoreGone(e) { + err = nil + + return + } + + res = reconcile.Result{} + err = gherrors.Wrap(e, "failed to patch TenantResource") + + return + } + + // Controller-runtime should not receive handled reconciliation errors. + err = nil }() - // Handle deleted TenantResource - if !tntResource.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, tntResource) + // On Deletion these checks are skipped. + //nolint:nestif + if tntResource.DeletionTimestamp.IsZero() { + if tntResource.Spec.Cordoned != nil && *tntResource.Spec.Cordoned { + log.V(5).Info("tenant resource cordoned") + + return reconcile.Result{}, nil + } + + for _, dep := range tntResource.Spec.DependsOn { + d := &capsulev1beta2.TenantResource{} + + if getErr := r.client.Get(ctx, types.NamespacedName{ + Name: dep.Name.String(), + Namespace: tntResource.GetNamespace(), + }, d); getErr != nil { + if apierrors.IsNotFound(getErr) { + statusErr = fmt.Errorf("dependency %s not found", dep.Name) + } else { + statusErr = getErr + } + + return requeue, nil + } + + stat := d.Status.Conditions.GetConditionByType(meta.ReadyCondition) + if stat == nil || stat.Status != metav1.ConditionTrue { + statusErr = fmt.Errorf("dependency %s not ready", dep.Name) + + return requeue, nil + } + } } - // Handle non-deleted TenantResource - return r.reconcileNormal(ctx, tntResource) + // Load client must be first since it updates the new serviceaccount used which can then be directly + // posted to the status. + c, loadErr := r.loadClient(ctx, log, tntResource) + if loadErr != nil { + statusErr = gherrors.Wrap(loadErr, "failed to load serviceaccount client") + + return requeue, nil + } + + if updateErr := r.updateReconcilingStatus(ctx, tntResource); updateErr != nil { + if caperrors.IgnoreGone(updateErr) { + return reconcile.Result{}, nil + } + + statusErr = gherrors.Wrap(updateErr, "failed to update status") + + return requeue, nil + } + + if c == nil { + statusErr = fmt.Errorf("received empty client for serviceaccount") + + return requeue, nil + } + + statusErr = r.reconcile(ctx, c, tntResource) + + if len(tntResource.Status.ProcessedItems) > 0 { + controllerutil.AddFinalizer(tntResource, meta.ControllerFinalizer) + } else { + controllerutil.RemoveFinalizer(tntResource, meta.ControllerFinalizer) + } + + controllerutil.RemoveFinalizer(tntResource, meta.LegacyResourceFinalizer) + + return requeue, nil } -func (r *Namespaced) reconcileNormal(ctx context.Context, tntResource *capsulev1beta2.TenantResource) (reconcile.Result, error) { - log := ctrllog.FromContext(ctx) - - if *tntResource.Spec.PruningOnDelete { - controllerutil.AddFinalizer(tntResource, finalizer) +func (r *namespacedResourceController) enqueueTenantResourcesForTenant(ctx context.Context, obj client.Object) []reconcile.Request { + tnt, ok := obj.(*capsulev1beta2.Tenant) + if !ok { + return nil } + seen := map[types.NamespacedName]struct{}{} + out := make([]reconcile.Request, 0) + + for _, ns := range tnt.Status.Namespaces { + list := &capsulev1beta2.TenantResourceList{} + if err := r.client.List(ctx, list, client.InNamespace(ns)); err != nil { + continue + } + + for i := range list.Items { + key := types.NamespacedName{ + Name: list.Items[i].Name, + Namespace: list.Items[i].Namespace, + } + if _, exists := seen[key]; exists { + continue + } + + seen[key] = struct{}{} + + out = append(out, reconcile.Request{NamespacedName: key}) + } + } + + return out +} + +//nolint:dupl +func (r *namespacedResourceController) enqueueDependentTenantResources( + ctx context.Context, + obj client.Object, +) []ctrl.Request { + changed, ok := obj.(*capsulev1beta2.TenantResource) + if !ok { + return nil + } + + var list capsulev1beta2.TenantResourceList + if err := r.client.List(ctx, &list); err != nil { + return nil + } + + reqs := make([]ctrl.Request, 0) + + for _, gtr := range list.Items { + for _, dep := range gtr.Spec.DependsOn { + if dep.Name.String() == changed.Name { + reqs = append(reqs, ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: gtr.Name, + Namespace: gtr.Namespace, + }, + }) + + break + } + } + } + + return reqs +} + +// Requeue TenantResources if there is changes to namespaces of the same tenant +// We are not relying on the tenant status, as we might have a terminating lock caused by TenantResources. +func (r *namespacedResourceController) enqueueTenantResourcesForNamespace( + ctx context.Context, + obj client.Object, +) []reconcile.Request { + ns, ok := obj.(*corev1.Namespace) + if !ok { + return nil + } + + labelValue, ok := ns.Labels[meta.TenantLabel] + if !ok || labelValue == "" { + return nil + } + + var namespaces corev1.NamespaceList + if err := r.client.List( + ctx, + &namespaces, + client.MatchingLabels{meta.TenantLabel: labelValue}, + ); err != nil { + r.log.Error(err, "failed to list namespaces by label", "label", meta.TenantLabel, "value", labelValue) + + return nil + } + + requests := make([]reconcile.Request, 0, 16) + seen := make(map[types.NamespacedName]struct{}) + + for i := range namespaces.Items { + var trList capsulev1beta2.TenantResourceList + if err := r.client.List( + ctx, + &trList, + client.InNamespace(namespaces.Items[i].Name), + ); err != nil { + r.log.Error(err, "failed to list TenantResources", "namespace", namespaces.Items[i].Name) + + continue + } + + for j := range trList.Items { + key := types.NamespacedName{ + Namespace: trList.Items[j].Namespace, + Name: trList.Items[j].Name, + } + + if _, exists := seen[key]; exists { + continue + } + + seen[key] = struct{}{} + + requests = append(requests, reconcile.Request{NamespacedName: key}) + } + } + + return requests +} + +//nolint:dupl +func (r *namespacedResourceController) enqueueAllResources(ctx context.Context, _ client.Object) []reconcile.Request { + var list capsulev1beta2.TenantResourceList + if err := r.client.List(ctx, &list); err != nil { + r.log.V(1).Error(err, "unable to list TenantResources for config-triggered reconcile") + + return nil + } + + reqs := make([]reconcile.Request, 0, len(list.Items)) + for i := range list.Items { + reqs = append(reqs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: list.Items[i].Name, + Namespace: list.Items[i].Namespace, + }, + }) + } + + return reqs +} + +func (r *namespacedResourceController) reconcile( + ctx context.Context, + c client.Client, + tntResource *capsulev1beta2.TenantResource, +) error { + log := ctrllog.FromContext(ctx) + // Adding the default value for the status if tntResource.Status.ProcessedItems == nil { - tntResource.Status.ProcessedItems = make([]capsulev1beta2.ObjectReferenceStatus, 0) + tntResource.Status.ProcessedItems = make([]meta.ObjectReferenceStatus, 0) } // Retrieving the parent of the Tenant Resource: // can be owned, or being deployed in one of its Namespace. - tl := &capsulev1beta2.TenantList{} - if err := r.client.List(ctx, tl, client.MatchingFieldsSelector{Selector: fields.OneTermEqualSelector(".status.namespaces", tntResource.GetNamespace())}); err != nil { - log.Error(err, "unable to detect the Tenant for the given TenantResource") - - return reconcile.Result{}, err + // we cant resolve via status.namespaces, as when a namespace is deleted it is no longer references by the tenant + // causing a deletion blockade. + ns := &corev1.Namespace{} + if err := r.client.Get(ctx, types.NamespacedName{Name: tntResource.GetNamespace()}, ns); err != nil { + return err } - if len(tl.Items) == 0 { - log.V(4).Info("skipping sync, the current Namespace is not belonging to any Global") - - return reconcile.Result{}, nil + tnt, err := tenant.GetTenantByOwnerreferences(ctx, r.client, ns.GetOwnerReferences()) + if err != nil { + return err } - // A TenantResource is made of several Resource sections, each one with specific options: - // the Status can be updated only in case of no errors across all of them to guarantee a valid and coherent status. - processedItems := sets.NewString() + if tnt == nil { + log.Info("skipping sync, the current Namespace is not belonging to any Tenant") - tenantLabel, labelErr := capsulev1beta2.GetTypeLabel(&capsulev1beta2.Tenant{}) - if labelErr != nil { - log.Error(labelErr, "expected label for selection") - - return reconcile.Result{}, labelErr + return nil } - // new empty error - var err error + acc := processor.Accumulator{} - for index, resource := range tntResource.Spec.Resources { - items, sectionErr := r.processor.HandleSection(ctx, tl.Items[0], false, tenantLabel, index, resource) - if sectionErr != nil { - // Upon a process error storing the last error occurred and continuing to iterate, - // avoid to block the whole processing. - err = errors.Join(err, sectionErr) - } else { - processedItems.Insert(items...) + // Gather Resources + if tntResource.DeletionTimestamp.IsZero() { + err := r.gatherResources( + ctx, + c, + log, + tntResource, + *tnt, + acc, + ) + if err != nil { + return err } } - if err != nil { - log.Error(err, "unable to replicate the requested resources") + return r.processor.Reconcile( + ctx, + log, + c, + &tntResource.Status.ProcessedItems, + acc, + processor.ProcessorOptions{ + FieldOwnerPrefix: getFieldOwner(tntResource.GetName(), tntResource.GetNamespace()), + Prune: *tntResource.Spec.PruningOnDelete, + Adopt: *tntResource.Spec.Settings.Adopt, + Force: *tntResource.Spec.Settings.Force, + Owner: nil, + }) +} - return reconcile.Result{}, err +func (r *namespacedResourceController) gatherResources( + ctx context.Context, + c client.Client, + log logr.Logger, + tntResource *capsulev1beta2.TenantResource, + tnt capsulev1beta2.Tenant, + acc processor.Accumulator, +) (err error) { + opts := CollectorOptions{ + Accumulator: acc, + AllowCrossNamespaceSelection: false, + ValidatorNamespaces: tpl.NewNamespaceValidator(false, sets.New[string](tnt.Status.Namespaces...)), } - if r.processor.HandlePruning(ctx, tntResource.Status.ProcessedItems.AsSet(), sets.Set[string](processedItems)) { - tntResource.Status.ProcessedItems = make([]capsulev1beta2.ObjectReferenceStatus, 0, len(processedItems)) + for resourceIndex, resource := range tntResource.Spec.Resources { + objs, err := r.collector.CollectNamespacedItems(ctx, c, opts, resource, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: tntResource.GetNamespace()}}, tnt) + if err != nil { + return err + } - for _, item := range processedItems.List() { - if or := (capsulev1beta2.ObjectReferenceStatus{}); or.ParseFromString(item) == nil { - tntResource.Status.ProcessedItems = append(tntResource.Status.ProcessedItems, or) + for g := range objs { + log.V(5).Info("found replication source object", "name", g.Name, "namespace", g.Namespace, "kind", g.Kind) + } + + namespaces, err := r.collector.selectedTenantNamespaces(ctx, log, tnt, resource) + if err != nil { + return err + } + + i := 0 + + for _, innerNs := range namespaces { + opts.Iterator = NewCollectorIteratorOptions(&tnt, innerNs, resource) + + for _, obj := range objs { + if obj.GetNamespace() == innerNs.GetName() { + continue + } + + target := obj.DeepCopy() + if err := sanitize.SanitizeObject(target, c.Scheme(), r.collector.objectSanitizeOptions); err != nil { + return err + } + + target.SetNamespace(innerNs.GetName()) + + log.V(4).Info("adding replication for namespaced item", "name", target.GetName(), "namespace", target.GetNamespace(), "kind", target.GetKind()) + + err = r.collector.AddToAccumulation(&tnt, innerNs, opts, resource, target, "replica", false) + if err != nil { + return err + } + } + + err = r.collector.Collect( + ctx, + c, + opts, + &tnt, + strconv.Itoa((resourceIndex)), + resource, + innerNs, + ) + if err != nil { + return err + } + + i++ + } + } + + return nil +} + +//nolint:dupl +func (r *namespacedResourceController) loadClient( + ctx context.Context, + log logr.Logger, + tntResource *capsulev1beta2.TenantResource, +) (client.Client, error) { + sa := r.impersonatedServiceAccount(ctx, log, tntResource) + if sa == nil { + sa, ns := configuration.ControllerServiceAccount() + + tntResource.Status.ServiceAccount = &meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: meta.RFC1123Name(sa), + Namespace: meta.RFC1123SubdomainName(ns), + } + + return r.client, nil + } + + tntResource.Status.ServiceAccount = &meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: sa.Name, + Namespace: sa.Namespace, + } + + re, err := r.configuration.ServiceAccountClient(ctx) + if err != nil { + log.Error(err, "failed to load impersonated rest client") + + return nil, err + } + + log.V(5).Info("using impersonation client", "serviceaccount", sa.Name, "namespace", sa.Namespace) + + return r.impersonation.LoadOrCreate(ctx, log, re, r.client.Scheme(), *sa) +} + +func (r *namespacedResourceController) impersonatedServiceAccount( + ctx context.Context, + log logr.Logger, + tntResource *capsulev1beta2.TenantResource, +) *meta.NamespacedRFC1123ObjectReferenceWithNamespace { + if tntResource.Spec.ServiceAccount != nil { + return &meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: tntResource.Spec.ServiceAccount.Name, + Namespace: meta.RFC1123SubdomainName(tntResource.Namespace), + } + } + + cfg := r.configuration.ServiceAccountClientProperties() + + if cfg.TenantDefaultServiceAccount == "" { + return nil + } + + return &meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: cfg.TenantDefaultServiceAccount, + Namespace: meta.RFC1123SubdomainName(tntResource.Namespace), + } +} + +func (r *namespacedResourceController) updateReconcilingStatus(ctx context.Context, instance *capsulev1beta2.TenantResource) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + latest := &capsulev1beta2.TenantResource{} + if err = r.reader.Get(ctx, types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}, latest); err != nil { + return err + } + + latest.Status.ServiceAccount = instance.Status.ServiceAccount + + latest.Status.Conditions.UpdateConditionByType(meta.NewReadyConditionReconcilingReason(instance)) + + return r.client.Status().Update(ctx, latest) + }) +} + +func (r *namespacedResourceController) updateStatus(ctx context.Context, instance *capsulev1beta2.TenantResource, reconcileError error) error { + instance.Status.UpdateStats() + + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + latest := &capsulev1beta2.TenantResource{} + if err = r.reader.Get(ctx, types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}, latest); err != nil { + return err + } + + latest.Status = instance.Status + + // Set Ready Condition + readyCondition := meta.NewReadyCondition(instance) + if reconcileError != nil { + readyCondition.Message = reconcileError.Error() + readyCondition.Status = metav1.ConditionFalse + readyCondition.Reason = meta.FailedReason + } + + latest.Status.Conditions.UpdateConditionByType(readyCondition) + + // Set Cordoned Condition + cordonedCondition := meta.NewCordonedCondition(instance) + + if *instance.Spec.Cordoned { + cordonedCondition.Reason = meta.CordonedReason + cordonedCondition.Message = "is cordoned" + cordonedCondition.Status = metav1.ConditionTrue + } + + latest.Status.Conditions.UpdateConditionByType(cordonedCondition) + + if err := r.client.Status().Update(ctx, latest); err != nil { + return err + } + + // Keep the in-memory object aligned with what we just wrote. + instance.Status = latest.Status + + return nil + }) +} + +func ForeachNamespace( + ctx context.Context, + controllerClient client.Client, + resourceClient client.Client, + collector Collector, + opts CollectorOptions, + log logr.Logger, + resource capsulev1beta2.ResourceSpec, + resourceIndex int, + tnt capsulev1beta2.Tenant, + acc processor.Accumulator, +) (err error) { + namespaces, err := tenant.CollectTenantNamespaceByLabel(ctx, controllerClient, tnt, resource.NamespaceSelector) + if err != nil { + return err + } + + for _, ns := range namespaces { + if ns.DeletionTimestamp != nil { + terminating, err := tenant.NamespaceIsPendingUnmanagedTerminationByStatus(ctx, controllerClient, &ns) + if err != nil { + return err + } + + // Skip this namespace so resources are cleaned + if terminating { + continue } } + + opts.Iterator = NewCollectorIteratorOptions(&tnt, &ns, resource) + + objs, err := collector.CollectNamespacedItems(ctx, resourceClient, opts, resource, &ns, tnt) + if err != nil { + return err + } + + i := 0 + + for _, obj := range objs { + for _, innerNs := range namespaces { + if obj.GetNamespace() == innerNs.GetName() { + continue + } + + target := obj.DeepCopy() + target.SetNamespace(innerNs.GetName()) + + log.V(4).Info("adding replication for namespaced item", "name", target.GetName(), "namespace", target.GetNamespace(), "kind", target.GetKind()) + + err = collector.AddToAccumulation(&tnt, &innerNs, opts, resource, target, strconv.Itoa(resourceIndex)+"/replica-"+strconv.Itoa(i), true) + if err != nil { + return err + } + } + + i++ + } + + err = collector.Collect( + ctx, + resourceClient, + opts, + &tnt, + strconv.Itoa((resourceIndex)), + resource, + &ns, + ) + if err != nil { + return err + } } - log.V(4).Info("processing completed") - - return reconcile.Result{Requeue: true, RequeueAfter: tntResource.Spec.ResyncPeriod.Duration}, nil -} - -func (r *Namespaced) reconcileDelete(ctx context.Context, tntResource *capsulev1beta2.TenantResource) (reconcile.Result, error) { - log := ctrllog.FromContext(ctx) - - if *tntResource.Spec.PruningOnDelete { - r.processor.HandlePruning(ctx, tntResource.Status.ProcessedItems.AsSet(), nil) - } - - controllerutil.RemoveFinalizer(tntResource, finalizer) - - log.V(4).Info("processing completed") - - return reconcile.Result{Requeue: true, RequeueAfter: tntResource.Spec.ResyncPeriod.Duration}, nil + return nil } diff --git a/internal/controllers/resources/processor.go b/internal/controllers/resources/processor.go deleted file mode 100644 index cc96d3d6..00000000 --- a/internal/controllers/resources/processor.go +++ /dev/null @@ -1,320 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package resources - -import ( - "context" - "errors" - "fmt" - "maps" - "sync" - - corev1 "k8s.io/api/core/v1" - apierr "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apimachinery/pkg/selection" - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - ctrllog "sigs.k8s.io/controller-runtime/pkg/log" - - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - tpl "github.com/projectcapsule/capsule/pkg/template" - "github.com/projectcapsule/capsule/pkg/tenant" -) - -const ( - Label = "capsule.clastix.io/resources" - finalizer = "capsule.clastix.io/resources" -) - -type Processor struct { - client client.Client -} - -func prepareAdditionalMetadata(m map[string]string) map[string]string { - if m == nil { - return make(map[string]string) - } - - // clone without mutating the original - return maps.Clone(m) -} - -func (r *Processor) HandlePruning(ctx context.Context, current, desired sets.Set[string]) (updateStatus bool) { - log := ctrllog.FromContext(ctx) - - diff := current.Difference(desired) - // We don't want to trigger a reconciliation of the Status every time, - // rather, only in case of a difference between the processed and the actual status. - // This can happen upon the first reconciliation, or a removal, or a change, of a resource. - updateStatus = diff.Len() > 0 || current.Len() != desired.Len() - - if diff.Len() > 0 { - log.Info("starting processing pruning", "length", diff.Len()) - } - - // The outer resources must be removed, iterating over these to clean-up - for item := range diff { - or := capsulev1beta2.ObjectReferenceStatus{} - if err := or.ParseFromString(item); err != nil { - log.Error(err, "unable to parse resource to prune", "resource", item) - - continue - } - - obj := unstructured.Unstructured{} - obj.SetNamespace(or.Namespace) - obj.SetName(or.Name) - obj.SetGroupVersionKind(schema.FromAPIVersionAndKind(or.APIVersion, or.Kind)) - - if err := r.client.Delete(ctx, &obj); err != nil { - if apierr.IsNotFound(err) { - // Object may have been already deleted, we can ignore this error - continue - } - - log.Error(err, "unable to prune resource", "resource", item) - - continue - } - - log.Info("resource has been pruned", "resource", item) - } - - return updateStatus -} - -//nolint:gocognit -func (r *Processor) HandleSection(ctx context.Context, tnt capsulev1beta2.Tenant, allowCrossNamespaceSelection bool, tenantLabel string, resourceIndex int, spec capsulev1beta2.ResourceSpec) ([]string, error) { - log := ctrllog.FromContext(ctx) - - var err error - // Creating Namespace selector - var selector labels.Selector - - if spec.NamespaceSelector != nil { - selector, err = metav1.LabelSelectorAsSelector(spec.NamespaceSelector) - if err != nil { - log.Error(err, "cannot create Namespace selector for Namespace filtering and resource replication", "index", resourceIndex) - - return nil, err - } - } else { - selector = labels.NewSelector() - } - // Resources can be replicated only on Namespaces belonging to the same Global: - // preventing a boundary cross by enforcing the selection. - tntRequirement, err := labels.NewRequirement(tenantLabel, selection.Equals, []string{tnt.GetName()}) - if err != nil { - log.Error(err, "unable to create requirement for Namespace filtering and resource replication", "index", resourceIndex) - - return nil, err - } - - selector = selector.Add(*tntRequirement) - // Selecting the targeted Namespace according to the TenantResource specification. - namespaces := corev1.NamespaceList{} - if err = r.client.List(ctx, &namespaces, client.MatchingLabelsSelector{Selector: selector}); err != nil { - log.Error(err, "cannot retrieve Namespaces for resource", "index", resourceIndex) - - return nil, err - } - // Generating additional metadata - objAnnotations, objLabels := map[string]string{}, map[string]string{} - - if spec.AdditionalMetadata != nil { - objAnnotations = prepareAdditionalMetadata(spec.AdditionalMetadata.Annotations) - objLabels = prepareAdditionalMetadata(spec.AdditionalMetadata.Labels) - } - - objAnnotations[tenantLabel] = tnt.GetName() - - objLabels[Label] = fmt.Sprintf("%d", resourceIndex) - objLabels[tenantLabel] = tnt.GetName() - // processed will contain the sets of resources replicated, both for the raw and the Namespaced ones: - // these are required to perform a final pruning once the replication has been occurred. - processed := sets.NewString() - - tntNamespaces := sets.NewString(tnt.Status.Namespaces...) - - var syncErr error - - codecFactory := serializer.NewCodecFactory(r.client.Scheme()) - - for _, ns := range namespaces.Items { - for nsIndex, item := range spec.NamespacedItems { - keysAndValues := []any{"index", nsIndex, "namespace", item.Namespace} - // A TenantResource is created by a TenantOwner, and potentially, they could point to a resource in a non-owned - // Namespace: this must be blocked by checking it this is the case. - if !allowCrossNamespaceSelection && !tntNamespaces.Has(item.Namespace) { - log.Info("skipping processing of namespacedItem, referring a Namespace that is not part of the given Tenant", keysAndValues...) - - continue - } - // Namespaced Items are relying on selecting resources, rather than specifying a specific name: - // creating it to get used by the client List action. - objSelector := item.Selector - - itemSelector, selectorErr := metav1.LabelSelectorAsSelector(&objSelector) - if selectorErr != nil { - log.Error(selectorErr, "cannot create Selector for namespacedItem", keysAndValues...) - - syncErr = errors.Join(syncErr, selectorErr) - - continue - } - - objs := unstructured.UnstructuredList{} - objs.SetGroupVersionKind(schema.FromAPIVersionAndKind(item.APIVersion, fmt.Sprintf("%sList", item.Kind))) - - if clientErr := r.client.List(ctx, &objs, client.InNamespace(item.Namespace), client.MatchingLabelsSelector{Selector: itemSelector}); clientErr != nil { - log.Error(clientErr, "cannot retrieve object for namespacedItem", keysAndValues...) - - syncErr = errors.Join(syncErr, clientErr) - - continue - } - - var wg sync.WaitGroup - - errorsChan := make(chan error, len(objs.Items)) - // processedRaw is used to avoid concurrent map writes during iteration of namespaced items: - // the objects will be then added to processed variable if the resulting string is not empty, - // meaning it has been processed correctly. - processedRaw := make([]string, len(objs.Items)) - // Iterating over all the retrieved objects from the resource spec to get replicated in all the selected Namespaces: - // in case of error during the create or update function, this will be appended to the list of errors. - for i, o := range objs.Items { - obj := o - obj.SetNamespace(ns.Name) - obj.SetOwnerReferences(nil) - - wg.Add(1) - - go func(index int, obj unstructured.Unstructured) { - defer wg.Done() - - kv := keysAndValues - kv = append(kv, "resource", fmt.Sprintf("%s/%s", obj.GetNamespace(), obj.GetNamespace())) - - if opErr := r.createOrUpdate(ctx, &obj, objLabels, objAnnotations); opErr != nil { - log.Error(opErr, "unable to sync namespacedItems", kv...) - - errorsChan <- opErr - - return - } - - log.Info("resource has been replicated", kv...) - - replicatedItem := &capsulev1beta2.ObjectReferenceStatus{} - replicatedItem.Name = obj.GetName() - replicatedItem.Kind = obj.GetKind() - replicatedItem.Namespace = ns.Name - replicatedItem.APIVersion = obj.GetAPIVersion() - - processedRaw[index] = replicatedItem.String() - }(i, obj) - } - - wg.Wait() - close(errorsChan) - - for err := range errorsChan { - if err != nil { - syncErr = errors.Join(syncErr, err) - } - } - - for _, p := range processedRaw { - if p == "" { - continue - } - - processed.Insert(p) - } - } - - for rawIndex, item := range spec.RawItems { - template := string(item.Raw) - - fastContext := tenant.ContextForTenantAndNamespace(&tnt, &ns) - - tmplString := tpl.FastTemplate(template, fastContext) - - obj, keysAndValues := unstructured.Unstructured{}, []any{"index", rawIndex} - - if _, _, decodeErr := codecFactory.UniversalDeserializer().Decode([]byte(tmplString), nil, &obj); decodeErr != nil { - log.Error(decodeErr, "unable to deserialize rawItem", keysAndValues...) - - syncErr = errors.Join(syncErr, decodeErr) - - continue - } - - obj.SetNamespace(ns.Name) - - if rawErr := r.createOrUpdate(ctx, &obj, objLabels, objAnnotations); rawErr != nil { - log.Info("unable to sync rawItem", keysAndValues...) - // In case of error processing an item in one of any selected Namespaces, storing it to report it lately - // to the upper call to ensure a partial sync that will be fixed by a subsequent reconciliation. - syncErr = errors.Join(syncErr, rawErr) - } else { - log.Info("resource has been replicated", keysAndValues...) - - replicatedItem := &capsulev1beta2.ObjectReferenceStatus{} - replicatedItem.Name = obj.GetName() - replicatedItem.Kind = obj.GetKind() - replicatedItem.Namespace = ns.Name - replicatedItem.APIVersion = obj.GetAPIVersion() - - processed.Insert(replicatedItem.String()) - } - } - } - - return processed.List(), syncErr -} - -// createOrUpdate replicates the provided unstructured object to all the provided Namespaces: -// this function mimics the CreateOrUpdate, by retrieving the object to understand if it must be created or updated, -// along adding the additional metadata, if required. -func (r *Processor) createOrUpdate(ctx context.Context, obj *unstructured.Unstructured, labels map[string]string, annotations map[string]string) (err error) { - actual, desired := &unstructured.Unstructured{}, obj.DeepCopy() - - actual.SetAPIVersion(desired.GetAPIVersion()) - actual.SetKind(desired.GetKind()) - actual.SetNamespace(desired.GetNamespace()) - actual.SetName(desired.GetName()) - - _, err = controllerutil.CreateOrUpdate(ctx, r.client, actual, func() error { - UID := actual.GetUID() - rv := actual.GetResourceVersion() - actual.SetUnstructuredContent(desired.Object) - - combinedLabels := map[string]string{} - maps.Copy(combinedLabels, obj.GetLabels()) - maps.Copy(combinedLabels, labels) - - actual.SetLabels(combinedLabels) - - combinedAnnotations := map[string]string{} - maps.Copy(combinedAnnotations, obj.GetAnnotations()) - maps.Copy(combinedAnnotations, annotations) - - actual.SetAnnotations(combinedAnnotations) - - actual.SetResourceVersion(rv) - actual.SetUID(UID) - - return nil - }) - - return err -} diff --git a/internal/controllers/resources/utils.go b/internal/controllers/resources/utils.go new file mode 100644 index 00000000..115de5bf --- /dev/null +++ b/internal/controllers/resources/utils.go @@ -0,0 +1,44 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "hash/fnv" + "strconv" + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +func getFieldOwner(name string, namespace string) string { + if namespace == "" { + namespace = "Cluster" + } + + h := fnv.New64a() + _, _ = h.Write([]byte(namespace)) + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(name)) + + return strconv.FormatUint(h.Sum64(), 36) +} + +func getSelectorForCreatedResourcesExclusion() (labels.Selector, error) { + selector := labels.NewSelector() + + req, err := labels.NewRequirement( + meta.CreatedByCapsuleLabel, + selection.NotIn, + []string{meta.ValueControllerResources}, + ) + if err != nil { + return nil, err + } + + selector.Add(*req) + + return selector, nil +} diff --git a/internal/controllers/rulestatus/manager.go b/internal/controllers/rulestatus/manager.go new file mode 100644 index 00000000..0a13962a --- /dev/null +++ b/internal/controllers/rulestatus/manager.go @@ -0,0 +1,185 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package rulestatus + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/events" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/cluster-api/util/patch" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/controllers/utils" + "github.com/projectcapsule/capsule/internal/metrics" + "github.com/projectcapsule/capsule/pkg/api" + meta "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/predicates" +) + +type Manager struct { + client.Client + + reader client.Reader + + Metrics *metrics.TenantRecorder + Log logr.Logger + Recorder events.EventRecorder + Configuration configuration.Configuration + RESTConfig *rest.Config +} + +func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.ControllerOptions) error { + r.reader = mgr.GetAPIReader() + + ctrlBuilder := ctrl.NewControllerManagedBy(mgr). + Named("capsule/rule-status"). + For( + &capsulev1beta2.RuleStatus{}, + builder.WithPredicates( + predicate.Or( + predicate.GenerationChangedPredicate{}, + predicates.UpdatedMetadataPredicate{}, + ), + ), + ). + WithOptions(controller.Options{MaxConcurrentReconciles: ctrlConfig.MaxConcurrentReconciles}) + + return ctrlBuilder.Complete(r) +} + +func (r Manager) Reconcile(ctx context.Context, request ctrl.Request) (result ctrl.Result, err error) { + r.Log = r.Log.WithValues("Request.Name", request.Name) + + instance := &capsulev1beta2.RuleStatus{} + if err = r.Get(ctx, request.NamespacedName, instance); err != nil { + if apierrors.IsNotFound(err) { + r.Log.V(5).Info("request object not found, could have been deleted after reconcile request") + + return reconcile.Result{}, nil + } + + r.Log.Error(err, "error reading the object") + + return result, err + } + + patchHelper, err := patch.NewHelper(instance, r.Client) + if err != nil { + return reconcile.Result{}, err + } + + defer func() { + if e := r.updateStatus(ctx, instance, err); e != nil { + if apierrors.IsNotFound(err) || apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) { + err = nil + + return + } + + err = fmt.Errorf("cannot update status: %w", e) + + return + } + + if e := patchHelper.Patch(ctx, instance); err != nil { + if apierrors.IsNotFound(e) || apierrors.HasStatusCause(e, corev1.NamespaceTerminatingCause) { + err = nil + + return + } + + err = fmt.Errorf("cannot patch: %w", e) + + return + } + + // Controller-Runtime should never receive error + err = nil + }() + + // Reconcile + if err = r.reconcile(ctx, instance); err != nil { + err = fmt.Errorf("cannot collect available resources: %w", err) + + return result, err + } + + var reconcileError error + if err != nil { + reconcileError = fmt.Errorf("had errors reconciling") + } + + r.Log.V(4).Info("reconciling completed") + + return ctrl.Result{}, reconcileError +} + +func (r Manager) reconcile(ctx context.Context, instance *capsulev1beta2.RuleStatus) (err error) { + out := api.NamespaceRuleBodyNamespace{} + + for _, rule := range instance.Spec { + if rule == nil { + continue + } + + // Merge enforce body (for now: only registries) + // Preserve order: append in the order rules are declared. + if len(rule.Enforce.Registries) > 0 { + out.Enforce.Registries = append(out.Enforce.Registries, rule.Enforce.Registries...) + } + } + + instance.Status.Rule = out + + return nil +} + +func (r *Manager) updateStatus(ctx context.Context, instance *capsulev1beta2.RuleStatus, reconcileError error) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + latest := &capsulev1beta2.RuleStatus{} + if err = r.reader.Get(ctx, types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}, latest); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + latest.Status = instance.Status + + // Set Ready Condition + readyCondition := meta.NewReadyCondition(instance) + if reconcileError != nil { + readyCondition.Message = reconcileError.Error() + readyCondition.Status = metav1.ConditionFalse + readyCondition.Reason = meta.FailedReason + } + + latest.Status.Conditions.UpdateConditionByType(readyCondition) + + if err := r.Client.Status().Update(ctx, latest); err != nil { + return err + } + + // Keep the in-memory object aligned with what we just wrote. + instance.Status = latest.Status + + return nil + }) +} diff --git a/internal/controllers/servicelabels/endpoint_slices.go b/internal/controllers/servicelabels/endpoint_slices.go index a64cf22f..7fff40bd 100644 --- a/internal/controllers/servicelabels/endpoint_slices.go +++ b/internal/controllers/servicelabels/endpoint_slices.go @@ -27,6 +27,7 @@ func (r *EndpointSlicesLabelsReconciler) SetupWithManager(ctx context.Context, m } return ctrl.NewControllerManagedBy(mgr). + Named("endpointslices"). For(r.abstractServiceLabelsReconciler.obj, r.abstractServiceLabelsReconciler.forOptionPerInstanceName(ctx)). Named("capsule/endpointslices"). Complete(r) diff --git a/internal/controllers/servicelabels/service.go b/internal/controllers/servicelabels/service.go index 745b31b4..78948fa9 100644 --- a/internal/controllers/servicelabels/service.go +++ b/internal/controllers/servicelabels/service.go @@ -25,6 +25,7 @@ func (r *ServicesLabelsReconciler) SetupWithManager(ctx context.Context, mgr ctr } return ctrl.NewControllerManagedBy(mgr). + Named("service"). For(r.abstractServiceLabelsReconciler.obj, r.abstractServiceLabelsReconciler.forOptionPerInstanceName(ctx)). Named("capsule/services"). Complete(r) diff --git a/internal/controllers/tenant/limitranges.go b/internal/controllers/tenant/limitranges.go index cf07acee..92b1a09c 100644 --- a/internal/controllers/tenant/limitranges.go +++ b/internal/controllers/tenant/limitranges.go @@ -1,6 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 +//nolint:dupl package tenant import ( @@ -8,8 +9,8 @@ import ( "fmt" "strconv" - "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -19,7 +20,7 @@ import ( // Ensuring all the LimitRange are applied to each Namespace handled by the Tenant. // -//nolint:dupl + func (r *Manager) syncLimitRanges(ctx context.Context, tenant *capsulev1beta2.Tenant) error { // getting requested LimitRange keys keys := make([]string, 0, len(tenant.Spec.LimitRanges.Items)) //nolint:staticcheck @@ -29,17 +30,9 @@ func (r *Manager) syncLimitRanges(ctx context.Context, tenant *capsulev1beta2.Te keys = append(keys, strconv.Itoa(i)) } - group := new(errgroup.Group) - - for _, ns := range tenant.Status.Namespaces { - namespace := ns - - group.Go(func() error { - return r.syncLimitRange(ctx, tenant, namespace, keys) - }) - } - - return group.Wait() + return runForTenantNamespaces(ctx, tenant, func(ctx context.Context, namespace string) error { + return r.syncLimitRange(ctx, tenant, namespace, keys) + }) } func (r *Manager) syncLimitRange(ctx context.Context, tenant *capsulev1beta2.Tenant, namespace string, keys []string) (err error) { @@ -69,19 +62,29 @@ func (r *Manager) syncLimitRange(ctx context.Context, tenant *capsulev1beta2.Ten labels[meta.LimitRangeLabel] = strconv.Itoa(i) // Remove Legacy labels - delete(target.Labels, meta.TenantLabel) + delete(labels, meta.TenantLabel) target.SetLabels(labels) target.Spec = spec return controllerutil.SetControllerReference(tenant, target, r.Scheme()) }) - - r.Log.V(4).Info("LimitRange sync result: "+string(res), "name", target.Name, "namespace", target.Namespace) - if err != nil { + if apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) { + r.Log.V(4).Info( + "skipping LimitRange sync because namespace is terminating", + "name", target.Name, + "namespace", target.Namespace, + "tenant", tenant.Name, + ) + + return nil + } + return err } + + r.Log.V(4).Info("LimitRange sync result: "+string(res), "name", target.Name, "namespace", target.Namespace) } return nil diff --git a/internal/controllers/tenant/manager.go b/internal/controllers/tenant/manager.go index 8cc70edd..6bba91b2 100644 --- a/internal/controllers/tenant/manager.go +++ b/internal/controllers/tenant/manager.go @@ -5,8 +5,10 @@ package tenant import ( "context" + "errors" "fmt" "slices" + "time" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" @@ -17,14 +19,14 @@ import ( schedulingv1 "k8s.io/api/scheduling/v1" storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/authentication/serviceaccount" + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" "k8s.io/client-go/tools/events" - "k8s.io/client-go/util/retry" "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/cluster-api/util/patch" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -36,10 +38,11 @@ import ( gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" "github.com/projectcapsule/capsule/internal/controllers/utils" "github.com/projectcapsule/capsule/internal/metrics" - "github.com/projectcapsule/capsule/pkg/api" - meta "github.com/projectcapsule/capsule/pkg/api/meta" + caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/gvk" "github.com/projectcapsule/capsule/pkg/runtime/predicates" @@ -48,12 +51,19 @@ import ( type Manager struct { client.Client + reader client.Reader + + DiscoveryClient discovery.DiscoveryInterface + DynamicClient dynamic.Interface + Metrics *metrics.TenantRecorder Log logr.Logger Recorder events.EventRecorder Configuration configuration.Configuration RESTConfig *rest.Config classes supportedClasses + + discoveryCache cache.DiscoveryNamespacedResourceCache } type supportedClasses struct { @@ -62,6 +72,9 @@ type supportedClasses struct { } func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.ControllerOptions) error { + r.reader = mgr.GetAPIReader() + r.discoveryCache = cache.NewDiscoveryNamespacedResourceCache() + ctrlBuilder := ctrl.NewControllerManagedBy(mgr). Named("capsule/tenants"). For( @@ -81,7 +94,7 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller &capsulev1beta2.CapsuleConfiguration{}, handler.EnqueueRequestsFromMapFunc(r.enqueueAllTenants), builder.WithPredicates( - predicates.CapsuleConfigSpecChangedPredicate{}, + predicates.CapsuleConfigSpecAdministratorsChangedPredicate{}, predicates.NamesMatchingPredicate{Names: []string{ctrlConfig.ConfigurationName}}, ), ). @@ -89,32 +102,21 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller &corev1.Namespace{}, handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &capsulev1beta2.Tenant{}), ). + Watches( + &capsulev1beta2.RuleStatus{}, + handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &capsulev1beta2.Tenant{}), + ). Watches( &storagev1.StorageClass{}, - r.statusOnlyHandlerClasses( - r.reconcileClassStatus, - r.collectAvailableStorageClasses, - "cannot collect storage classes", - ), - builder.WithPredicates(predicates.UpdatedLabelsPredicate{}), + handler.EnqueueRequestsFromMapFunc(r.enqueueAllTenants), ). Watches( &schedulingv1.PriorityClass{}, - r.statusOnlyHandlerClasses( - r.reconcileClassStatus, - r.collectAvailablePriorityClasses, - "cannot collect priority classes", - ), - builder.WithPredicates(predicates.UpdatedLabelsPredicate{}), + handler.EnqueueRequestsFromMapFunc(r.enqueueAllTenants), ). Watches( &nodev1.RuntimeClass{}, - r.statusOnlyHandlerClasses( - r.reconcileClassStatus, - r.collectAvailableRuntimeClasses, - "cannot collect runtime classes", - ), - builder.WithPredicates(predicates.UpdatedLabelsPredicate{}), + handler.EnqueueRequestsFromMapFunc(r.enqueueAllTenants), ). Watches( &capsulev1beta2.TenantOwner{}, @@ -151,7 +153,14 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller e event.TypedDeleteEvent[client.Object], q workqueue.TypedRateLimitingInterface[reconcile.Request], ) { - r.enqueueTenantsForTenantOwner(ctx, e.Object, q) + r.enqueueForTenantsWithCondition( + ctx, + e.Object, + q, + func(tnt *capsulev1beta2.Tenant, _ client.Object) bool { + return len(tnt.Spec.Permissions.MatchOwners) > 0 + }, + ) }, }, ). @@ -184,7 +193,7 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller r.enqueueForTenantsWithCondition(ctx, e.Object, q, func(tnt *capsulev1beta2.Tenant, c client.Object) bool { _, found := tnt.Status.Owners.FindOwner( serviceaccount.ServiceAccountUsernamePrefix+c.GetNamespace()+":"+c.GetName(), - api.ServiceAccountOwner, + rbac.ServiceAccountOwner, ) return found @@ -205,12 +214,7 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller if r.classes.gateway { ctrlBuilder = ctrlBuilder.Watches( &gatewayv1.GatewayClass{}, - r.statusOnlyHandlerClasses( - r.reconcileClassStatus, - r.collectAvailableGatewayClasses, - "cannot collect gateway classes", - ), - builder.WithPredicates(predicates.UpdatedLabelsPredicate{}), + handler.EnqueueRequestsFromMapFunc(r.enqueueAllTenants), ) } @@ -224,26 +228,21 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller if r.classes.device { ctrlBuilder = ctrlBuilder.Watches( &resourcesv1.DeviceClass{}, - r.statusOnlyHandlerClasses( - r.reconcileClassStatus, - r.collectAvailableDeviceClasses, - "cannot collect device classes", - ), - builder.WithPredicates(predicates.UpdatedLabelsPredicate{}), + handler.EnqueueRequestsFromMapFunc(r.enqueueAllTenants), ) } return ctrlBuilder.Complete(r) } -func (r Manager) Reconcile(ctx context.Context, request ctrl.Request) (result ctrl.Result, err error) { +func (r *Manager) Reconcile(ctx context.Context, request ctrl.Request) (result ctrl.Result, err error) { r.Log = r.Log.WithValues("Request.Name", request.Name) // Fetch the Tenant instance instance := &capsulev1beta2.Tenant{} if err = r.Get(ctx, request.NamespacedName, instance); err != nil { if apierrors.IsNotFound(err) { - r.Log.V(3).Info("request object not found, could have been deleted after reconcile request") + r.Log.V(5).Info("request object not found, could have been deleted after reconcile request") // If tenant was deleted or cannot be found, clean up metrics r.Metrics.DeleteAllMetricsForTenant(request.Name) @@ -251,140 +250,123 @@ func (r Manager) Reconcile(ctx context.Context, request ctrl.Request) (result ct return reconcile.Result{}, nil } - r.Log.Error(err, "error reading the object") - return result, err } + patchHelper, err := patch.NewHelper(instance, r.Client) + if err != nil { + return reconcile.Result{}, err + } + + if updateErr := r.updateReconcilingStatus(ctx, instance); updateErr != nil { + if apierrors.IsNotFound(updateErr) { + return reconcile.Result{}, nil + } + + return reconcile.Result{}, updateErr + } + + reconcileError := r.reconcile(ctx, instance) + defer func() { r.syncTenantStatusMetrics(instance) - if uerr := r.updateTenantStatus(ctx, instance, err); uerr != nil { - err = fmt.Errorf("cannot update tenant status: %w", uerr) + if statusErr := r.updateTenantStatus(ctx, instance, reconcileError); statusErr != nil { + statusErr = fmt.Errorf("cannot update tenant status: %w", statusErr) - return + if err == nil { + err = statusErr + } else { + err = errors.Join(err, statusErr) + } } }() - // Collect Ownership for Status - if err = r.collectOwners(ctx, instance); err != nil { - err = fmt.Errorf("cannot collect available owners: %w", err) + if e := patchHelper.Patch(ctx, instance); e != nil { + if caperrors.IgnoreGone(e) { + err = nil - return result, err - } + return result, err + } - // Ensuring Metadata. - err, updated := r.ensureMetadata(ctx, instance) - if err != nil { - err = fmt.Errorf("cannot ensure metadata: %w", err) - - return result, err - } - - if updated { - return result, nil - } - - // Reconcile Namespaces - r.Log.V(4).Info("starting processing of Namespaces", "items", len(instance.Status.Namespaces)) - - if err = r.reconcileNamespaces(ctx, instance); err != nil { - err = fmt.Errorf("namespace(s) had reconciliation errors") - - return result, err - } - - // Ensuring ResourceQuota - r.Log.V(4).Info("ensuring limit resources count is updated") - - if err = r.syncCustomResourceQuotaUsages(ctx, instance); err != nil { - err = fmt.Errorf("cannot count limited resources: %w", err) - - return result, err - } - - // Ensuring NetworkPolicy resources - r.Log.V(4).Info("starting processing of Network Policies") - - if err = r.syncNetworkPolicies(ctx, instance); err != nil { - err = fmt.Errorf("cannot sync networkPolicy items: %w", err) - - return result, err - } - - // Ensuring LimitRange resources - r.Log.V(4).Info("Starting processing of Limit Ranges", "items", len(instance.Spec.LimitRanges.Items)) //nolint:staticcheck - - if err = r.syncLimitRanges(ctx, instance); err != nil { - err = fmt.Errorf("cannot sync limitrange items: %w", err) - - return result, err - } - - // Ensuring ResourceQuota resources - r.Log.V(4).Info("Starting processing of Resource Quotas", "items", len(instance.Spec.ResourceQuota.Items)) - - if err = r.syncResourceQuotas(ctx, instance); err != nil { - err = fmt.Errorf("cannot sync resourcequota items: %w", err) - - return result, err - } - - // Ensuring RoleBinding resources - r.Log.V(4).Info("Ensuring RoleBindings for Owners and Tenant") - - if err = r.syncRoleBindings(ctx, instance); err != nil { - err = fmt.Errorf("cannot sync rolebindings items: %w", err) - - return result, err + return reconcile.Result{}, e } // Collect available resources if err = r.collectAvailableResources(ctx, instance); err != nil { err = fmt.Errorf("cannot collect available resources: %w", err) - return result, err + return reconcile.Result{}, err + } + + if instance.DeletionTimestamp != nil && len(instance.Status.Spaces) > 0 { + return reconcile.Result{RequeueAfter: 2 * time.Second}, nil + } + + return reconcile.Result{}, reconcileError +} + +func (r *Manager) reconcile(ctx context.Context, instance *capsulev1beta2.Tenant) (err error) { + var errs []error + + // Collect Ownership/Promotions for Status + if err = r.collectRBAC(ctx, instance); err != nil { + errs = append(errs, fmt.Errorf("cannot collect available rbac: %w", err)) + } + + // Reconcile Namespaces + r.Log.V(4).Info("starting processing of Namespaces", "items", len(instance.Status.Namespaces)) + + if err = r.reconcileNamespaces(ctx, instance); err != nil { + errs = append(errs, fmt.Errorf("namespace(s) had reconciliation errors: %w", err)) + } + + // Ensuring Metadata. + err = r.ensureMetadata(ctx, instance) + if err != nil { + errs = append(errs, fmt.Errorf("cannot ensure metadata: %w", err)) + } + + // Ensuring ResourceQuota + r.Log.V(4).Info("ensuring limit resources count is updated") + + if err = r.syncCustomResourceQuotaUsages(ctx, instance); err != nil { + errs = append(errs, fmt.Errorf("cannot count limited resources: %w", err)) + } + + // Ensuring NetworkPolicy resources + r.Log.V(4).Info("starting processing of Network Policies") + + if err = r.syncNetworkPolicies(ctx, instance); err != nil { + errs = append(errs, fmt.Errorf("cannot sync networkPolicy items: %w", err)) + } + + // Ensuring LimitRange resources + r.Log.V(4).Info("Starting processing of Limit Ranges", "items", len(instance.Spec.LimitRanges.Items)) //nolint:staticcheck + + if err = r.syncLimitRanges(ctx, instance); err != nil { + errs = append(errs, fmt.Errorf("cannot sync limitrange items: %w", err)) + } + + // Ensuring ResourceQuota resources + r.Log.V(4).Info("Starting processing of Resource Quotas", "items", len(instance.Spec.ResourceQuota.Items)) + + if err = r.syncResourceQuotas(ctx, instance); err != nil { + errs = append(errs, fmt.Errorf("cannot sync resourcequota items: %w", err)) + } + + // Ensuring RoleBinding resources + r.Log.V(4).Info("Ensuring RoleBindings for Owners and Tenant") + + if err = r.syncRoleBindings(ctx, r.Log, instance); err != nil { + errs = append(errs, fmt.Errorf("cannot sync rolebindings items: %w", err)) + } + + if err = errors.Join(errs...); err != nil { + return err } r.Log.V(4).Info("Tenant reconciling completed") - return ctrl.Result{}, err -} - -func (r *Manager) updateTenantStatus(ctx context.Context, tnt *capsulev1beta2.Tenant, reconcileError error) error { - return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { - latest := &capsulev1beta2.Tenant{} - if err = r.Get(ctx, types.NamespacedName{Name: tnt.GetName()}, latest); err != nil { - return err - } - - latest.Status = tnt.Status - - // Set Ready Condition - readyCondition := meta.NewReadyCondition(tnt) - if reconcileError != nil { - readyCondition.Message = reconcileError.Error() - readyCondition.Status = metav1.ConditionFalse - readyCondition.Reason = meta.FailedReason - } - - latest.Status.Conditions.UpdateConditionByType(readyCondition) - - // Set Cordoned Condition - cordonedCondition := meta.NewCordonedCondition(tnt) - - if tnt.Spec.Cordoned { - latest.Status.State = capsulev1beta2.TenantStateCordoned - - cordonedCondition.Reason = meta.CordonedReason - cordonedCondition.Message = "Tenant is cordoned" - cordonedCondition.Status = metav1.ConditionTrue - } else { - latest.Status.State = capsulev1beta2.TenantStateActive - } - - latest.Status.Conditions.UpdateConditionByType(cordonedCondition) - - return r.Client.Status().Update(ctx, latest) - }) + return err } diff --git a/internal/controllers/tenant/metadata.go b/internal/controllers/tenant/metadata.go index 49cc0c5c..b2fd899a 100644 --- a/internal/controllers/tenant/metadata.go +++ b/internal/controllers/tenant/metadata.go @@ -6,24 +6,27 @@ package tenant import ( "context" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api/meta" ) // Sets a label on the Tenant object with it's name. -func (r *Manager) ensureMetadata(ctx context.Context, tnt *capsulev1beta2.Tenant) (err error, changed bool) { - // Assign Labels +func (r *Manager) ensureMetadata(ctx context.Context, tnt *capsulev1beta2.Tenant) (err error) { if tnt.Labels == nil { - tnt.Labels = make(map[string]string) + tnt.Labels = map[string]string{} } if v, ok := tnt.Labels[meta.TenantNameLabel]; !ok || v != tnt.Name { - if err := r.Update(ctx, tnt); err != nil { - return err, false - } - - return nil, true + tnt.Labels[meta.TenantNameLabel] = tnt.Name } - return nil, false + if len(tnt.Status.Spaces) == 0 { + controllerutil.RemoveFinalizer(tnt, meta.ControllerFinalizer) + } else { + controllerutil.AddFinalizer(tnt, meta.ControllerFinalizer) + } + + return nil } diff --git a/internal/controllers/tenant/namespaces.go b/internal/controllers/tenant/namespaces.go index 57b14687..41846568 100644 --- a/internal/controllers/tenant/namespaces.go +++ b/internal/controllers/tenant/namespaces.go @@ -5,15 +5,16 @@ package tenant import ( "context" + "errors" "fmt" "maps" - "slices" "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -23,70 +24,145 @@ import ( ) // Ensuring all annotations are applied to each Namespace handled by the Tenant. -func (r *Manager) reconcileNamespaces(ctx context.Context, tenant *capsulev1beta2.Tenant) (err error) { - if err = r.collectNamespaces(ctx, tenant); err != nil { - err = fmt.Errorf("cannot collect namespaces: %w", err) +func (r *Manager) reconcileNamespaces(ctx context.Context, tnt *capsulev1beta2.Tenant) (err error) { + if tnt.DeletionTimestamp != nil { + for _, ns := range tnt.Status.Spaces { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns.Name, + }, + } + if err := r.Delete(ctx, ns, &client.DeleteOptions{ + PropagationPolicy: ptr.To(metav1.DeletePropagationBackground), + }); err != nil && !apierrors.IsNotFound(err) { + r.Log.Error(err, "unable to delete tenant namespace", + "tenant", tnt.GetName(), + "namespace", ns.Name, + ) + + return err + } + } + } + + list := &corev1.NamespaceList{} + if err := r.List(ctx, list, client.MatchingFields{".metadata.ownerReferences[*].capsule": tnt.GetName()}); err != nil { return err } - gcSet := make(map[string]struct{}) - for _, inst := range tenant.Status.Spaces { - gcSet[inst.Name] = struct{}{} + oldStatus := make(map[string]struct{}, len(tnt.Status.Spaces)) + for i := range tnt.Status.Spaces { + oldStatus[tnt.Status.Spaces[i].Name] = struct{}{} } - group := new(errgroup.Group) + group, ctx := errgroup.WithContext(ctx) + group.SetLimit(8) - for _, item := range tenant.Status.Namespaces { - namespace := item + results := make(chan *capsulev1beta2.TenantStatusNamespaceItem, len(list.Items)) + errs := make(chan error, len(list.Items)) - delete(gcSet, namespace) + for i := range list.Items { + ns := list.Items[i].DeepCopy() group.Go(func() error { - return r.reconcileNamespace(ctx, namespace, tenant) + stat, err := r.reconcileNamespace(ctx, ns, tnt) + if stat != nil { + results <- stat + } + + if err != nil { + r.Log.Error(err, "failed to reconcile namespace", + "tenant", tnt.GetName(), + "namespace", ns.GetName(), + ) + + errs <- fmt.Errorf("namespace %q: %w", ns.Name, err) + } + + return nil }) } - if err = group.Wait(); err != nil { - err = fmt.Errorf("cannot sync Namespaces: %w", err) + _ = group.Wait() + + close(results) + close(errs) + + var joined []error + for itemErr := range errs { + joined = append(joined, itemErr) } - for name := range gcSet { + err = errors.Join(joined...) + + desiredStatus := make(map[string]struct{}, len(list.Items)) + + for stat := range results { + if stat == nil { + continue + } + + tnt.Status.UpdateInstance(stat) + + desiredStatus[stat.Name] = struct{}{} + } + + for name := range oldStatus { + if _, keep := desiredStatus[name]; keep { + continue + } + r.Metrics.DeleteAllMetricsForNamespace(name) - tenant.Status.RemoveInstance(&capsulev1beta2.TenantStatusNamespaceItem{ - Name: name, - }) + tnt.Status.RemoveInstance(&capsulev1beta2.TenantStatusNamespaceItem{Name: name}) } - tenant.Status.Size = uint(len(tenant.Status.Namespaces)) + tnt.Status.Size = uint(len(tnt.Status.Spaces)) + + tnt.AssignNamespaces(list.Items) return err } -func (r *Manager) reconcileNamespace(ctx context.Context, namespace string, tnt *capsulev1beta2.Tenant) (err error) { - ns := &corev1.Namespace{} - if err = r.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil { - return err - } +func (r *Manager) reconcileNamespace(ctx context.Context, namespace *corev1.Namespace, tnt *capsulev1beta2.Tenant) ( + stat *capsulev1beta2.TenantStatusNamespaceItem, + err error, +) { + terminating := false - stat := &capsulev1beta2.TenantStatusNamespaceItem{ - Name: namespace, - UID: ns.GetUID(), + stat = &capsulev1beta2.TenantStatusNamespaceItem{ + Name: namespace.GetName(), + UID: namespace.GetUID(), } metaStatus := &capsulev1beta2.TenantStatusNamespaceMetadata{} + instance := tnt.Status.GetInstance(stat) + if instance != nil { + stat = instance + } + + dropFromStatus := false + // Always update tenant status condition after reconciliation defer func() { - instance := tnt.Status.GetInstance(stat) - if instance != nil { - stat = instance + if dropFromStatus { + stat = nil + + r.Metrics.DeleteAllMetricsForNamespace(namespace.GetName()) + + return } - readCondition := meta.NewReadyCondition(ns) + readCondition := meta.NewReadyCondition(namespace) - if err != nil { + switch { + case terminating: + readCondition.Status = metav1.ConditionFalse + readCondition.Reason = meta.TerminatingReason + readCondition.Message = "Namespace is terminating" + case err != nil: readCondition.Status = metav1.ConditionFalse readCondition.Reason = meta.FailedReason readCondition.Message = fmt.Sprintf("Failed to reconcile: %v", err) @@ -94,15 +170,17 @@ func (r *Manager) reconcileNamespace(ctx context.Context, namespace string, tnt if instance != nil && instance.Metadata != nil { stat.Metadata = instance.Metadata } - } else if metaStatus != nil { - stat.Metadata = metaStatus + default: + if metaStatus != nil { + stat.Metadata = metaStatus + } } stat.Conditions.UpdateConditionByType(readCondition) - cordonedCondition := meta.NewCordonedCondition(ns) + cordonedCondition := meta.NewCordonedCondition(namespace) - if ns.Labels[meta.CordonedLabel] == meta.ValueTrue { + if namespace.Labels[meta.CordonedLabel] == meta.ValueTrue { cordonedCondition.Reason = meta.CordonedReason cordonedCondition.Message = "namespace is cordoned" cordonedCondition.Status = metav1.ConditionTrue @@ -110,25 +188,72 @@ func (r *Manager) reconcileNamespace(ctx context.Context, namespace string, tnt stat.Conditions.UpdateConditionByType(cordonedCondition) - tnt.Status.UpdateInstance(stat) - - r.syncNamespaceStatusMetrics(tnt, ns) + r.syncNamespaceStatusMetrics(tnt, namespace) }() - // Collect Rules for namespace - ruleBody, err := tenant.BuildNamespaceRuleBodyForNamespace(ns, tnt) - if err != nil { - return err + // Verify if namespace is still active or terminating + if namespace.DeletionTimestamp != nil { + terminating = true + + terminatingState := meta.NewTerminatingConditionReason(namespace) + + pending, err := tenant.NamespaceIsPendingPodTerminating(ctx, r.Client, namespace) + if err != nil { + terminatingState.Reason = meta.FailedReason + terminatingState.Status = metav1.ConditionFalse + terminatingState.Message = err.Error() + stat.Conditions.UpdateConditionByType(terminatingState) + + return stat, err + } + + if pending { + terminatingState.Reason = meta.PendingUnmanagedContentReason + terminatingState.Status = metav1.ConditionFalse + terminatingState.Message = "waiting for pods to finalize" + stat.Conditions.UpdateConditionByType(terminatingState) + + return stat, nil + } + + cleaned, err := tenant.NamespacedCascadingCleanup(ctx, r.Client, r.DiscoveryClient, &r.discoveryCache, r.DynamicClient, namespace) + if err != nil { + terminatingState.Reason = meta.FailedReason + terminatingState.Status = metav1.ConditionFalse + terminatingState.Message = err.Error() + stat.Conditions.UpdateConditionByType(terminatingState) + + return stat, err + } + + if cleaned { + terminatingState.Reason = meta.PendingUnmanagedContentReason + terminatingState.Status = metav1.ConditionFalse + terminatingState.Message = "performing cascading deletion" + stat.Conditions.UpdateConditionByType(terminatingState) + + return stat, nil + } + + terminatingState.Message = "removed managed resources" + stat.Conditions.UpdateConditionByType(terminatingState) + + r.Metrics.DeleteAllMetricsForNamespace(namespace.GetName()) + + dropFromStatus = true + + return nil, nil } - err = r.ensureRuleStatus(ctx, ns, tnt, ruleBody, namespace) + // Collect Rules for namespace + err = r.reconcileRuleStatus(ctx, tnt, namespace) if err != nil { - return err + return stat, err } err = retry.RetryOnConflict(retry.DefaultBackoff, func() (conflictErr error) { - _, conflictErr = controllerutil.CreateOrUpdate(ctx, r.Client, ns, func() error { - metaStatus, err = r.reconcileNamespaceMetadata(ctx, ns, tnt, stat) + _, conflictErr = controllerutil.CreateOrUpdate(ctx, r.Client, namespace, func() error { + metaStatus, err = r.reconcileNamespaceMetadata(ctx, namespace, tnt, stat) return err }) @@ -136,52 +261,7 @@ func (r *Manager) reconcileNamespace(ctx context.Context, namespace string, tnt return conflictErr }) - return err -} - -func (r *Manager) ensureRuleStatus( - ctx context.Context, - ns *corev1.Namespace, - tnt *capsulev1beta2.Tenant, - rule *capsulev1beta2.NamespaceRuleBody, - namespace string, -) error { - nsStatus := &capsulev1beta2.RuleStatus{ - ObjectMeta: metav1.ObjectMeta{ - Name: meta.NameForManagedRuleStatus(), - Namespace: namespace, - }, - } - - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, nsStatus, func() error { - labels := nsStatus.GetLabels() - if labels == nil { - labels = make(map[string]string) - } - - labels[meta.NewManagedByCapsuleLabel] = meta.ValueController - labels[meta.CapsuleNameLabel] = nsStatus.Name - - nsStatus.SetLabels(labels) - - err := controllerutil.SetOwnerReference(tnt, nsStatus, r.Scheme()) - if err != nil { - return err - } - - return controllerutil.SetOwnerReference(ns, nsStatus, r.Scheme()) - }) - if err != nil { - return err - } - - nsStatus.Status.Rule = *rule - - if err := r.Status().Update(ctx, nsStatus); err != nil { - return err - } - - return nil + return stat, err } //nolint:nestif @@ -258,21 +338,3 @@ func (r *Manager) reconcileNamespaceMetadata( return managed, err } - -func (r *Manager) collectNamespaces(ctx context.Context, tenant *capsulev1beta2.Tenant) (err error) { - list := &corev1.NamespaceList{} - - err = r.List(ctx, list, client.MatchingFields{".metadata.ownerReferences[*].capsule": tenant.GetName()}) - if err != nil { - return err - } - - // Drop namespaces that are currently being deleted (DeletionTimestamp != nil) - activeNamespaces := slices.DeleteFunc(list.Items, func(ns corev1.Namespace) bool { - return ns.DeletionTimestamp != nil - }) - - tenant.AssignNamespaces(activeNamespaces) - - return err -} diff --git a/internal/controllers/tenant/networkpolicies.go b/internal/controllers/tenant/networkpolicies.go index f65d23c4..69e18732 100644 --- a/internal/controllers/tenant/networkpolicies.go +++ b/internal/controllers/tenant/networkpolicies.go @@ -1,6 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 +//nolint:dupl package tenant import ( @@ -8,8 +9,9 @@ import ( "fmt" "strconv" - "golang.org/x/sync/errgroup" + corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -19,7 +21,7 @@ import ( // Ensuring all the NetworkPolicies are applied to each Namespace handled by the Tenant. // -//nolint:dupl + func (r *Manager) syncNetworkPolicies(ctx context.Context, tenant *capsulev1beta2.Tenant) error { keys := make([]string, 0, len(tenant.Spec.NetworkPolicies.Items)) //nolint:staticcheck @@ -28,17 +30,9 @@ func (r *Manager) syncNetworkPolicies(ctx context.Context, tenant *capsulev1beta keys = append(keys, strconv.Itoa(i)) } - group := new(errgroup.Group) - - for _, ns := range tenant.Status.Namespaces { - namespace := ns - - group.Go(func() error { - return r.syncNetworkPolicy(ctx, tenant, namespace, keys) - }) - } - - return group.Wait() + return runForTenantNamespaces(ctx, tenant, func(ctx context.Context, namespace string) error { + return r.syncNetworkPolicy(ctx, tenant, namespace, keys) + }) } func (r *Manager) syncNetworkPolicy(ctx context.Context, tenant *capsulev1beta2.Tenant, namespace string, keys []string) (err error) { @@ -75,12 +69,22 @@ func (r *Manager) syncNetworkPolicy(ctx context.Context, tenant *capsulev1beta2. return controllerutil.SetControllerReference(tenant, target, r.Scheme()) }) - - r.Log.V(4).Info("network Policy sync result: "+string(res), "name", target.Name, "namespace", target.Namespace) - if err != nil { + if apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) { + r.Log.V(4).Info( + "skipping NetworkPolicy sync because namespace is terminating", + "name", target.Name, + "namespace", target.Namespace, + "tenant", tenant.Name, + ) + + return nil + } + return err } + + r.Log.V(4).Info("network Policy sync result: "+string(res), "name", target.Name, "namespace", target.Namespace) } return nil diff --git a/internal/controllers/tenant/resourcequotas.go b/internal/controllers/tenant/resourcequotas.go index 65c85c15..53c04652 100644 --- a/internal/controllers/tenant/resourcequotas.go +++ b/internal/controllers/tenant/resourcequotas.go @@ -5,12 +5,14 @@ package tenant import ( "context" + "errors" "fmt" "strconv" "strings" "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -40,6 +42,7 @@ import ( // // In case of Namespace-scoped Resource Budget, we're just replicating the resources across all registered Namespaces. +//nolint:cyclop func (r *Manager) syncResourceQuotas(ctx context.Context, tenant *capsulev1beta2.Tenant) (err error) { //nolint:gocognit // Remove prior metrics, to avoid cleaning up for metrics of deleted ResourceQuotas r.Metrics.DeleteTenantResourceMetrics(tenant.Name) @@ -52,6 +55,7 @@ func (r *Manager) syncResourceQuotas(ctx context.Context, tenant *capsulev1beta2 //nolint:nestif if tenant.Spec.ResourceQuota.Scope == api.ResourceQuotaScopeTenant { + scopeErrs := make(chan error, len(tenant.Spec.ResourceQuota.Items)) group := new(errgroup.Group) for i, q := range tenant.Spec.ResourceQuota.Items { @@ -63,6 +67,12 @@ func (r *Manager) syncResourceQuotas(ctx context.Context, tenant *capsulev1beta2 } group.Go(func() (scopeErr error) { + defer func() { + if scopeErr != nil { + scopeErrs <- fmt.Errorf("resource quota %d: %w", index, scopeErr) + } + }() + // Calculating the Resource Budget at Tenant scope just if this is put in place. // Requirement to list ResourceQuota of the current Tenant var tntRequirement *labels.Requirement @@ -80,7 +90,7 @@ func (r *Manager) syncResourceQuotas(ctx context.Context, tenant *capsulev1beta2 // These are required since Capsule is going to sum all the used quota to // sum them and get the Tenant one. list := &corev1.ResourceQuotaList{} - if scopeErr = r.List(ctx, list, &client.ListOptions{LabelSelector: labels.NewSelector().Add(*tntRequirement).Add(*indexRequirement)}); scopeErr != nil { + if scopeErr = r.reader.List(ctx, list, &client.ListOptions{LabelSelector: labels.NewSelector().Add(*tntRequirement).Add(*indexRequirement)}); scopeErr != nil { r.Log.Error(scopeErr, "cannot list ResourceQuota", "tenantFilter", tntRequirement.String(), "indexFilter", indexRequirement.String()) return scopeErr @@ -168,14 +178,24 @@ func (r *Manager) syncResourceQuotas(ctx context.Context, tenant *capsulev1beta2 } } - return scopeErr + return nil }) } - // Waiting the update of all ResourceQuotas - if err = group.Wait(); err != nil { + + _ = group.Wait() + + close(scopeErrs) + + var joined []error + for scopeErr := range scopeErrs { + joined = append(joined, scopeErr) + } + + if err = errors.Join(joined...); err != nil { return err } } + // getting requested ResourceQuota keys keys := make([]string, 0, len(tenant.Spec.ResourceQuota.Items)) @@ -185,8 +205,13 @@ func (r *Manager) syncResourceQuotas(ctx context.Context, tenant *capsulev1beta2 group := new(errgroup.Group) - for _, ns := range tenant.Status.Namespaces { - namespace := ns + for _, ns := range tenant.Status.Spaces { + namespace := ns.Name + + cond := ns.Conditions.GetConditionByType(meta.ReadyCondition) + if cond != nil && cond.Reason == meta.TerminatingReason { + continue + } group.Go(func() error { return r.syncResourceQuota(ctx, tenant, namespace, keys) @@ -233,10 +258,11 @@ func (r *Manager) syncResourceQuota(ctx context.Context, tenant *capsulev1beta2. delete(targetLabels, meta.TenantLabel) target.SetLabels(targetLabels) + target.Spec.Scopes = resQuota.Scopes target.Spec.ScopeSelector = resQuota.ScopeSelector - // In case of Namespace scope for the ResourceQuota we can easily apply the bare specification + // In case of Namespace scope for the ResourceQuota we can easily apply the bare specification. if tenant.Spec.ResourceQuota.Scope == api.ResourceQuotaScopeNamespace { target.Spec.Hard = resQuota.Hard } @@ -246,12 +272,22 @@ func (r *Manager) syncResourceQuota(ctx context.Context, tenant *capsulev1beta2. return retryErr }) - - r.Log.V(4).Info("resource Quota sync result: "+string(res), "name", target.Name, "namespace", target.Namespace) - if err != nil { + if apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) { + r.Log.V(4).Info( + "skipping ResourceQuota sync because namespace is terminating", + "name", target.Name, + "namespace", target.Namespace, + "tenant", tenant.Name, + ) + + return nil + } + return err } + + r.Log.V(4).Info("resource Quota sync result: "+string(res), "name", target.Name, "namespace", target.Namespace) } return nil diff --git a/internal/controllers/tenant/rolebindings.go b/internal/controllers/tenant/rolebindings.go index 89f16f22..e3e83fd3 100644 --- a/internal/controllers/tenant/rolebindings.go +++ b/internal/controllers/tenant/rolebindings.go @@ -7,49 +7,65 @@ import ( "context" "fmt" - "golang.org/x/sync/errgroup" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" ) // Sync the dynamic Tenant Owner specific cluster-roles and additional Role Bindings, which can be used in many ways: // applying Pod Security Policies or giving access to CRDs or specific API groups. -func (r *Manager) syncRoleBindings(ctx context.Context, tenant *capsulev1beta2.Tenant) (err error) { - roleBindings := tenant.GetRoleBindings() +func (r *Manager) syncRoleBindings(ctx context.Context, log logr.Logger, tenant *capsulev1beta2.Tenant) (err error) { + namespaceBindings := map[string]map[string]rbac.AdditionalRoleBindingsSpec{} - // Hashing - hashes := map[string]api.AdditionalRoleBindingsSpec{} + for _, ns := range tenant.Status.Spaces { + namespace := ns.Name - for _, binding := range roleBindings { + if _, ok := namespaceBindings[namespace]; !ok { + namespaceBindings[namespace] = map[string]rbac.AdditionalRoleBindingsSpec{} + } + } + + for _, binding := range tenant.GetRoleBindings() { hash := utils.RoleBindingHashFunc(binding) - hashes[hash] = binding + for namespace := range namespaceBindings { + namespaceBindings[namespace][hash] = binding + } } - group := new(errgroup.Group) + // Does not target all namespaces + for _, promotion := range tenant.GetPromotionRoleBindings() { + namespace := string(promotion.Namespace) - for _, ns := range tenant.Status.Namespaces { - namespace := ns + if _, ok := namespaceBindings[namespace]; !ok { + // Ignore namespaces that are not part of the tenant. + continue + } - group.Go(func() error { - return r.syncAdditionalRoleBinding(ctx, tenant, namespace, hashes) - }) + binding := promotion.AdditionalRoleBindingsSpec + hash := utils.RoleBindingHashFunc(binding) + + namespaceBindings[namespace][hash] = binding } - return group.Wait() + return runForTenantNamespaces(ctx, tenant, func(ctx context.Context, namespace string) error { + return r.syncAdditionalRoleBinding(ctx, tenant, namespace, namespaceBindings[namespace]) + }) } func (r *Manager) syncAdditionalRoleBinding( ctx context.Context, tenant *capsulev1beta2.Tenant, ns string, - bindings map[string]api.AdditionalRoleBindingsSpec, + bindings map[string]rbac.AdditionalRoleBindingsSpec, ) (err error) { keys := []string{} @@ -97,14 +113,21 @@ func (r *Manager) syncAdditionalRoleBinding( return controllerutil.SetControllerReference(tenant, target, r.Scheme()) }) if err != nil { - r.Log.Error(err, "cannot sync RoleBinding") + if apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) { + r.Log.V(4).Info( + "skipping RoleBinding sync because namespace is terminating", + "name", target.Name, + "namespace", target.Namespace, + "clusterRole", roleBinding.ClusterRoleName, + ) + + continue + } + + return fmt.Errorf("%w (role: %s)", err, roleBinding.ClusterRoleName) } r.Log.V(4).Info(fmt.Sprintf("roleBinding sync result: %s", string(res)), "name", target.Name, "namespace", target.Namespace) - - if err != nil { - return err - } } // Prune at finish to prevent gaps diff --git a/internal/controllers/tenant/rulestatus.go b/internal/controllers/tenant/rulestatus.go new file mode 100644 index 00000000..a0b3dc97 --- /dev/null +++ b/internal/controllers/tenant/rulestatus.go @@ -0,0 +1,85 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenant + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/tenant" +) + +func (r *Manager) reconcileRuleStatus( + ctx context.Context, + tnt *capsulev1beta2.Tenant, + ns *corev1.Namespace, +) error { + // Collect Rules for namespace + ruleBody, err := tenant.BuildNamespaceRuleBodyStatus(ctx, r.Client, ns, tnt) + if err != nil { + return err + } + + return r.ensureRuleStatus( + ctx, + tnt, + ns, + ruleBody, + ) +} + +func (r *Manager) ensureRuleStatus( + ctx context.Context, + tnt *capsulev1beta2.Tenant, + namespace *corev1.Namespace, + body *api.NamespaceRuleBodyNamespace, +) error { + rule := &capsulev1beta2.RuleStatus{ + ObjectMeta: metav1.ObjectMeta{ + Name: meta.NameForManagedRuleStatus(), + Namespace: namespace.GetName(), + }, + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, rule, func() error { + labels := rule.GetLabels() + if labels == nil { + labels = make(map[string]string) + } + + labels[meta.NewManagedByCapsuleLabel] = meta.ValueController + labels[meta.CapsuleNameLabel] = rule.GetName() + + rule.SetLabels(labels) + + if body != nil { + rule.Spec = []*api.NamespaceRuleBodyNamespace{body} + } + + return controllerutil.SetControllerReference(tnt, rule, r.Scheme()) + }) + if err != nil { + if apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) { + r.Log.V(4).Info( + "skipping RuleStatus sync because namespace is terminating", + "name", rule.Name, + "namespace", rule.Namespace, + "tenant", tnt.Name, + ) + + return nil + } + + return err + } + + return nil +} diff --git a/internal/controllers/tenant/status.go b/internal/controllers/tenant/status.go index 7fbe2eec..967d4c08 100644 --- a/internal/controllers/tenant/status.go +++ b/internal/controllers/tenant/status.go @@ -5,70 +5,146 @@ package tenant import ( "context" - "fmt" + "regexp" "sort" nodev1 "k8s.io/api/node/v1" resources "k8s.io/api/resource/v1" schedulingv1 "k8s.io/api/scheduling/v1" storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + capmeta "github.com/projectcapsule/capsule/pkg/api/meta" "github.com/projectcapsule/capsule/pkg/tenant" ) +func setTenantStatusState(tnt *capsulev1beta2.Tenant) { + if tnt.DeletionTimestamp != nil { + tnt.Status.State = capsulev1beta2.TenantStateTerminating + + return + } + + if tnt.Spec.Cordoned { + tnt.Status.State = capsulev1beta2.TenantStateCordoned + + return + } + + tnt.Status.State = capsulev1beta2.TenantStateActive +} + +func (r *Manager) updateTenantStatus(ctx context.Context, instance *capsulev1beta2.Tenant, reconcileError error) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + latest := &capsulev1beta2.Tenant{} + if err := r.reader.Get(ctx, types.NamespacedName{Name: instance.GetName()}, latest); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + latest.Status = instance.Status + setTenantStatusState(latest) + + readyCondition := capmeta.NewReadyCondition(instance) + if reconcileError != nil { + readyCondition.Message = reconcileError.Error() + readyCondition.Status = metav1.ConditionFalse + readyCondition.Reason = capmeta.FailedReason + } + + latest.Status.Conditions.UpdateConditionByType(readyCondition) + + if err := r.Client.Status().Update(ctx, latest); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + instance.Status = latest.Status + + return nil + }) +} + +func (r *Manager) updateReconcilingStatus(ctx context.Context, instance *capsulev1beta2.Tenant) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + latest := &capsulev1beta2.Tenant{} + if err = r.reader.Get(ctx, types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}, latest); err != nil { + return err + } + + latest.Status.Conditions.UpdateConditionByType(capmeta.NewReadyConditionReconcilingReason(instance)) + + setTenantStatusState(latest) + + cordonedCondition := capmeta.NewCordonedCondition(instance) + + if instance.Spec.Cordoned { + latest.Status.State = capsulev1beta2.TenantStateCordoned + + cordonedCondition.Reason = capmeta.CordonedReason + cordonedCondition.Message = "Tenant is cordoned" + cordonedCondition.Status = metav1.ConditionTrue + } + + latest.Status.Conditions.UpdateConditionByType(cordonedCondition) + + if err := r.Client.Status().Update(ctx, latest); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + instance.Status = latest.Status + + return nil + }) +} + // Sets a label on the Tenant object with it's name. -func (r *Manager) collectOwners(ctx context.Context, tnt *capsulev1beta2.Tenant) (err error) { +func (r *Manager) collectRBAC(ctx context.Context, tnt *capsulev1beta2.Tenant) (err error) { owners, err := tenant.CollectOwners( ctx, r.Client, tnt, r.Configuration, ) + tnt.Status.Owners = owners + if err != nil { return err } - // No Direct Update needed as status is always posted - tnt.Status.Owners = owners + promotions, err := tenant.CollectPromotions( + ctx, + r.Client, + tnt, + r.Configuration, + ) + tnt.Status.Promotions = promotions - return nil -} - -func (r Manager) reconcileClassStatus( - ctx context.Context, - fn func(context.Context, *capsulev1beta2.Tenant) error, -) (err error) { - tntList := &capsulev1beta2.TenantList{} - if err = r.List(ctx, tntList); err != nil { + if err != nil { return err } - for i := range tntList.Items { - t := &tntList.Items[i] - - // Collect Ownership for Status - if err = fn(ctx, t); err != nil { - err = fmt.Errorf("cannot collect available classes: %w", err) - - return err - } - - if err = r.updateTenantStatus(ctx, t, err); err != nil { - err = fmt.Errorf("cannot update tenant status: %w", err) - - return err - } - } - - return err + return nil } func (r *Manager) collectAvailableResources(ctx context.Context, tnt *capsulev1beta2.Tenant) (err error) { @@ -118,7 +194,7 @@ func (r *Manager) collectAvailableResources(ctx context.Context, tnt *capsulev1b func (r *Manager) collectAvailableDeviceClasses(ctx context.Context, tnt *capsulev1beta2.Tenant) (err error) { if tnt.Status.Classes.DeviceClasses, err = listObjectNamesBySelector2( ctx, - r.Client, + r.reader, tnt.Spec.DeviceClasses, &resources.DeviceClassList{}, ); err != nil { @@ -131,7 +207,7 @@ func (r *Manager) collectAvailableDeviceClasses(ctx context.Context, tnt *capsul func (r *Manager) collectAvailableStorageClasses(ctx context.Context, tnt *capsulev1beta2.Tenant) (err error) { if tnt.Status.Classes.StorageClasses, err = listObjectNamesBySelector( ctx, - r.Client, + r.reader, tnt.Spec.StorageClasses, &storagev1.StorageClassList{}, ); err != nil { @@ -144,7 +220,7 @@ func (r *Manager) collectAvailableStorageClasses(ctx context.Context, tnt *capsu func (r *Manager) collectAvailablePriorityClasses(ctx context.Context, tnt *capsulev1beta2.Tenant) (err error) { if tnt.Status.Classes.PriorityClasses, err = listObjectNamesBySelector( ctx, - r.Client, + r.reader, tnt.Spec.PriorityClasses, &schedulingv1.PriorityClassList{}, ); err != nil { @@ -157,7 +233,7 @@ func (r *Manager) collectAvailablePriorityClasses(ctx context.Context, tnt *caps func (r *Manager) collectAvailableGatewayClasses(ctx context.Context, tnt *capsulev1beta2.Tenant) (err error) { if tnt.Status.Classes.GatewayClasses, err = listObjectNamesBySelector( ctx, - r.Client, + r.reader, tnt.Spec.GatewayOptions.AllowedClasses, &gatewayv1.GatewayClassList{}, ); err != nil { @@ -170,7 +246,7 @@ func (r *Manager) collectAvailableGatewayClasses(ctx context.Context, tnt *capsu func (r *Manager) collectAvailableRuntimeClasses(ctx context.Context, tnt *capsulev1beta2.Tenant) (err error) { if tnt.Status.Classes.RuntimeClasses, err = listObjectNamesBySelector( ctx, - r.Client, + r.reader, tnt.Spec.RuntimeClasses, &nodev1.RuntimeClassList{}, ); err != nil { @@ -184,7 +260,7 @@ func (r *Manager) collectAvailableRuntimeClasses(ctx context.Context, tnt *capsu // matching the provided LabelSelector, and returns their .metadata.name values. func listObjectNamesBySelector( ctx context.Context, - c client.Client, + c client.Reader, allowed *api.DefaultAllowedListSpec, list client.ObjectList, opts ...client.ListOption, @@ -265,6 +341,24 @@ func listObjectNamesBySelector( selected[name] = struct{}{} } + var regex *regexp.Regexp + + //nolint:staticcheck + if allowed.Regex != "" { + regex, err = regexp.Compile(allowed.Regex) + if err != nil { + return nil, err + } + } + + if regex != nil { + for name := range allNames { + if regex.MatchString(name) { + selected[name] = struct{}{} + } + } + } + for name := range selected { objects = append(objects, name) } @@ -276,7 +370,7 @@ func listObjectNamesBySelector( func listObjectNamesBySelector2( ctx context.Context, - c client.Client, + c client.Reader, allowed *api.SelectorAllowedListSpec, list client.ObjectList, opts ...client.ListOption, diff --git a/internal/controllers/tenant/utils.go b/internal/controllers/tenant/utils.go index ee0e2195..89a5c2a7 100644 --- a/internal/controllers/tenant/utils.go +++ b/internal/controllers/tenant/utils.go @@ -5,90 +5,74 @@ package tenant import ( "context" + "errors" + "fmt" + "golang.org/x/sync/errgroup" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" "github.com/projectcapsule/capsule/pkg/utils" ) -func (r *Manager) statusOnlyHandlerClasses( - fn func(ctx context.Context, perTenant func(context.Context, *capsulev1beta2.Tenant) error) error, - perTenant func(context.Context, *capsulev1beta2.Tenant) error, - errMsg string, -) *handler.TypedFuncs[client.Object, reconcile.Request] { - return &handler.TypedFuncs[client.Object, reconcile.Request]{ - CreateFunc: func( - ctx context.Context, - _ event.TypedCreateEvent[client.Object], - _ workqueue.TypedRateLimitingInterface[reconcile.Request], - ) { - if err := fn(ctx, perTenant); err != nil { - r.Log.Error(err, errMsg) - } - }, - UpdateFunc: func( - ctx context.Context, - _ event.TypedUpdateEvent[client.Object], - _ workqueue.TypedRateLimitingInterface[reconcile.Request], - ) { - if err := fn(ctx, perTenant); err != nil { - r.Log.Error(err, errMsg) - } - }, - DeleteFunc: func( - ctx context.Context, - _ event.TypedDeleteEvent[client.Object], - _ workqueue.TypedRateLimitingInterface[reconcile.Request], - ) { - if err := fn(ctx, perTenant); err != nil { - r.Log.Error(err, errMsg) - } - }, - } -} +func readyTenantNamespaces(tnt *capsulev1beta2.Tenant) []string { + namespaces := make([]string, 0, len(tnt.Status.Spaces)) -func (r *Manager) enqueueTenantsForTenantOwner( - ctx context.Context, - tenantOwner client.Object, - q workqueue.TypedRateLimitingInterface[reconcile.Request], -) { - var tenants capsulev1beta2.TenantList - if err := r.List(ctx, &tenants); err != nil { - r.Log.Error(err, "failed to list Tenants for Tenant Owner event") - - return - } - - owner, ok := tenantOwner.(*capsulev1beta2.TenantOwner) - if !ok { - return - } - - for i := range tenants.Items { - tnt := &tenants.Items[i] - - if _, found := tnt.Status.Owners.FindOwner( - owner.Spec.Name, - owner.Spec.Kind, - ); !found { + for _, ns := range tnt.Status.Spaces { + ready := ns.Conditions.GetConditionByType(meta.ReadyCondition) + if ready != nil && ready.Status != metav1.ConditionTrue { continue } - q.Add(reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: tnt.Name, - }, + terminating := ns.Conditions.GetConditionByType(meta.TerminatingCondition) + if terminating != nil && terminating.Status == metav1.ConditionTrue { + continue + } + + namespaces = append(namespaces, ns.Name) + } + + return namespaces +} + +func runForTenantNamespaces( + ctx context.Context, + tnt *capsulev1beta2.Tenant, + fn func(context.Context, string) error, +) error { + errs := make(chan error, len(tnt.Status.Spaces)) + group := new(errgroup.Group) + + for _, namespace := range readyTenantNamespaces(tnt) { + group.Go(func() error { + if err := fn(ctx, namespace); err != nil { + errs <- fmt.Errorf("namespace %q: %w", namespace, err) + } + + return nil }) } + + _ = group.Wait() + + close(errs) + + var joined []error + for err := range errs { + joined = append(joined, err) + } + + return errors.Join(joined...) } func (r *Manager) enqueueForTenantsWithCondition( @@ -171,12 +155,27 @@ func (r *Manager) pruningResources(ctx context.Context, ns string, keys []string r.Log.V(4).Info("pruning objects with label selector " + selector.String()) return retry.RetryOnConflict(retry.DefaultBackoff, func() error { - return r.DeleteAllOf(ctx, obj, &client.DeleteAllOfOptions{ + err := r.DeleteAllOf(ctx, obj, &client.DeleteAllOfOptions{ ListOptions: client.ListOptions{ LabelSelector: selector, Namespace: ns, }, DeleteOptions: client.DeleteOptions{}, }) + if err != nil { + if apierrors.IsNotFound(err) || apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) { + r.Log.V(4).Info( + "skipping pruning because target namespace or object is gone/terminating", + "namespace", ns, + "labelSelector", selector.String(), + ) + + return nil + } + + return err + } + + return nil }) } diff --git a/internal/controllers/tls/manager.go b/internal/controllers/tls/manager.go index 185c931f..2b2e979c 100644 --- a/internal/controllers/tls/manager.go +++ b/internal/controllers/tls/manager.go @@ -5,12 +5,12 @@ package tls import ( "context" + "crypto/rsa" + "crypto/x509" "fmt" - "os" "time" "github.com/go-logr/logr" - "github.com/pkg/errors" "golang.org/x/sync/errgroup" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" corev1 "k8s.io/api/core/v1" @@ -19,25 +19,27 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" - "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/projectcapsule/capsule/internal/controllers/utils" - caperrors "github.com/projectcapsule/capsule/pkg/api/errors" "github.com/projectcapsule/capsule/pkg/runtime/cert" + capsuleclient "github.com/projectcapsule/capsule/pkg/runtime/client" "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/predicates" ) const ( certificateExpirationThreshold = 3 * 24 * time.Hour certificateValidity = 6 * 30 * 24 * time.Hour - PodUpdateAnnotationName = "capsule.clastix.io/updated" + + // caPrivateKeyKey is intentionally not a Kubernetes core constant. + // The TLS Secret remains type kubernetes.io/tls, but we persist the CA key + // so serving cert renewal does not require CA rotation. + caPrivateKeyKey = "ca.key" ) type Reconciler struct { @@ -62,286 +64,536 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { }) return ctrl.NewControllerManagedBy(mgr). - For(&corev1.Secret{}, utils.NamesMatchingPredicate(r.Configuration.TLSSecretName())). + For( + &corev1.Secret{}, + builder.WithPredicates( + predicates.NamesMatchingPredicate{ + Names: []string{r.Configuration.TLSSecretName()}, + }, + ), + ). Named("capsule/tls"). - Watches(&admissionregistrationv1.ValidatingWebhookConfiguration{}, enqueueFn, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { - return object.GetName() == r.Configuration.ValidatingWebhookConfigurationName() - }))). - Watches(&admissionregistrationv1.MutatingWebhookConfiguration{}, enqueueFn, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { - return object.GetName() == r.Configuration.MutatingWebhookConfigurationName() - }))). - Watches(&apiextensionsv1.CustomResourceDefinition{}, enqueueFn, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { - return object.GetName() == r.Configuration.TenantCRDName() - }))). + Watches( + &admissionregistrationv1.ValidatingWebhookConfiguration{}, + enqueueFn, + builder.WithPredicates( + predicates.NamesMatchingPredicate{ + Names: []string{string(r.Configuration.Admission().Validating.Name)}, + }, + ), + ). + Watches( + &admissionregistrationv1.MutatingWebhookConfiguration{}, + enqueueFn, + builder.WithPredicates( + predicates.NamesMatchingPredicate{ + Names: []string{string(r.Configuration.Admission().Mutating.Name)}, + }, + ), + ). + Watches( + &apiextensionsv1.CustomResourceDefinition{}, + enqueueFn, + builder.WithPredicates( + predicates.NamesMatchingPredicate{ + Names: r.managedCRDNames(), + }, + ), + ). Complete(r) } -func (r Reconciler) ReconcileCertificates(ctx context.Context, certSecret *corev1.Secret) error { - if r.shouldUpdateCertificate(certSecret) { - r.Log.V(3).Info("Generating new TLS certificate") +func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + r.Log = r.Log.WithValues( + "Request.Namespace", request.Namespace, + "Request.Name", request.Name, + ) - ca, err := cert.GenerateCertificateAuthority() - if err != nil { - return err - } - - opts := cert.NewCertOpts(time.Now().Add(certificateValidity), fmt.Sprintf("capsule-webhook-service.%s.svc", r.Namespace)) - - crt, key, err := ca.GenerateCertificate(opts) - if err != nil { - r.Log.Error(err, "Cannot generate new TLS certificate") - - return err - } - - caCrt, _ := ca.CACertificatePem() - - certSecret.Data = map[string][]byte{ - corev1.TLSCertKey: crt.Bytes(), - corev1.TLSPrivateKeyKey: key.Bytes(), - corev1.ServiceAccountRootCAKey: caCrt.Bytes(), - } - - t := &corev1.Secret{ObjectMeta: certSecret.ObjectMeta} - - _, err = controllerutil.CreateOrUpdate(ctx, r.Client, t, func() error { - t.Data = certSecret.Data - - return nil - }) - if err != nil { - r.Log.Error(err, "cannot update Capsule TLS") - - return err - } + if request.Namespace == "" { + request.Namespace = r.Namespace } - var caBundle []byte - - var ok bool - - if caBundle, ok = certSecret.Data[corev1.ServiceAccountRootCAKey]; !ok { - return fmt.Errorf("missing %s field in %s secret", corev1.ServiceAccountRootCAKey, r.Configuration.TLSSecretName()) + if request.Name == "" { + request.Name = r.Configuration.TLSSecretName() } - r.Log.V(4).Info("Updating caBundle in webhooks and crd") + certSecret := &corev1.Secret{} + if err := r.Get(ctx, request.NamespacedName, certSecret); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } - group := new(errgroup.Group) - group.Go(func() error { - return r.updateMutatingWebhookConfiguration(ctx, caBundle) - }) - group.Go(func() error { - return r.updateValidatingWebhookConfiguration(ctx, caBundle) - }) - group.Go(func() error { - return r.updateTenantCustomResourceDefinition(ctx, "tenants.capsule.clastix.io", caBundle) - }) - group.Go(func() error { - return r.updateTenantCustomResourceDefinition(ctx, "capsuleconfigurations.capsule.clastix.io", caBundle) - }) + certSecret = &corev1.Secret{} + certSecret.Name = request.Name + certSecret.Namespace = request.Namespace + certSecret.Data = map[string][]byte{} + } - operatorPods, err := r.getOperatorPods(ctx) + if err := r.ReconcileCertificates(ctx, certSecret); err != nil { + return ctrl.Result{}, err + } + + servingCert, err := cert.GetCertificateFromBytes(certSecret.Data[corev1.TLSCertKey]) if err != nil { - if errors.As(err, &caperrors.RunningInOutOfClusterModeError{}) { - r.Log.Info("skipping annotation of Pods for cert-manager", "error", err.Error()) + return ctrl.Result{}, err + } - return nil - } + requeueTime := servingCert.NotAfter.Add(-(certificateExpirationThreshold - time.Second)) + requeueAfter := max(time.Until(requeueTime), 0) + + r.Log.V(4).Info("TLS reconciliation completed", "requeueAfter", requeueAfter.String()) + + return ctrl.Result{ + Requeue: true, + RequeueAfter: requeueAfter, + }, nil +} + +func (r *Reconciler) ReconcileCertificates(ctx context.Context, certSecret *corev1.Secret) error { + dnsName := r.webhookDNSName() + + ca, caBundle, rotateServingCert, err := r.ensureCertificateMaterial(certSecret, dnsName) + if err != nil { return err } - r.Log.V(4).Info("Updating capsule operator pods") + if rotateServingCert { + if ca == nil { + return fmt.Errorf("cannot rotate serving certificate without CA private key") + } - for _, pod := range operatorPods.Items { - p := pod + r.Log.V(3).Info("Generating new serving TLS certificate", "dnsName", dnsName) - group.Go(func() error { - return r.updateOperatorPod(ctx, p) + crt, key, err := ca.GenerateCertificate(cert.NewCertOpts( + time.Now().Add(certificateValidity), + dnsName, + )) + if err != nil { + r.Log.Error(err, "cannot generate serving TLS certificate") + + return err + } + + certSecret.Data[corev1.TLSCertKey] = crt.Bytes() + certSecret.Data[corev1.TLSPrivateKeyKey] = key.Bytes() + + if err := r.validateSecretCertificate(certSecret, dnsName); err != nil { + return err + } + + if err := r.upsertTLSSecret(ctx, certSecret); err != nil { + return err + } + } + + caBundle = certSecret.Data[corev1.ServiceAccountRootCAKey] + if len(caBundle) == 0 { + return fmt.Errorf("missing %q field in %q secret", corev1.ServiceAccountRootCAKey, r.Configuration.TLSSecretName()) + } + + r.Log.V(4).Info("Patching caBundle in webhooks and managed CRD conversions") + + patchGroup, groupCtx := errgroup.WithContext(ctx) + + patchGroup.Go(func() error { + return r.patchMutatingWebhookConfigurationCABundle(groupCtx, caBundle) + }) + + patchGroup.Go(func() error { + return r.patchValidatingWebhookConfigurationCABundle(groupCtx, caBundle) + }) + + for key, managed := range r.conversionManagedCRDs() { + patchGroup.Go(func() error { + if err := r.updateManagedCustomResourceDefinition(groupCtx, managed, caBundle); err != nil { + return fmt.Errorf("cannot update managed CRD %q (%s): %w", key, managed.Name, err) + } + + return nil }) } - if err := group.Wait(); err != nil { + return patchGroup.Wait() +} + +// ensureCertificateMaterial ensures that the Secret contains a stable CA +// certificate/key pair and decides whether the serving certificate must be +// regenerated. +// +// Important behavior: +// - Missing Secret or missing ca.key creates a new CA. +// - Existing valid CA is reused. +// - Serving certificate renewal never rotates the CA. +// - Legacy Secrets without ca.key rotate once into the stable format. +func (r *Reconciler) ensureCertificateMaterial( + certSecret *corev1.Secret, + dnsName string, +) (*cert.CapsuleCA, []byte, bool, error) { + if certSecret.Data == nil { + certSecret.Data = map[string][]byte{} + } + + caBundle := certSecret.Data[corev1.ServiceAccountRootCAKey] + caKey := certSecret.Data[caPrivateKeyKey] + tlsCrt := certSecret.Data[corev1.TLSCertKey] + tlsKey := certSecret.Data[corev1.TLSPrivateKeyKey] + + hasCA := len(caBundle) > 0 + hasCAKey := len(caKey) > 0 + hasServingCert := len(tlsCrt) > 0 && len(tlsKey) > 0 + + // Fresh empty Secret or completely broken Secret. + if !hasCA || !hasServingCert { + r.Log.Info( + "Generating new certificate authority and serving certificate", + "reason", "missing ca.crt or serving certificate", + "secret", certSecret.Name, + "namespace", certSecret.Namespace, + ) + + ca, newCABundle, newCAKey, err := generateCertificateAuthorityMaterial() + if err != nil { + return nil, nil, false, err + } + + certSecret.Data[corev1.ServiceAccountRootCAKey] = newCABundle + certSecret.Data[caPrivateKeyKey] = newCAKey + + return ca, newCABundle, true, nil + } + + // Legacy mode: + // The Secret has a CA and serving cert, but not the CA private key. + // If the serving cert is still valid and chains to ca.crt, do NOT rotate now. + // Rotating here causes a temporary caBundle/server-cert mismatch during startup. + if !hasCAKey { + if err := r.validateSecretCertificate(certSecret, dnsName); err == nil { + r.Log.Info( + "TLS Secret is using legacy CA material without ca.key; keeping existing CA and serving certificate", + "secret", certSecret.Name, + "namespace", certSecret.Namespace, + ) + + return nil, caBundle, false, nil + } + + r.Log.Info( + "TLS Secret is missing ca.key and serving certificate is invalid or expiring; rotating CA", + "secret", certSecret.Name, + "namespace", certSecret.Namespace, + ) + + ca, newCABundle, newCAKey, err := generateCertificateAuthorityMaterial() + if err != nil { + return nil, nil, false, err + } + + certSecret.Data[corev1.ServiceAccountRootCAKey] = newCABundle + certSecret.Data[caPrivateKeyKey] = newCAKey + + return ca, newCABundle, true, nil + } + + ca, err := cert.NewCertificateAuthorityFromBytes(caBundle, caKey) + if err != nil { + r.Log.Error(err, "existing CA material is invalid, regenerating CA") + + newCA, newCABundle, newCAKey, err := generateCertificateAuthorityMaterial() + if err != nil { + return nil, nil, false, err + } + + certSecret.Data[corev1.ServiceAccountRootCAKey] = newCABundle + certSecret.Data[caPrivateKeyKey] = newCAKey + + return newCA, newCABundle, true, nil + } + + if err := validateCAKeyPair(caBundle, caKey); err != nil { + r.Log.Error(err, "existing CA certificate/key pair is invalid, regenerating CA") + + newCA, newCABundle, newCAKey, err := generateCertificateAuthorityMaterial() + if err != nil { + return nil, nil, false, err + } + + certSecret.Data[corev1.ServiceAccountRootCAKey] = newCABundle + certSecret.Data[caPrivateKeyKey] = newCAKey + + return newCA, newCABundle, true, nil + } + + if err := r.validateSecretCertificate(certSecret, dnsName); err != nil { + r.Log.Info("serving certificate requires renewal", "reason", err.Error()) + + return ca, caBundle, true, nil + } + + r.Log.V(4).Info("Skipping TLS certificate generation as existing certificate is valid") + + return ca, caBundle, false, nil +} + +func generateCertificateAuthorityMaterial() (*cert.CapsuleCA, []byte, []byte, error) { + ca, err := cert.GenerateCertificateAuthority() + if err != nil { + return nil, nil, nil, err + } + + caCrt, err := ca.CACertificatePem() + if err != nil { + return nil, nil, nil, err + } + + caKey, err := ca.CAPrivateKeyPem() + if err != nil { + return nil, nil, nil, err + } + + return ca, caCrt.Bytes(), caKey.Bytes(), nil +} + +func (r *Reconciler) upsertTLSSecret(ctx context.Context, certSecret *corev1.Secret) error { + desired := &corev1.Secret{ + ObjectMeta: certSecret.ObjectMeta, + Type: corev1.SecretTypeTLS, + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, desired, func() error { + if desired.Labels == nil { + desired.Labels = map[string]string{} + } + + if desired.Annotations == nil { + desired.Annotations = map[string]string{} + } + + desired.Data = copySecretData(certSecret.Data) + + return nil + }) + if err != nil { + r.Log.Error(err, "cannot update Capsule TLS Secret") + return err } + certSecret.ObjectMeta = desired.ObjectMeta + certSecret.Data = copySecretData(desired.Data) + + return nil +} + +func (r *Reconciler) validateSecretCertificate(secret *corev1.Secret, dnsName string) error { + if secret == nil { + return fmt.Errorf("secret is nil") + } + + if secret.Data == nil { + return fmt.Errorf("secret data is nil") + } + + caBundle := secret.Data[corev1.ServiceAccountRootCAKey] + if len(caBundle) == 0 { + return fmt.Errorf("missing %q", corev1.ServiceAccountRootCAKey) + } + + leafPEM := secret.Data[corev1.TLSCertKey] + if len(leafPEM) == 0 { + return fmt.Errorf("missing %q", corev1.TLSCertKey) + } + + keyPEM := secret.Data[corev1.TLSPrivateKeyKey] + if len(keyPEM) == 0 { + return fmt.Errorf("missing %q", corev1.TLSPrivateKeyKey) + } + + leaf, key, err := cert.GetCertificateWithPrivateKeyFromBytes(leafPEM, keyPEM) + if err != nil { + return fmt.Errorf("cannot parse serving certificate/key pair: %w", err) + } + + if err := cert.ValidateCertificate(leaf, key, certificateExpirationThreshold); err != nil { + return fmt.Errorf("serving certificate is invalid or expiring: %w", err) + } + + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(caBundle) { + return fmt.Errorf("cannot parse caBundle") + } + + if _, err := leaf.Verify(x509.VerifyOptions{ + DNSName: dnsName, + Roots: roots, + KeyUsages: []x509.ExtKeyUsage{ + x509.ExtKeyUsageServerAuth, + }, + }); err != nil { + return fmt.Errorf("serving certificate does not verify against caBundle: %w", err) + } + return nil } -func (r Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { - r.Log = r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) - - certSecret := &corev1.Secret{} - - if err := r.Get(ctx, request.NamespacedName, certSecret); err != nil { - // Error reading the object - requeue the request. - return reconcile.Result{}, err - } - - if err := r.ReconcileCertificates(ctx, certSecret); err != nil { - return reconcile.Result{}, err - } - - certificate, err := cert.GetCertificateFromBytes(certSecret.Data[corev1.TLSCertKey]) +func validateCAKeyPair(caCertPEM, caKeyPEM []byte) error { + caCert, caKey, err := cert.GetCertificateWithPrivateKeyFromBytes(caCertPEM, caKeyPEM) if err != nil { - return reconcile.Result{}, err + return fmt.Errorf("cannot parse CA certificate/key pair: %w", err) + } + + if !caCert.IsCA { + return fmt.Errorf("ca.crt is not a CA certificate") + } + + if !publicKeysEqual(caCert.PublicKey, &caKey.PublicKey) { + return fmt.Errorf("ca.crt does not match ca.key") } now := time.Now() - requeueTime := certificate.NotAfter.Add(-(certificateExpirationThreshold - 1*time.Second)) - rq := requeueTime.Sub(now) + if now.Before(caCert.NotBefore) { + return fmt.Errorf("CA certificate is not valid yet") + } - r.Log.V(4).Info("Reconciliation completed, processing back in " + rq.String()) + if now.After(caCert.NotAfter.Add(-certificateExpirationThreshold)) { + return fmt.Errorf("CA certificate expired or expires soon") + } - return reconcile.Result{Requeue: true, RequeueAfter: rq}, nil + return nil } -func (r Reconciler) shouldUpdateCertificate(secret *corev1.Secret) bool { - if _, ok := secret.Data[corev1.ServiceAccountRootCAKey]; !ok { - return true +func publicKeysEqual(a any, b *rsa.PublicKey) bool { + pub, ok := a.(*rsa.PublicKey) + if !ok { + return false } - certificate, key, err := cert.GetCertificateWithPrivateKeyFromBytes(secret.Data[corev1.TLSCertKey], secret.Data[corev1.TLSPrivateKeyKey]) - if err != nil { - return true - } - - if err := cert.ValidateCertificate(certificate, key, certificateExpirationThreshold); err != nil { - r.Log.Error(err, "failed to validate certificate, generating new one") - - return true - } - - r.Log.V(4).Info("Skipping TLS certificate generation as it is still valid") - - return false -} - -// By default helm doesn't allow to use templates in CRD (https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#method-1-let-helm-do-it-for-you). -// In order to overcome this, we are setting conversion strategy in helm chart to None, and then update it with CA and namespace information. -func (r *Reconciler) updateTenantCustomResourceDefinition(ctx context.Context, name string, caBundle []byte) error { - return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { - crd := &apiextensionsv1.CustomResourceDefinition{} - - err = r.Get(ctx, types.NamespacedName{Name: name}, crd) - if err != nil { - r.Log.Error(err, "cannot retrieve CustomResourceDefinition") - - return err - } - - _, err = controllerutil.CreateOrUpdate(ctx, r.Client, crd, func() error { - crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ - Strategy: "Webhook", - Webhook: &apiextensionsv1.WebhookConversion{ - ClientConfig: &apiextensionsv1.WebhookClientConfig{ - Service: &apiextensionsv1.ServiceReference{ - Namespace: r.Namespace, - Name: "capsule-webhook-service", - Path: ptr.To("/convert"), - Port: ptr.To(int32(443)), - }, - CABundle: caBundle, - }, - ConversionReviewVersions: []string{"v1beta1", "v1beta2"}, - }, - } - - return nil - }) - - return err - }) + return pub.Equal(b) } //nolint:dupl -func (r Reconciler) updateValidatingWebhookConfiguration(ctx context.Context, caBundle []byte) error { - return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { +func (r *Reconciler) patchValidatingWebhookConfigurationCABundle(ctx context.Context, caBundle []byte) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { vw := &admissionregistrationv1.ValidatingWebhookConfiguration{} - - err = r.Get(ctx, types.NamespacedName{Name: r.Configuration.ValidatingWebhookConfigurationName()}, vw) - if err != nil { - r.Log.Error(err, "cannot retrieve ValidatingWebhookConfiguration") + if err := r.Get(ctx, types.NamespacedName{ + Name: string(r.Configuration.Admission().Validating.Name), + }, vw); err != nil { + if apierrors.IsNotFound(err) { + return nil + } return err } - for i, w := range vw.Webhooks { - // Updating CABundle only in case of an internal service reference - if w.ClientConfig.Service != nil { - vw.Webhooks[i].ClientConfig.CABundle = caBundle - } + patches := r.validatingWebhookCABundlePatches(vw.Webhooks, caBundle) + if len(patches) == 0 { + return nil } - return r.Update(ctx, vw, &client.UpdateOptions{}) + return capsuleclient.ApplyPatches(ctx, r.Client, vw, patches, "capsule-tls-controller") }) } //nolint:dupl -func (r Reconciler) updateMutatingWebhookConfiguration(ctx context.Context, caBundle []byte) error { - return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { +func (r *Reconciler) patchMutatingWebhookConfigurationCABundle(ctx context.Context, caBundle []byte) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { mw := &admissionregistrationv1.MutatingWebhookConfiguration{} - - err = r.Get(ctx, types.NamespacedName{Name: r.Configuration.MutatingWebhookConfigurationName()}, mw) - if err != nil { - r.Log.Error(err, "cannot retrieve MutatingWebhookConfiguration") - - return err - } - - for i, w := range mw.Webhooks { - // Updating CABundle only in case of an internal service reference - if w.ClientConfig.Service != nil { - mw.Webhooks[i].ClientConfig.CABundle = caBundle + if err := r.Get(ctx, types.NamespacedName{ + Name: string(r.Configuration.Admission().Mutating.Name), + }, mw); err != nil { + if apierrors.IsNotFound(err) { + return nil } + + return err } - return r.Update(ctx, mw, &client.UpdateOptions{}) + patches := r.mutatingWebhookCABundlePatches(mw.Webhooks, caBundle) + if len(patches) == 0 { + return nil + } + + return capsuleclient.ApplyPatches(ctx, r.Client, mw, patches, "capsule-tls-controller") }) } -func (r Reconciler) updateOperatorPod(ctx context.Context, pod corev1.Pod) error { - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - // Need to get latest version of pod - p := &corev1.Pod{} - - if err := r.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, p); err != nil && !apierrors.IsNotFound(err) { - r.Log.Error(err, "cannot get pod", "name", pod.Name, "namespace", pod.Namespace) - - return err - } - - if p.Annotations == nil { - p.Annotations = map[string]string{} - } - - p.Annotations[PodUpdateAnnotationName] = time.Now().Format(time.RFC3339Nano) - - if err := r.Update(ctx, p, &client.UpdateOptions{}); err != nil { - r.Log.Error(err, "cannot update pod", "name", pod.Name, "namespace", pod.Namespace) - - return err - } - +func (r *Reconciler) updateManagedCustomResourceDefinition( + ctx context.Context, + managed ManagedCRD, + caBundle []byte, +) error { + if !managed.ManageConversion { return nil + } + + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + crd := &apiextensionsv1.CustomResourceDefinition{} + if err := r.Get(ctx, types.NamespacedName{Name: managed.Name}, crd); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + before := crd.DeepCopy() + + path := managed.ConversionPath + if path == "" { + path = "/convert" + } + + versions := managed.ConversionReviewVersions + if len(versions) == 0 { + versions = []string{"v1", "v1beta1"} + } + + port := int32(443) + + crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ + Strategy: apiextensionsv1.WebhookConverter, + Webhook: &apiextensionsv1.WebhookConversion{ + ClientConfig: &apiextensionsv1.WebhookClientConfig{ + Service: &apiextensionsv1.ServiceReference{ + Namespace: r.Namespace, + Name: r.Configuration.Admission().ServiceName, + Path: &path, + Port: &port, + }, + CABundle: caBundle, + }, + ConversionReviewVersions: versions, + }, + } + + return r.Patch(ctx, crd, client.MergeFrom(before)) }) } -func (r Reconciler) getOperatorPods(ctx context.Context) (*corev1.PodList, error) { - hostname, _ := os.Hostname() +func (r *Reconciler) webhookDNSName() string { + return fmt.Sprintf("%s.%s.svc", r.Configuration.Admission().ServiceName, r.Namespace) +} - leaderPod := &corev1.Pod{} +func copySecretData(in map[string][]byte) map[string][]byte { + out := make(map[string][]byte, len(in)) - if err := r.Get(ctx, types.NamespacedName{Namespace: os.Getenv("NAMESPACE"), Name: hostname}, leaderPod); err != nil { - return nil, caperrors.RunningInOutOfClusterModeError{} + for key, value := range in { + out[key] = append([]byte(nil), value...) } - podList := &corev1.PodList{} - if err := r.List(ctx, podList, client.MatchingLabels(leaderPod.Labels)); err != nil { - r.Log.Error(err, "cannot retrieve list of Capsule pods") + return out +} - return nil, err +func equalBytes(a, b []byte) bool { + if len(a) != len(b) { + return false } - return podList, nil + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true } diff --git a/internal/controllers/tls/utils.go b/internal/controllers/tls/utils.go new file mode 100644 index 00000000..339a290c --- /dev/null +++ b/internal/controllers/tls/utils.go @@ -0,0 +1,163 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tls + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + + capsuleclient "github.com/projectcapsule/capsule/pkg/runtime/client" +) + +type ManagedCRD struct { + Name string + ManageConversion bool + ConversionPath string + ConversionReviewVersions []string +} + +func (r Reconciler) managedCRDs() map[string]ManagedCRD { + return map[string]ManagedCRD{ + "tenants": { + Name: r.Configuration.TenantCRDName(), + ManageConversion: true, + ConversionPath: "/convert", + ConversionReviewVersions: []string{"v1", "v1beta1"}, + }, + "capsuleconfigurations": { + Name: "capsuleconfigurations.capsule.clastix.io", + ManageConversion: true, + ConversionPath: "/convert", + ConversionReviewVersions: []string{"v1", "v1beta1"}, + }, + "customquotas": { + Name: "customquotas.capsule.clastix.io", + }, + "globalcustomquotas": { + Name: "globalcustomquotas.capsule.clastix.io", + }, + "globaltenantresources": { + Name: "globaltenantresources.capsule.clastix.io", + }, + "quantityledgers": { + Name: "quantityledgers.capsule.clastix.io", + }, + "resourcepoolclaims": { + Name: "resourcepoolclaims.capsule.clastix.io", + }, + "resourcepools": { + Name: "resourcepools.capsule.clastix.io", + }, + "rulestatuses": { + Name: "rulestatuses.capsule.clastix.io", + }, + "tenantowners": { + Name: "tenantowners.capsule.clastix.io", + }, + "tenantresources": { + Name: "tenantresources.capsule.clastix.io", + }, + } +} + +func (r Reconciler) managedCRDNames() []string { + crds := r.managedCRDs() + + names := make([]string, 0, len(crds)) + + for _, crd := range crds { + names = append(names, crd.Name) + } + + return names +} + +func (r Reconciler) conversionManagedCRDs() map[string]ManagedCRD { + crds := r.managedCRDs() + + out := make(map[string]ManagedCRD, len(crds)) + + for key, crd := range crds { + if crd.ManageConversion { + out[key] = crd + } + } + + return out +} + +//nolint:dupl +func (r *Reconciler) validatingWebhookCABundlePatches( + webhooks []admissionregistrationv1.ValidatingWebhook, + caBundle []byte, +) []capsuleclient.JSONPatch { + patches := make([]capsuleclient.JSONPatch, 0, len(webhooks)) + + for i := range webhooks { + if webhooks[i].ClientConfig.Service == nil { + continue + } + + if equalBytes(webhooks[i].ClientConfig.CABundle, caBundle) { + continue + } + + r.Log.V(3).Info( + "Patching webhook caBundle", + "webhook", webhooks[i].Name, + "old", certFingerprint(webhooks[i].ClientConfig.CABundle), + "new", certFingerprint(caBundle), + ) + + patches = append(patches, capsuleclient.JSONPatch{ + Operation: capsuleclient.JSONPatchAdd, + Path: fmt.Sprintf("/webhooks/%d/clientConfig/caBundle", i), + Value: caBundle, + }) + } + + return patches +} + +//nolint:dupl +func (r *Reconciler) mutatingWebhookCABundlePatches( + webhooks []admissionregistrationv1.MutatingWebhook, + caBundle []byte, +) []capsuleclient.JSONPatch { + patches := make([]capsuleclient.JSONPatch, 0, len(webhooks)) + + for i := range webhooks { + if webhooks[i].ClientConfig.Service == nil { + continue + } + + if equalBytes(webhooks[i].ClientConfig.CABundle, caBundle) { + continue + } + + r.Log.V(3).Info( + "Patching webhook caBundle", + "webhook", webhooks[i].Name, + "old", certFingerprint(webhooks[i].ClientConfig.CABundle), + "new", certFingerprint(caBundle), + ) + + patches = append(patches, capsuleclient.JSONPatch{ + Operation: capsuleclient.JSONPatchAdd, + Path: fmt.Sprintf("/webhooks/%d/clientConfig/caBundle", i), + Value: caBundle, + }) + } + + return patches +} + +func certFingerprint(pemBytes []byte) string { + sum := sha256.Sum256(pemBytes) + + return hex.EncodeToString(sum[:8]) +} diff --git a/internal/controllers/utils/gvk.go b/internal/controllers/utils/gvk.go deleted file mode 100644 index 3dbcd9af..00000000 --- a/internal/controllers/utils/gvk.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/runtime/schema" - ctrl "sigs.k8s.io/controller-runtime" -) - -func HasGVK(mapper meta.RESTMapper, gvk schema.GroupVersionKind) bool { - _, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - if err != nil { - if meta.IsNoMatchError(err) { - return false - } - - ctrl.Log.WithName("gvk-check").Error(err, "failed to check RESTMapping", "gvk", gvk.String()) - - return false - } - - return true -} diff --git a/internal/controllers/utils/predicates.go b/internal/controllers/utils/predicates.go deleted file mode 100644 index 8da24eca..00000000 --- a/internal/controllers/utils/predicates.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package utils - -import ( - "slices" - - "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api/meta" -) - -var CapsuleConfigSpecChangedPredicate = predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - oldObj, ok1 := e.ObjectOld.(*capsulev1beta2.CapsuleConfiguration) - newObj, ok2 := e.ObjectNew.(*capsulev1beta2.CapsuleConfiguration) - if !ok1 || !ok2 { - return false - } - - if len(oldObj.Spec.Administrators) != len(newObj.Spec.Administrators) { - return true - } - - return false - }, - - CreateFunc: func(e event.CreateEvent) bool { return false }, - DeleteFunc: func(e event.DeleteEvent) bool { return false }, - GenericFunc: func(e event.GenericEvent) bool { return false }, -} - -var PromotedServiceaccountPredicate = predicate.TypedFuncs[client.Object]{ - CreateFunc: func(e event.TypedCreateEvent[client.Object]) bool { - v, ok := e.Object.GetLabels()[meta.OwnerPromotionLabel] - - return ok && v == meta.ValueTrue - }, - - DeleteFunc: func(e event.TypedDeleteEvent[client.Object]) bool { - v, ok := e.Object.GetLabels()[meta.OwnerPromotionLabel] - - return ok && v == meta.ValueTrue - }, - - UpdateFunc: func(e event.TypedUpdateEvent[client.Object]) bool { - oldVal, oldOK := e.ObjectOld.GetLabels()[meta.OwnerPromotionLabel] - newVal, newOK := e.ObjectNew.GetLabels()[meta.OwnerPromotionLabel] - - return oldOK != newOK || oldVal != newVal - }, - - GenericFunc: func(event.TypedGenericEvent[client.Object]) bool { - return false - }, -} - -var UpdatedMetadataPredicate = predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { return true }, - DeleteFunc: func(e event.DeleteEvent) bool { return true }, - - UpdateFunc: func(e event.UpdateEvent) bool { - return !labelsEqual(e.ObjectOld.GetLabels(), e.ObjectNew.GetLabels()) - }, - - GenericFunc: func(e event.GenericEvent) bool { return false }, -} - -func labelsEqual(a, b map[string]string) bool { - if len(a) != len(b) { - return false - } - - for k, v := range a { - if bv, ok := b[k]; !ok || bv != v { - return false - } - } - - return true -} - -func LabelsChanged(keys []string, oldLabels, newLabels map[string]string) bool { - for _, key := range keys { - oldVal, oldOK := oldLabels[key] - newVal, newOK := newLabels[key] - - if oldOK != newOK || oldVal != newVal { - return true - } - } - - return false -} - -func NamesMatchingPredicate(names ...string) builder.Predicates { - return builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { - return slices.Contains(names, object.GetName()) - })) -} diff --git a/internal/metrics/customquota_recorder.go b/internal/metrics/customquota_recorder.go new file mode 100644 index 00000000..188ba93a --- /dev/null +++ b/internal/metrics/customquota_recorder.go @@ -0,0 +1,105 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + crtlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +type CustomQuotaRecorder struct { + ConditionGauge *prometheus.GaugeVec + ResourceUsageGauge *prometheus.GaugeVec + ResourceLimitGauge *prometheus.GaugeVec + ResourceAvailableGauge *prometheus.GaugeVec + ResourceItemUsageGauge *prometheus.GaugeVec +} + +func MustMakeCustomQuotaRecorder() *CustomQuotaRecorder { + metricsRecorder := NewCustomQuotaRecorder() + crtlmetrics.Registry.MustRegister(metricsRecorder.Collectors()...) + + return metricsRecorder +} + +func NewCustomQuotaRecorder() *CustomQuotaRecorder { + return &CustomQuotaRecorder{ + ConditionGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "custom_quota_condition", + Help: "Provides per custom quota condition status", + }, []string{"custom_quota", "target_namespace", "condition"}, + ), + ResourceUsageGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "custom_quota_resource_usage", + Help: "Current resource usage for given custom quota", + }, []string{"custom_quota", "target_namespace"}, + ), + ResourceLimitGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "custom_quota_resource_limit", + Help: "Current resource limit for given custom quota", + }, []string{"custom_quota", "target_namespace"}, + ), + ResourceAvailableGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "custom_quota_resource_available", + Help: "Available resources for given custom quota", + }, []string{"custom_quota", "target_namespace"}, + ), + ResourceItemUsageGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "custom_quota_resource_item_usage", + Help: "Claimed resources from given item", + }, []string{"custom_quota", "target_namespace", "name", "kind", "group"}, + ), + } +} + +func (r *CustomQuotaRecorder) Collectors() []prometheus.Collector { + return []prometheus.Collector{ + r.ConditionGauge, + r.ResourceUsageGauge, + r.ResourceLimitGauge, + r.ResourceAvailableGauge, + r.ResourceItemUsageGauge, + } +} + +func (r *CustomQuotaRecorder) DeleteAllMetricsForCustomQuota(name string, namespace string) { + r.ConditionGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + "target_namespace": namespace, + }) + r.ResourceUsageGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + "target_namespace": namespace, + }) + r.ResourceLimitGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + "target_namespace": namespace, + }) + r.ResourceAvailableGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + "target_namespace": namespace, + }) + r.ResourceItemUsageGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + "target_namespace": namespace, + }) +} + +func (r *CustomQuotaRecorder) DeleteConditionMetricByType(name string, namespace string, condition string) { + r.ConditionGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + "target_namespace": namespace, + "condition": condition, + }) +} diff --git a/internal/metrics/global_customquota_recorder.go b/internal/metrics/global_customquota_recorder.go new file mode 100644 index 00000000..48c29c24 --- /dev/null +++ b/internal/metrics/global_customquota_recorder.go @@ -0,0 +1,99 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + crtlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +type GlobalCustomQuotaRecorder struct { + ConditionGauge *prometheus.GaugeVec + ResourceUsageGauge *prometheus.GaugeVec + ResourceLimitGauge *prometheus.GaugeVec + ResourceAvailableGauge *prometheus.GaugeVec + ResourceItemUsageGauge *prometheus.GaugeVec +} + +func MustMakeGlobalCustomQuotaRecorder() *GlobalCustomQuotaRecorder { + metricsRecorder := NewGlobalCustomQuotaRecorder() + crtlmetrics.Registry.MustRegister(metricsRecorder.Collectors()...) + + return metricsRecorder +} + +func NewGlobalCustomQuotaRecorder() *GlobalCustomQuotaRecorder { + return &GlobalCustomQuotaRecorder{ + ConditionGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_custom_quota_condition", + Help: "Provides per global custom quota condition status", + }, []string{"custom_quota", "condition"}, + ), + ResourceUsageGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_custom_quota_resource_usage", + Help: "Current resource usage for given global custom quota", + }, []string{"custom_quota"}, + ), + ResourceLimitGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_custom_quota_resource_limit", + Help: "Current resource limit for given global custom quota", + }, []string{"custom_quota"}, + ), + ResourceAvailableGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_custom_quota_resource_available", + Help: "Available resources for given global_custom quota", + }, []string{"custom_quota"}, + ), + ResourceItemUsageGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_custom_quota_resource_item_usage", + Help: "Claimed resources from given item", + }, []string{"custom_quota", "name", "target_namespace", "kind", "group"}, + ), + } +} + +func (r *GlobalCustomQuotaRecorder) Collectors() []prometheus.Collector { + return []prometheus.Collector{ + r.ConditionGauge, + r.ResourceUsageGauge, + r.ResourceLimitGauge, + r.ResourceAvailableGauge, + r.ResourceItemUsageGauge, + } +} + +func (r *GlobalCustomQuotaRecorder) DeleteAllMetricsForGlobalCustomQuota(name string) { + r.ConditionGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + }) + r.ResourceUsageGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + }) + r.ResourceLimitGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + }) + r.ResourceAvailableGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + }) + r.ResourceItemUsageGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + }) +} + +func (r *GlobalCustomQuotaRecorder) DeleteConditionMetricByType(name string, condition string) { + r.ConditionGauge.DeletePartialMatch(map[string]string{ + "custom_quota": name, + "condition": condition, + }) +} diff --git a/internal/metrics/global_tenantresource_recorder.go b/internal/metrics/global_tenantresource_recorder.go new file mode 100644 index 00000000..ed105aad --- /dev/null +++ b/internal/metrics/global_tenantresource_recorder.go @@ -0,0 +1,84 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + crtlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +type GlobalTenantResourceRecorder struct { + resourceConditionGauge *prometheus.GaugeVec +} + +func MustMakeGlobalTenantResourceRecorder() *GlobalTenantResourceRecorder { + metricsRecorder := NewGlobalTenantResourceRecorder() + crtlmetrics.Registry.MustRegister(metricsRecorder.Collectors()...) + + return metricsRecorder +} + +func NewGlobalTenantResourceRecorder() *GlobalTenantResourceRecorder { + return &GlobalTenantResourceRecorder{ + resourceConditionGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "global_resource_condition", + Help: "The current condition status of a global tenant resource.", + }, + []string{"name", "condition"}, + ), + } +} + +func (r *GlobalTenantResourceRecorder) Collectors() []prometheus.Collector { + return []prometheus.Collector{ + r.resourceConditionGauge, + } +} + +func (r *GlobalTenantResourceRecorder) RecordConditions(resource *capsulev1beta2.GlobalTenantResource) { + for _, status := range []string{meta.ReadyCondition, meta.CordonedCondition} { + var value float64 + + cond := resource.Status.Conditions.GetConditionByType(status) + if cond == nil { + r.DeleteConditionMetricByType(resource.GetName(), status) + + continue + } + + if cond.Status == metav1.ConditionTrue { + value = 1 + } + + r.resourceConditionGauge.WithLabelValues(resource.GetName(), status).Set(value) + } +} + +func (r *GlobalTenantResourceRecorder) DeleteConditionMetrics(name string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": name, + }) +} + +func (r *GlobalTenantResourceRecorder) DeleteConditionMetricByType(name string, condition string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": name, + "condition": condition, + }) +} + +// DeleteCondition deletes the condition metrics for the ref. +func (r *GlobalTenantResourceRecorder) DeleteMetrics(resourceName string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": resourceName, + }) + + r.DeleteConditionMetrics(resourceName) +} diff --git a/internal/metrics/tenantresource_recorder.go b/internal/metrics/tenantresource_recorder.go new file mode 100644 index 00000000..efe34764 --- /dev/null +++ b/internal/metrics/tenantresource_recorder.go @@ -0,0 +1,88 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + crtlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +type TenantResourceRecorder struct { + resourceConditionGauge *prometheus.GaugeVec +} + +func MustMakeTenantResourceRecorder() *TenantResourceRecorder { + metricsRecorder := NewTenantResourceRecorder() + crtlmetrics.Registry.MustRegister(metricsRecorder.Collectors()...) + + return metricsRecorder +} + +func NewTenantResourceRecorder() *TenantResourceRecorder { + return &TenantResourceRecorder{ + resourceConditionGauge: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsPrefix, + Name: "resource_condition", + Help: "The current condition status of a tenant resource.", + }, + []string{"name", "target_namespace", "condition"}, + ), + } +} + +func (r *TenantResourceRecorder) Collectors() []prometheus.Collector { + return []prometheus.Collector{ + r.resourceConditionGauge, + } +} + +// RecordCondition records the condition as given for the ref. +func (r *TenantResourceRecorder) RecordConditions(resource *capsulev1beta2.TenantResource) { + for _, status := range []string{meta.ReadyCondition, meta.CordonedCondition} { + var value float64 + + cond := resource.Status.Conditions.GetConditionByType(status) + if cond == nil { + r.DeleteConditionMetricByType(resource.GetName(), resource.GetNamespace(), status) + + continue + } + + if cond.Status == metav1.ConditionTrue { + value = 1 + } + + r.resourceConditionGauge.WithLabelValues(resource.GetName(), resource.GetNamespace(), status).Set(value) + } +} + +func (r *TenantResourceRecorder) DeleteConditionMetrics(name string, namespace string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": name, + "target_namespace": namespace, + }) +} + +func (r *TenantResourceRecorder) DeleteConditionMetricByType(name string, namespace string, condition string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": name, + "target_namespace": namespace, + "condition": condition, + }) +} + +// DeleteCondition deletes the condition metrics for the ref. +func (r *TenantResourceRecorder) DeleteMetrics(resourceName string, resourceNamespace string) { + r.resourceConditionGauge.DeletePartialMatch(map[string]string{ + "name": resourceName, + "target_namespace": resourceNamespace, + }) + + r.DeleteConditionMetrics(resourceName, resourceNamespace) +} diff --git a/internal/webhook/cfg/handler.go b/internal/webhook/cfg/handler.go new file mode 100644 index 00000000..139c2a78 --- /dev/null +++ b/internal/webhook/cfg/handler.go @@ -0,0 +1,102 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package cfg + +import ( + "context" + + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" +) + +func Handler(configuration configuration.Configuration, handlers ...handlers.TypedHandler[*capsulev1beta2.CapsuleConfiguration]) handlers.Handler { + return &handler{ + cfg: configuration, + handlers: handlers, + } +} + +type handler struct { + cfg configuration.Configuration + handlers []handlers.TypedHandler[*capsulev1beta2.CapsuleConfiguration] +} + +//nolint:dupl +func (h *handler) OnCreate( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + config := &capsulev1beta2.CapsuleConfiguration{} + if err := decoder.Decode(req, config); err != nil { + return ad.ErroredResponse(err) + } + + for _, hndl := range h.handlers { + if response := hndl.OnCreate(c, reader, config, decoder, recorder)(ctx, req); response != nil { + return response + } + } + + return nil + } +} + +//nolint:dupl +func (h *handler) OnDelete( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + config := &capsulev1beta2.CapsuleConfiguration{} + if err := decoder.Decode(req, config); err != nil { + return ad.ErroredResponse(err) + } + + for _, hndl := range h.handlers { + if response := hndl.OnDelete(c, reader, config, decoder, recorder)(ctx, req); response != nil { + return response + } + } + + return nil + } +} + +func (h *handler) OnUpdate( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + config := &capsulev1beta2.CapsuleConfiguration{} + if err := decoder.Decode(req, config); err != nil { + return ad.ErroredResponse(err) + } + + old := &capsulev1beta2.CapsuleConfiguration{} + if err := decoder.DecodeRaw(req.OldObject, old); err != nil { + return ad.ErroredResponse(err) + } + + for _, hndl := range h.handlers { + if response := hndl.OnUpdate(c, reader, config, old, decoder, recorder)(ctx, req); response != nil { + return response + } + } + + return nil + } +} diff --git a/internal/webhook/cfg/serviceaccount.go b/internal/webhook/cfg/serviceaccount.go new file mode 100644 index 00000000..421e320f --- /dev/null +++ b/internal/webhook/cfg/serviceaccount.go @@ -0,0 +1,72 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package cfg + +import ( + "context" + + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" +) + +type serviceAccountHandler struct{} + +func ServiceAccountHandler() handlers.TypedHandler[*capsulev1beta2.CapsuleConfiguration] { + return &serviceAccountHandler{} +} + +func (h *serviceAccountHandler) OnCreate( + _ client.Client, + _ client.Reader, + cfg *capsulev1beta2.CapsuleConfiguration, + _ admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return func(_ context.Context, req admission.Request) *admission.Response { + return h.handle(cfg, req) + } +} + +func (h *serviceAccountHandler) OnDelete( + client.Client, + client.Reader, + *capsulev1beta2.CapsuleConfiguration, + admission.Decoder, + events.EventRecorder, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} + +func (h *serviceAccountHandler) OnUpdate( + _ client.Client, + _ client.Reader, + cfg *capsulev1beta2.CapsuleConfiguration, + old *capsulev1beta2.CapsuleConfiguration, + _ admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return func(_ context.Context, req admission.Request) *admission.Response { + return h.handle(cfg, req) + } +} + +func (h *serviceAccountHandler) handle(config *capsulev1beta2.CapsuleConfiguration, req admission.Request) *admission.Response { + nameSet := config.Spec.Impersonation.GlobalDefaultServiceAccount != "" + nsSet := config.Spec.Impersonation.GlobalDefaultServiceAccountNamespace != "" + + if nameSet != nsSet { + return ad.Deny( + "both globalDefaultServiceAccount and globalDefaultServiceAccountNamespace must be set together", + ) + } + + return nil +} diff --git a/internal/webhook/cfg/warnings.go b/internal/webhook/cfg/warnings.go index 65ae39fb..3132960f 100644 --- a/internal/webhook/cfg/warnings.go +++ b/internal/webhook/cfg/warnings.go @@ -12,40 +12,53 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) type warningHandler struct{} -func WarningHandler() handlers.Handler { +func WarningHandler() handlers.TypedHandler[*capsulev1beta2.CapsuleConfiguration] { return &warningHandler{} } -func (h *warningHandler) OnCreate(_ client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (h *warningHandler) OnCreate( + _ client.Client, + _ client.Reader, + cfg *capsulev1beta2.CapsuleConfiguration, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(_ context.Context, req admission.Request) *admission.Response { - return h.handle(decoder, req) + return h.handle(cfg, req) } } -func (h *warningHandler) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *warningHandler) OnDelete( + client.Client, + client.Reader, + *capsulev1beta2.CapsuleConfiguration, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *warningHandler) OnUpdate(_ client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (h *warningHandler) OnUpdate( + _ client.Client, + _ client.Reader, + cfg *capsulev1beta2.CapsuleConfiguration, + _ *capsulev1beta2.CapsuleConfiguration, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(_ context.Context, req admission.Request) *admission.Response { - return h.handle(decoder, req) + return h.handle(cfg, req) } } -func (h *warningHandler) handle(decoder admission.Decoder, req admission.Request) *admission.Response { - config := &capsulev1beta2.CapsuleConfiguration{} - if err := decoder.Decode(req, config); err != nil { - return utils.ErroredResponse(err) - } - +func (h *warningHandler) handle(config *capsulev1beta2.CapsuleConfiguration, req admission.Request) *admission.Response { response := &admission.Response{ AdmissionResponse: admissionv1.AdmissionResponse{ UID: req.UID, diff --git a/internal/webhook/customquota/calculation.go b/internal/webhook/customquota/calculation.go new file mode 100644 index 00000000..43d7298d --- /dev/null +++ b/internal/webhook/customquota/calculation.go @@ -0,0 +1,973 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 +package customquota + +import ( + "context" + "fmt" + "slices" + "sort" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/conversion" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/events" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" + controller "github.com/projectcapsule/capsule/internal/controllers/customquotas" + "github.com/projectcapsule/capsule/pkg/api/meta" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" + index "github.com/projectcapsule/capsule/pkg/runtime/indexers/customquota" + "github.com/projectcapsule/capsule/pkg/runtime/quota" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" +) + +// Might need some tuning in the. +var customAdmissionBackoff = wait.Backoff{ + Steps: 6, + Duration: 20 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, +} + +var ledgerMutationBackoff = wait.Backoff{ + Steps: 8, + Duration: 10 * time.Millisecond, + Factor: 1.6, + Jitter: 0.2, +} + +type objectCalculationHandler struct { + targetsCache *cache.CompiledTargetsCache[string] + jsonPathCache *cache.JSONPathCache +} + +func ObjectCalculationHandler( + targetsCache *cache.CompiledTargetsCache[string], + jsonPathCache *cache.JSONPathCache, +) handlers.Handler { + return &objectCalculationHandler{ + targetsCache: targetsCache, + jsonPathCache: jsonPathCache, + } +} + +func (h *objectCalculationHandler) OnCreate( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + log := log.FromContext(ctx).WithValues( + "op", "create", + "kind", req.Kind.String(), + "namespace", req.Namespace, + "requestUID", string(req.UID), + "name", req.Name, + ) + + u, err := getUnstructured(req.Object) + if err != nil { + return ad.ErroredResponse(err) + } + + var finalResp *admission.Response + + err = retry.OnError(customAdmissionBackoff, apierrors.IsConflict, func() error { + matched, err := h.matchAllQuotas(ctx, c, req, u) + if err != nil { + finalResp = ad.ErroredResponse(err) + + return nil + } + + if len(matched) == 0 { + return nil + } + + evaluated, err := h.evaluateMatchedQuotas(ctx, u, matched) + if err != nil { + finalResp = ad.Deny( + fmt.Sprintf( + "creating resource %s/%s (%s) cannot be admitted because custom quota usage could not be calculated: %v", + req.Namespace, + req.Name, + req.Kind.String(), + err, + ), + ) + + return nil + } + + type appliedReservation struct { + LedgerKey types.NamespacedName + ReservationID string + } + + applied := make([]appliedReservation, 0, len(evaluated)) + + for _, item := range evaluated { + ledgerKey := quantityLedgerKeyForMatchedQuota(item) + + reservation := buildReservation(req, u, item.Usage, item.Key) + + allowed, effectiveUsed, reserved, err := reserveCreateOnLedger( + ctx, + c, + reader, + item, + &reservation, + ) + if err != nil { + for _, a := range applied { + _ = deleteLedgerReservation(ctx, c, reader, a.LedgerKey, a.ReservationID) + } + + return err + } + + if !allowed { + for _, a := range applied { + _ = deleteLedgerReservation(ctx, c, reader, a.LedgerKey, a.ReservationID) + } + + available := item.Limit.DeepCopy() + available.Sub(effectiveUsed) + + if available.Sign() < 0 { + available = resource.MustParse("0") + } + + log.V(5).Info("denying create due to quota", + "quotaKey", item.Key, + "quotaName", item.Name, + "isGlobal", item.IsGlobal, + "requestedUsage", item.Usage.String(), + "currentUsed", effectiveUsed.String(), + "available", available.String(), + "limit", item.Limit.String(), + "inflightReserved", reserved.String(), + ) + + finalResp = ad.Deny( + fmt.Sprintf( + "creating resource exceeds limit for %s %q (requested=%s, currentUsed=%s, available=%s, limit=%s, inflightReserved=%s)", + quotaTypeName(item.IsGlobal), + item.Name, + item.Usage.String(), + effectiveUsed.String(), + available.String(), + item.Limit.String(), + reserved.String(), + ), + ) + + return nil + } + + applied = append(applied, appliedReservation{ + LedgerKey: ledgerKey, + ReservationID: reservation.ID, + }) + } + + finalResp = nil + + return nil + }) + if err != nil { + if apierrors.IsConflict(err) { + return ad.Deny( + fmt.Sprintf( + "custom quota admission could not reserve usage due to concurrent quota updates after %d attempts; please retry the request: %v", + customAdmissionBackoff.Steps, + err, + ), + ) + } + + return ad.ErroredResponse(err) + } + + return finalResp + } +} + +//nolint:gocognit,cyclop,maintidx +func (h *objectCalculationHandler) OnUpdate( + c client.Client, + reader client.Reader, + _ admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + oldObj, err := getUnstructured(req.OldObject) + if err != nil { + return ad.ErroredResponse(err) + } + + newObj, err := getUnstructured(req.Object) + if err != nil { + return ad.ErroredResponse(err) + } + + var finalResp *admission.Response + + err = retry.OnError(customAdmissionBackoff, apierrors.IsConflict, func() error { + oldMatched, err := h.matchAllQuotas(ctx, c, req, oldObj) + if err != nil { + finalResp = ad.ErroredResponse(err) + + return nil + } + + newMatched, err := h.matchAllQuotas(ctx, c, req, newObj) + if err != nil { + finalResp = ad.ErroredResponse(err) + + return nil + } + + oldEvaluated, err := h.evaluateMatchedQuotas(ctx, oldObj, oldMatched) + if err != nil { + finalResp = ad.Deny( + fmt.Sprintf( + "updating resource %s/%s (%s) cannot be admitted because previous custom quota usage could not be calculated: %v", + req.Namespace, + req.Name, + req.Kind.String(), + err, + ), + ) + + return nil + } + + newEvaluated, err := h.evaluateMatchedQuotas(ctx, newObj, newMatched) + if err != nil { + finalResp = ad.Deny( + fmt.Sprintf( + "updating resource %s/%s (%s) cannot be admitted because new custom quota usage could not be calculated: %v", + req.Namespace, + req.Name, + req.Kind.String(), + err, + ), + ) + + return nil + } + + oldByKey := evaluatedByKey(oldEvaluated) + newByKey := evaluatedByKey(newEvaluated) + + relevantChange := meta.LabelsChangedUnstructured(oldObj, newObj) || len(oldByKey) != len(newByKey) + if !relevantChange { + for key, oldItem := range oldByKey { + newItem, ok := newByKey[key] + if !ok || oldItem.Usage.Cmp(newItem.Usage) != 0 { + relevantChange = true + + break + } + } + } + + if !relevantChange { + finalResp = nil + + return nil + } + + type appliedUpdate struct { + LedgerKey types.NamespacedName + ReservationID string + OldUsage resource.Quantity + NewUsage resource.Quantity + } + + applied := make([]appliedUpdate, 0, len(oldByKey)+len(newByKey)) + + for _, key := range allKeys(oldByKey, newByKey) { + oldItem, hadOld := oldByKey[key] + newItem, hadNew := newByKey[key] + + var base evaluatedQuota + + switch { + case hadNew: + base = newItem + case hadOld: + base = oldItem + default: + continue + } + + oldUsage := resource.MustParse("0") + if hadOld { + oldUsage = oldItem.Usage.DeepCopy() + } + + newUsage := resource.MustParse("0") + if hadNew { + newUsage = newItem.Usage.DeepCopy() + } + + ledgerKey := quantityLedgerKeyForMatchedQuota(base) + + var pendingDelete *capsulev1beta2.QuantityLedgerObjectRef + if hadOld { + pendingDelete = &capsulev1beta2.QuantityLedgerObjectRef{ + APIGroup: req.Kind.Group, + APIVersion: req.Kind.Version, + Kind: req.Kind.Kind, + Namespace: oldObj.GetNamespace(), + Name: oldObj.GetName(), + UID: oldObj.GetUID(), + } + } + + var reservation *capsulev1beta2.QuantityLedgerReservation + + if hadNew && newUsage.Sign() > 0 { + r := buildReservation(req, newObj, newUsage, base.Key) + reservation = &r + } + + allowed, effectiveUsed, reserved, err := replaceUsageOnLedger( + ctx, + c, + reader, + base, + oldUsage, + newUsage, + reservation, + pendingDelete, + ) + if err != nil { + for _, v := range slices.Backward(applied) { + _ = rollbackUsageReplacementOnLedger( + ctx, + c, + reader, + v.LedgerKey, + v.ReservationID, + v.OldUsage, + v.NewUsage, + ) + } + + return err + } + + if !allowed { + for _, v := range slices.Backward(applied) { + _ = rollbackUsageReplacementOnLedger( + ctx, + c, + reader, + v.LedgerKey, + v.ReservationID, + v.OldUsage, + v.NewUsage, + ) + } + + available := base.Limit.DeepCopy() + available.Sub(effectiveUsed) + + if available.Sign() < 0 { + available = resource.MustParse("0") + } + + finalResp = ad.Deny( + fmt.Sprintf( + "updating resource exceeds limit for %s %q (requested=%s, currentUsed=%s, available=%s, limit=%s, inflightReserved=%s)", + quotaTypeName(base.IsGlobal), + base.Name, + newUsage.String(), + effectiveUsed.String(), + available.String(), + base.Limit.String(), + reserved.String(), + ), + ) + + return nil + } + + reservationID := "" + if reservation != nil { + reservationID = reservation.ID + } + + applied = append(applied, appliedUpdate{ + LedgerKey: ledgerKey, + ReservationID: reservationID, + OldUsage: oldUsage.DeepCopy(), + NewUsage: newUsage.DeepCopy(), + }) + } + + finalResp = nil + + return nil + }) + if err != nil { + if apierrors.IsConflict(err) { + return ad.Deny( + fmt.Sprintf( + "custom quota admission could not reserve usage due to concurrent quota updates after %d attempts; please retry the request: %v", + customAdmissionBackoff.Steps, + err, + ), + ) + } + + return ad.ErroredResponse(err) + } + + return finalResp + } +} + +func (h *objectCalculationHandler) OnDelete( + c client.Client, + reader client.Reader, + _ admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + oldObj, err := getUnstructured(req.OldObject) + if err != nil { + return ad.ErroredResponse(err) + } + + uid := oldObj.GetUID() + if uid == "" { + return nil + } + + objRef := capsulev1beta2.QuantityLedgerObjectRef{ + APIGroup: req.Kind.Group, + APIVersion: req.Kind.Version, + Kind: req.Kind.Kind, + Namespace: oldObj.GetNamespace(), + Name: oldObj.GetName(), + UID: uid, + } + + namespacedcq := &capsulev1beta2.CustomQuotaList{} + if err := c.List(ctx, namespacedcq, client.InNamespace(req.Namespace), client.MatchingFields{ + index.ObjectUIDIndexerFieldName: string(uid), + }); err != nil { + return ad.ErroredResponse(err) + } + + for _, nscq := range namespacedcq.Items { + ledgerKey := types.NamespacedName{ + Name: nscq.GetName(), + Namespace: nscq.GetNamespace(), + } + + if err := addLedgerPendingDelete(ctx, c, reader, ledgerKey, objRef); err != nil { + return ad.ErroredResponse(err) + } + } + + globalcq := &capsulev1beta2.GlobalCustomQuotaList{} + if err := c.List(ctx, globalcq, client.MatchingFields{ + index.ObjectUIDIndexerFieldName: string(uid), + }); err != nil { + return ad.ErroredResponse(err) + } + + for _, gcq := range globalcq.Items { + ledgerKey := types.NamespacedName{ + Name: gcq.GetName(), + Namespace: configuration.ControllerNamespace(), + } + + if err := addLedgerPendingDelete(ctx, c, reader, ledgerKey, objRef); err != nil { + return ad.ErroredResponse(err) + } + } + + return nil + } +} + +func deleteLedgerReservation( + ctx context.Context, + c client.Client, + reader client.Reader, + ledgerKey types.NamespacedName, + reservationID string, +) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + ledger := &capsulev1beta2.QuantityLedger{} + if err := reader.Get(ctx, ledgerKey, ledger); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + active := make([]capsulev1beta2.QuantityLedgerReservation, 0, len(ledger.Status.Reservations)) + released := resource.MustParse("0") + + for _, res := range ledger.Status.Reservations { + if res.ID == reservationID { + released.Add(res.Usage) + + continue + } + + active = append(active, res) + } + + if released.Sign() == 0 { + return nil + } + + allocated := ledger.Status.Allocated.DeepCopy() + allocated.Sub(released) + quota.ClampQuantityToZero(&allocated) + + reserved := resource.MustParse("0") + for _, res := range active { + reserved.Add(res.Usage) + } + + ledger.Status.Reservations = active + ledger.Status.Reserved = reserved + ledger.Status.Allocated = allocated + + return c.Status().Update(ctx, ledger) + }) +} + +func (h *objectCalculationHandler) matchAllQuotas( + ctx context.Context, + c client.Client, + req admission.Request, + u unstructured.Unstructured, +) ([]quota.MatchedQuota, error) { + namespaced, err := h.matchCustomQuotas(ctx, c, req, u) + if err != nil { + return nil, err + } + + global, err := h.matchGlobalCustomQuotas(ctx, c, req, u) + if err != nil { + return nil, err + } + + out := make([]quota.MatchedQuota, 0, len(namespaced)+len(global)) + + out = append(out, namespaced...) + + out = append(out, global...) + + sort.SliceStable(out, func(i, j int) bool { + if out[i].Limit.Cmp(out[j].Limit) != 0 { + return out[i].Limit.Cmp(out[j].Limit) < 0 + } + + if out[i].IsGlobal != out[j].IsGlobal { + return out[i].IsGlobal + } + + if out[i].Namespace != out[j].Namespace { + return out[i].Namespace < out[j].Namespace + } + + if out[i].SourceRank != out[j].SourceRank { + return out[i].SourceRank < out[j].SourceRank + } + + return out[i].Name < out[j].Name + }) + + return out, nil +} + +func (h *objectCalculationHandler) matchCustomQuotas( + ctx context.Context, + c client.Client, + req admission.Request, + u unstructured.Unstructured, +) ([]quota.MatchedQuota, error) { + if req.Namespace == "" { + return nil, nil + } + + list := &capsulev1beta2.CustomQuotaList{} + + err := c.List(ctx, list, + client.InNamespace(req.Namespace), + client.MatchingFields{ + index.TargetIndexerFieldName: req.Kind.String(), + }, + ) + if err != nil { + return nil, err + } + + if len(list.Items) == 0 { + return nil, nil + } + + objLabels := labels.Set(u.GetLabels()) + out := make([]quota.MatchedQuota, 0) + + for _, cq := range list.Items { + if !selectors.MatchesSelectors(objLabels, cq.Spec.ScopeSelectors) { + continue + } + + compiledTargets, err := h.getOrCompileCustomQuotaTargets(&cq) + if err != nil { + return nil, fmt.Errorf("compile targets for CustomQuota %s/%s: %w", cq.Namespace, cq.Name, err) + } + + for i, target := range compiledTargets { + if target.Group != req.Kind.Group || + target.Version != req.Kind.Version || + target.Kind != req.Kind.Kind { + continue + } + + matches, err := controller.MatchesCompiledSelectorsWithFields(u, target.CompiledSelectors) + if err != nil { + return nil, fmt.Errorf( + "evaluate selectors for %s/%s on CustomQuota %s/%s: %w", + u.GetNamespace(), + u.GetName(), + cq.Namespace, + cq.Name, + err, + ) + } + + if !matches { + continue + } + + out = append(out, quota.MatchedQuota{ + Key: controller.MakeCustomQuotaCacheKey(cq.Namespace, cq.Name), + Name: cq.Name, + Namespace: cq.Namespace, + Path: target.Path, + CompiledPath: target.CompiledPath, + Operation: target.Operation, + Limit: cq.Spec.Limit.DeepCopy(), + Used: cq.Status.Usage.Used.DeepCopy(), + IsGlobal: false, + SourceRank: i, + }) + } + } + + return out, nil +} + +func (h *objectCalculationHandler) matchGlobalCustomQuotas( + ctx context.Context, + c client.Client, + req admission.Request, + u unstructured.Unstructured, +) ([]quota.MatchedQuota, error) { + list := &capsulev1beta2.GlobalCustomQuotaList{} + + err := c.List(ctx, list, client.MatchingFields{ + index.TargetIndexerFieldName: req.Kind.String(), + }) + if err != nil { + return nil, err + } + + if len(list.Items) == 0 { + return nil, nil + } + + objLabels := labels.Set(u.GetLabels()) + + out := make([]quota.MatchedQuota, 0) + + for _, gcq := range list.Items { + if !gcq.Status.NamespacePresent("*") && !gcq.Status.NamespacePresent(req.Namespace) { + continue + } + + if !selectors.MatchesSelectors(objLabels, gcq.Spec.ScopeSelectors) { + continue + } + + compiledTargets, err := h.getOrCompileGlobalCustomQuotaTargets(&gcq) + if err != nil { + return nil, fmt.Errorf("compile targets for GlobalCustomQuota %s: %w", gcq.Name, err) + } + + for i, target := range compiledTargets { + if target.Group != req.Kind.Group || + target.Version != req.Kind.Version || + target.Kind != req.Kind.Kind { + continue + } + + matches, err := controller.MatchesCompiledSelectorsWithFields(u, target.CompiledSelectors) + if err != nil { + return nil, fmt.Errorf( + "evaluate selectors for %s/%s on GlobalCustomQuota %s: %w", + u.GetNamespace(), + u.GetName(), + gcq.Name, + err, + ) + } + + if !matches { + continue + } + + out = append(out, quota.MatchedQuota{ + Key: controller.MakeGlobalCustomQuotaCacheKey(gcq.Name), + Name: gcq.Name, + Namespace: "", + Path: target.Path, + CompiledPath: target.CompiledPath, + Operation: target.Operation, + Limit: gcq.Spec.Limit.DeepCopy(), + Used: gcq.Status.Usage.Used.DeepCopy(), + IsGlobal: true, + SourceRank: i, + }) + } + } + + return out, nil +} + +func getUnstructured(rawExt runtime.RawExtension) (unstructured.Unstructured, error) { + var ( + obj runtime.Object + scope conversion.Scope + ) + + err := runtime.Convert_runtime_RawExtension_To_runtime_Object(&rawExt, &obj, scope) + if err != nil { + return unstructured.Unstructured{}, err + } + + innerObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return unstructured.Unstructured{}, err + } + + u := unstructured.Unstructured{Object: innerObj} + + return u, nil +} + +func quotaTypeName(global bool) string { + if global { + return "GlobalCustomQuota" + } + + return "CustomQuota" +} + +type evaluatedQuota struct { + quota.MatchedQuota + + Usage resource.Quantity +} + +func (h *objectCalculationHandler) evaluateMatchedQuotas( + ctx context.Context, + u unstructured.Unstructured, + matched []quota.MatchedQuota, +) ([]evaluatedQuota, error) { + log := log.FromContext(ctx) + + usageByPath := make(map[string]resource.Quantity, len(matched)) + + for _, mq := range matched { + // count does not use a path + if mq.Operation == quota.OpCount { + continue + } + + if _, ok := usageByPath[mq.Path]; ok { + continue + } + + usage, err := quota.ParseQuantityFromUnstructured(u, mq.CompiledPath) + if err != nil { + return nil, fmt.Errorf( + "%s %q source path %q op %q did not resolve to a valid quantity: %w", + quotaTypeName(mq.IsGlobal), + mq.Name, + mq.Path, + mq.Operation, + err, + ) + } + + log.V(5).Info("parsed usage", "path", mq.Path, "parsed", usage.String()) + + usageByPath[mq.Path] = usage + } + + byKey := make(map[string]evaluatedQuota, len(matched)) + + order := make([]string, 0, len(matched)) + + for _, mq := range matched { + ev, ok := byKey[mq.Key] + if !ok { + ev = evaluatedQuota{ + MatchedQuota: mq, + Usage: resource.MustParse("0"), + } + + order = append(order, mq.Key) + } + + var usage resource.Quantity + + switch mq.Operation { + case quota.OpCount: + usage = *resource.NewQuantity(1, resource.DecimalSI) + + case quota.OpSub: + usage = usageByPath[mq.Path].DeepCopy() + usage.Neg() + ev.Usage.Add(usage) + quota.ClampQuantityToZero(&ev.Usage) + byKey[mq.Key] = ev + + continue + + case quota.OpAdd: + usage = usageByPath[mq.Path].DeepCopy() + + default: + return nil, fmt.Errorf("unsupported quota operation %q for key %q", mq.Operation, mq.Key) + } + + ev.Usage.Add(usage) + byKey[mq.Key] = ev + } + + out := make([]evaluatedQuota, 0, len(order)) + for _, key := range order { + out = append(out, byKey[key]) + } + + return out, nil +} + +func addLedgerPendingDelete( + ctx context.Context, + c client.Client, + reader client.Reader, + ledgerKey types.NamespacedName, + objRef capsulev1beta2.QuantityLedgerObjectRef, +) error { + return retry.RetryOnConflict(ledgerMutationBackoff, func() error { + ledger := &capsulev1beta2.QuantityLedger{} + if err := reader.Get(ctx, ledgerKey, ledger); err != nil { + return err + } + + now := metav1.Now() + + for _, pd := range ledger.Status.PendingDeletes { + if pd.ObjectRef.UID != "" && pd.ObjectRef.UID == objRef.UID { + return nil + } + } + + ledger.Status.PendingDeletes = append(ledger.Status.PendingDeletes, capsulev1beta2.QuantityLedgerPendingDelete{ + ObjectRef: objRef, + CreatedAt: now, + }) + + return c.Status().Update(ctx, ledger) + }) +} + +func (h *objectCalculationHandler) getOrCompileCustomQuotaTargets( + cq *capsulev1beta2.CustomQuota, +) ([]cache.CompiledTarget, error) { + key := controller.MakeCustomQuotaCacheKey(cq.Namespace, cq.Name) + + return h.targetsCache.GetOrBuild(key, func() ([]cache.CompiledTarget, error) { + targets := make([]capsulev1beta2.CustomQuotaStatusTarget, 0, len(cq.Spec.Sources)) + for _, src := range cq.Spec.Sources { + targets = append(targets, capsulev1beta2.CustomQuotaStatusTarget{ + GroupVersionKind: metav1.GroupVersionKind(src.GroupVersionKind()), + CustomQuotaSpecSourceConfig: src.CustomQuotaSpecSourceConfig, + }) + } + + return controller.CompileTargets(h.jsonPathCache, targets) + }) +} + +func (h *objectCalculationHandler) getOrCompileGlobalCustomQuotaTargets( + gcq *capsulev1beta2.GlobalCustomQuota, +) ([]cache.CompiledTarget, error) { + key := controller.MakeGlobalCustomQuotaCacheKey(gcq.Name) + + return h.targetsCache.GetOrBuild(key, func() ([]cache.CompiledTarget, error) { + targets := make([]capsulev1beta2.CustomQuotaStatusTarget, 0, len(gcq.Spec.Sources)) + for _, src := range gcq.Spec.Sources { + targets = append(targets, capsulev1beta2.CustomQuotaStatusTarget{ + GroupVersionKind: metav1.GroupVersionKind(src.GroupVersionKind()), + CustomQuotaSpecSourceConfig: src.CustomQuotaSpecSourceConfig, + }) + } + + return controller.CompileTargets(h.jsonPathCache, targets) + }) +} + +func evaluatedByKey(in []evaluatedQuota) map[string]evaluatedQuota { + out := make(map[string]evaluatedQuota, len(in)) + for _, item := range in { + existing, ok := out[item.Key] + if !ok { + copyItem := item + copyItem.Usage = item.Usage.DeepCopy() + out[item.Key] = copyItem + + continue + } + + existing.Usage.Add(item.Usage) + quota.ClampQuantityToZero(&existing.Usage) + out[item.Key] = existing + } + + return out +} diff --git a/internal/webhook/customquota/customquota_validating.go b/internal/webhook/customquota/customquota_validating.go new file mode 100644 index 00000000..2aab52dd --- /dev/null +++ b/internal/webhook/customquota/customquota_validating.go @@ -0,0 +1,133 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 +package customquota + +import ( + "context" + "fmt" + + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" + controller "github.com/projectcapsule/capsule/internal/controllers/customquotas" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" + "github.com/projectcapsule/capsule/pkg/runtime/quota" +) + +type customQuotaValidationHandler struct { + targetsCache *cache.CompiledTargetsCache[string] + jsonPathCache *cache.JSONPathCache +} + +func CustomQuotaValidationHandler( + targetsCache *cache.CompiledTargetsCache[string], + jsonPathCache *cache.JSONPathCache, +) handlers.Handler { + return &customQuotaValidationHandler{ + targetsCache: targetsCache, + jsonPathCache: jsonPathCache, + } +} + +//nolint:dupl +func (h *customQuotaValidationHandler) OnCreate( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return func(_ context.Context, req admission.Request) *admission.Response { + q := &capsulev1beta2.CustomQuota{} + + if err := decoder.Decode(req, q); err != nil { + return ad.ErroredResponse(fmt.Errorf("failed to decode new object: %w", err)) + } + + if err := quota.ValidateQuantity(q.Spec.Limit); err != nil { + response := admission.Denied(fmt.Sprintf("invalid spec.limit: %v", err)) + + return &response + } + + return nil + } +} + +// Invalidate Cache. +func (h *customQuotaValidationHandler) OnDelete( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return func(_ context.Context, req admission.Request) *admission.Response { + obj := &capsulev1beta2.CustomQuota{} + if err := decoder.DecodeRaw(req.OldObject, obj); err != nil { + return ad.ErroredResponse(err) + } + + key := controller.MakeCustomQuotaCacheKey(obj.GetNamespace(), obj.GetName()) + + if h.targetsCache != nil { + h.targetsCache.Delete(key) + } + + h.jsonPathCache.DeleteMany(obj.Spec.CollectJSONPathExpressions()...) + + return nil + } +} + +//nolint:dupl +func (h *customQuotaValidationHandler) OnUpdate( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return func(_ context.Context, req admission.Request) *admission.Response { + oldQuota := &capsulev1beta2.CustomQuota{} + newQuota := &capsulev1beta2.CustomQuota{} + + if err := decoder.DecodeRaw(req.OldObject, oldQuota); err != nil { + return ad.ErroredResponse(fmt.Errorf("failed to decode old object: %w", err)) + } + + if err := decoder.Decode(req, newQuota); err != nil { + return ad.ErroredResponse(fmt.Errorf("failed to decode new object: %w", err)) + } + + if err := quota.ValidateQuantity(newQuota.Spec.Limit); err != nil { + return ad.Deny(fmt.Sprintf("invalid spec.limit: %v", err)) + } + + used := oldQuota.Status.Usage.Used + + // No recorded usage: allow normal mutation rules below. + hasUsage := used.Sign() > 0 + + if hasUsage { + if sourcesChanged(oldQuota.Spec.Sources, newQuota.Spec.Sources) { + return ad.Deny( + fmt.Sprintf("spec.sources cannot be changed while usage is recorded (usage: %s); create a new CustomQuota instead", used.String()), + ) + } + + if newQuota.Spec.Limit.Cmp(used) < 0 { + return ad.Deny( + fmt.Sprintf( + "spec.limit cannot be lowered below current usage (%s); requested limit: %s", + used.String(), + newQuota.Spec.Limit.String(), + ), + ) + } + } + + return nil + } +} diff --git a/internal/webhook/customquota/globalcustomquota_validating.go b/internal/webhook/customquota/globalcustomquota_validating.go new file mode 100644 index 00000000..167443dd --- /dev/null +++ b/internal/webhook/customquota/globalcustomquota_validating.go @@ -0,0 +1,134 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 +package customquota + +import ( + "context" + "fmt" + + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/internal/cache" + controller "github.com/projectcapsule/capsule/internal/controllers/customquotas" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" + "github.com/projectcapsule/capsule/pkg/runtime/quota" +) + +type globalCustomQuotaValidationHandler struct { + targetsCache *cache.CompiledTargetsCache[string] + jsonPathCache *cache.JSONPathCache +} + +func GlobalCustomQuotaValidationHandler( + targetsCache *cache.CompiledTargetsCache[string], + jsonPathCache *cache.JSONPathCache, +) handlers.Handler { + return &globalCustomQuotaValidationHandler{ + targetsCache: targetsCache, + jsonPathCache: jsonPathCache, + } +} + +//nolint:dupl +func (h *globalCustomQuotaValidationHandler) OnCreate( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return func(_ context.Context, req admission.Request) *admission.Response { + q := &capsulev1beta2.GlobalCustomQuota{} + + if err := decoder.Decode(req, q); err != nil { + return ad.ErroredResponse(fmt.Errorf("failed to decode new object: %w", err)) + } + + if err := quota.ValidateQuantity(q.Spec.Limit); err != nil { + response := admission.Denied(fmt.Sprintf("invalid spec.limit: %v", err)) + + return &response + } + + return nil + } +} + +func (h *globalCustomQuotaValidationHandler) OnDelete( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return func(_ context.Context, req admission.Request) *admission.Response { + obj := &capsulev1beta2.GlobalCustomQuota{} + if err := decoder.DecodeRaw(req.OldObject, obj); err != nil { + return ad.ErroredResponse(err) + } + + key := controller.MakeGlobalCustomQuotaCacheKey(obj.GetName()) + + if h.targetsCache != nil { + h.targetsCache.Delete(key) + } + + h.jsonPathCache.DeleteMany(obj.Spec.CollectJSONPathExpressions()...) + + return nil + } +} + +//nolint:dupl +func (h *globalCustomQuotaValidationHandler) OnUpdate( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return func(_ context.Context, req admission.Request) *admission.Response { + oldQuota := &capsulev1beta2.GlobalCustomQuota{} + newQuota := &capsulev1beta2.GlobalCustomQuota{} + + if err := decoder.DecodeRaw(req.OldObject, oldQuota); err != nil { + return ad.ErroredResponse(fmt.Errorf("failed to decode old object: %w", err)) + } + + if err := decoder.Decode(req, newQuota); err != nil { + return ad.ErroredResponse(fmt.Errorf("failed to decode new object: %w", err)) + } + + if err := quota.ValidateQuantity(newQuota.Spec.Limit); err != nil { + return ad.Deny( + fmt.Sprintf("invalid spec.limit: %v", err), + ) + } + + used := oldQuota.Status.Usage.Used + + // No recorded usage: allow normal mutation rules below. + hasUsage := used.Sign() > 0 + + if hasUsage { + if sourcesChanged(oldQuota.Spec.Sources, newQuota.Spec.Sources) { + return ad.Deny( + fmt.Sprintf("spec.sources cannot be changed while usage is recorded (usage: %s); create a new CustomQuota instead", used.String()), + ) + } + + if newQuota.Spec.Limit.Cmp(used) < 0 { + return ad.Deny( + fmt.Sprintf( + "spec.limit cannot be lowered below current usage (%s); requested limit: %s", + used.String(), + newQuota.Spec.Limit.String(), + ), + ) + } + } + + return nil + } +} diff --git a/internal/webhook/customquota/utils.go b/internal/webhook/customquota/utils.go new file mode 100644 index 00000000..3250e3f5 --- /dev/null +++ b/internal/webhook/customquota/utils.go @@ -0,0 +1,357 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 +package customquota + +import ( + "context" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/quota" +) + +func quantityLedgerKeyForMatchedQuota(item evaluatedQuota) types.NamespacedName { + if item.IsGlobal { + return types.NamespacedName{ + Name: item.Name, + Namespace: configuration.ControllerNamespace(), + } + } + + return types.NamespacedName{ + Name: item.Name, + Namespace: item.Namespace, + } +} + +func reserveCreateOnLedger( + ctx context.Context, + c client.Client, + reader client.Reader, + item evaluatedQuota, + reservation *capsulev1beta2.QuantityLedgerReservation, +) (bool, resource.Quantity, resource.Quantity, error) { + var ( + allowed bool + effectiveUsed resource.Quantity + reserved resource.Quantity + ) + + ledgerKey := quantityLedgerKeyForMatchedQuota(item) + + err := retry.RetryOnConflict(ledgerMutationBackoff, func() error { + ledger := &capsulev1beta2.QuantityLedger{} + if err := reader.Get(ctx, ledgerKey, ledger); err != nil { + return err + } + + now := metav1.Now() + + allocated := ledger.Status.Allocated.DeepCopy() + if allocated.IsZero() { + allocated = resource.MustParse("0") + } + + requested := reservation.Usage.DeepCopy() + + // Idempotency: if this admission request already has a reservation, + // do not increment Allocated a second time. + activeReservations := make([]capsulev1beta2.QuantityLedgerReservation, 0, len(ledger.Status.Reservations)+1) + foundReservation := false + + for _, existing := range ledger.Status.Reservations { + if existing.ExpiresAt != nil && existing.ExpiresAt.Before(&now) { + continue + } + + if existing.ID == reservation.ID { + foundReservation = true + + // Keep Allocated unchanged for retry/idempotent update. + existing.Usage = reservation.Usage.DeepCopy() + existing.ObjectRef = reservation.ObjectRef + existing.UpdatedAt = now + existing.ExpiresAt = reservation.ExpiresAt + } + + activeReservations = append(activeReservations, existing) + } + + nextAllocated := allocated.DeepCopy() + if !foundReservation { + nextAllocated.Add(requested) + } + + if nextAllocated.Cmp(item.Limit) > 0 { + allowed = false + effectiveUsed = nextAllocated + reserved = allocated + + return nil + } + + if !foundReservation { + activeReservations = append(activeReservations, *reservation) + } + + newReserved := resource.MustParse("0") + for _, r := range activeReservations { + newReserved.Add(r.Usage) + } + + ledger.Status.Reservations = activeReservations + ledger.Status.Reserved = newReserved + ledger.Status.Allocated = nextAllocated + + if err := c.Status().Update(ctx, ledger); err != nil { + return err + } + + allowed = true + effectiveUsed = nextAllocated + reserved = newReserved + + return nil + }) + + return allowed, effectiveUsed, reserved, err +} + +func replaceUsageOnLedger( + ctx context.Context, + c client.Client, + reader client.Reader, + item evaluatedQuota, + oldUsage resource.Quantity, + newUsage resource.Quantity, + reservation *capsulev1beta2.QuantityLedgerReservation, + pendingDelete *capsulev1beta2.QuantityLedgerObjectRef, +) (bool, resource.Quantity, resource.Quantity, error) { + var ( + allowed bool + effectiveUsed resource.Quantity + reserved resource.Quantity + ) + + ledgerKey := quantityLedgerKeyForMatchedQuota(item) + + err := retry.RetryOnConflict(ledgerMutationBackoff, func() error { + ledger := &capsulev1beta2.QuantityLedger{} + if err := reader.Get(ctx, ledgerKey, ledger); err != nil { + return err + } + + now := metav1.Now() + + activeReservations := make([]capsulev1beta2.QuantityLedgerReservation, 0, len(ledger.Status.Reservations)+1) + foundReservation := false + + for _, existing := range ledger.Status.Reservations { + if existing.ExpiresAt != nil && existing.ExpiresAt.Before(&now) { + continue + } + + if reservation != nil && existing.ID == reservation.ID { + foundReservation = true + existing.Usage = reservation.Usage.DeepCopy() + existing.ObjectRef = reservation.ObjectRef + existing.UpdatedAt = now + existing.ExpiresAt = reservation.ExpiresAt + } + + activeReservations = append(activeReservations, existing) + } + + if reservation != nil && !foundReservation { + activeReservations = append(activeReservations, *reservation) + } + + activeDeletes := make([]capsulev1beta2.QuantityLedgerPendingDelete, 0, len(ledger.Status.PendingDeletes)+1) + activeDeletes = append(activeDeletes, ledger.Status.PendingDeletes...) + + if pendingDelete != nil { + exists := false + + for _, pd := range activeDeletes { + if pd.ObjectRef.UID != "" && pd.ObjectRef.UID == pendingDelete.UID { + exists = true + + break + } + } + + if !exists { + activeDeletes = append(activeDeletes, capsulev1beta2.QuantityLedgerPendingDelete{ + ObjectRef: *pendingDelete, + CreatedAt: now, + }) + } + } + + nextAllocated := ledger.Status.Allocated.DeepCopy() + if nextAllocated.IsZero() { + nextAllocated = resource.MustParse("0") + } + + nextAllocated.Sub(oldUsage) + quota.ClampQuantityToZero(&nextAllocated) + + nextAllocated.Add(newUsage) + + if nextAllocated.Cmp(item.Limit) > 0 { + allowed = false + effectiveUsed = nextAllocated + reserved = ledger.Status.Reserved.DeepCopy() + + return nil + } + + newReserved := resource.MustParse("0") + for _, res := range activeReservations { + newReserved.Add(res.Usage) + } + + ledger.Status.Reservations = activeReservations + ledger.Status.PendingDeletes = activeDeletes + ledger.Status.Reserved = newReserved + ledger.Status.Allocated = nextAllocated + + if err := c.Status().Update(ctx, ledger); err != nil { + return err + } + + allowed = true + effectiveUsed = nextAllocated + reserved = newReserved + + return nil + }) + + return allowed, effectiveUsed, reserved, err +} + +func rollbackUsageReplacementOnLedger( + ctx context.Context, + c client.Client, + reader client.Reader, + ledgerKey types.NamespacedName, + reservationID string, + oldUsage resource.Quantity, + newUsage resource.Quantity, +) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + ledger := &capsulev1beta2.QuantityLedger{} + if err := reader.Get(ctx, ledgerKey, ledger); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + activeReservations := make([]capsulev1beta2.QuantityLedgerReservation, 0, len(ledger.Status.Reservations)) + + for _, res := range ledger.Status.Reservations { + if reservationID != "" && res.ID == reservationID { + continue + } + + activeReservations = append(activeReservations, res) + } + + allocated := ledger.Status.Allocated.DeepCopy() + if allocated.IsZero() { + allocated = resource.MustParse("0") + } + + allocated.Sub(newUsage) + quota.ClampQuantityToZero(&allocated) + allocated.Add(oldUsage) + + newReserved := resource.MustParse("0") + for _, res := range activeReservations { + newReserved.Add(res.Usage) + } + + ledger.Status.Allocated = allocated + ledger.Status.Reservations = activeReservations + ledger.Status.Reserved = newReserved + + return c.Status().Update(ctx, ledger) + }) +} + +func buildReservation( + req admission.Request, + u unstructured.Unstructured, + usage resource.Quantity, + quotaKey string, +) capsulev1beta2.QuantityLedgerReservation { + now := metav1.Now() + expiresAt := metav1.NewTime(now.Add(2 * time.Minute)) + + return capsulev1beta2.QuantityLedgerReservation{ + ID: fmt.Sprintf("%s/%s", req.UID, quotaKey), + ObjectRef: capsulev1beta2.QuantityLedgerObjectRef{ + APIGroup: req.Kind.Group, + APIVersion: req.Kind.Version, + Kind: req.Kind.Kind, + Namespace: u.GetNamespace(), + Name: u.GetName(), + UID: u.GetUID(), + }, + Usage: usage.DeepCopy(), + CreatedAt: now, + UpdatedAt: now, + ExpiresAt: &expiresAt, + } +} + +func allKeys[K comparable, V any](a map[K]V, b map[K]V) []K { + out := make([]K, 0, len(a)+len(b)) + seen := make(map[K]struct{}, len(a)+len(b)) + + for k := range a { + seen[k] = struct{}{} + + out = append(out, k) + } + + for k := range b { + if _, ok := seen[k]; ok { + continue + } + + out = append(out, k) + } + + return out +} + +func sourcesChanged(a, b []capsulev1beta2.CustomQuotaSpecSource) bool { + if len(a) != len(b) { + return true + } + + for i := range a { + if a[i].APIVersion != b[i].APIVersion || + a[i].Kind != b[i].Kind || + a[i].Path != b[i].Path || + a[i].Operation != b[i].Operation { + return true + } + } + + return false +} diff --git a/internal/webhook/defaults/gateway.go b/internal/webhook/defaults/gateway.go index 8032f58d..5776fabb 100644 --- a/internal/webhook/defaults/gateway.go +++ b/internal/webhook/defaults/gateway.go @@ -16,19 +16,26 @@ import ( capsulegateway "github.com/projectcapsule/capsule/internal/webhook/gateway" "github.com/projectcapsule/capsule/internal/webhook/utils" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" ) -func mutateGatewayDefaults(ctx context.Context, req admission.Request, c client.Client, decoder admission.Decoder, namespce string) *admission.Response { +func mutateGatewayDefaults( + ctx context.Context, + req admission.Request, + c client.Client, + decoder admission.Decoder, + namespce string, +) *admission.Response { gatewayObj := &gatewayv1.Gateway{} if err := decoder.Decode(req, gatewayObj); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } gatewayObj.SetNamespace(namespce) tnt, err := capsulegateway.TenantFromGateway(ctx, c, gatewayObj) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } if tnt == nil { @@ -49,17 +56,13 @@ func mutateGatewayDefaults(ctx context.Context, req admission.Request, c client. if gatewayObj.Spec.GatewayClassName == ("") { mutate = true } else { - response := admission.Denied(caperrors.NewGatewayError(gatewayObj.Spec.GatewayClassName, err).Error()) - - return &response + return ad.Deny(caperrors.NewGatewayError(gatewayObj.Spec.GatewayClassName, err).Error()) } } if gatewayClass != nil && gatewayClass.Name != allowed.Default { if err != nil && !k8serrors.IsNotFound(err) { - response := admission.Denied(caperrors.NewGatewayClassError(gatewayClass.Name, err).Error()) - - return &response + return ad.Deny(caperrors.NewGatewayClassError(gatewayClass.Name, err).Error()) } } else { mutate = true diff --git a/internal/webhook/defaults/handler.go b/internal/webhook/defaults/handler.go index daec3d4a..d49d9c12 100644 --- a/internal/webhook/defaults/handler.go +++ b/internal/webhook/defaults/handler.go @@ -28,21 +28,36 @@ func Handler(cfg configuration.Configuration, version *version.Version) handlers } } -func (h *handler) OnCreate(client client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (h *handler) OnCreate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.mutate(ctx, req, client, decoder) + return h.mutate(ctx, req, c, decoder) } } -func (h *handler) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *handler) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *handler) OnUpdate(client client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (h *handler) OnUpdate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.mutate(ctx, req, client, decoder) + return h.mutate(ctx, req, c, decoder) } } diff --git a/internal/webhook/defaults/ingress.go b/internal/webhook/defaults/ingress.go index eaef66f9..ea1e4ce3 100644 --- a/internal/webhook/defaults/ingress.go +++ b/internal/webhook/defaults/ingress.go @@ -17,12 +17,20 @@ import ( capsuleingress "github.com/projectcapsule/capsule/internal/webhook/ingress" "github.com/projectcapsule/capsule/internal/webhook/utils" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" ) -func mutateIngressDefaults(ctx context.Context, req admission.Request, version *version.Version, c client.Client, decoder admission.Decoder, namespace string) *admission.Response { +func mutateIngressDefaults( + ctx context.Context, + req admission.Request, + version *version.Version, + c client.Client, + decoder admission.Decoder, + namespace string, +) *admission.Response { ingress, err := capsuleingress.FromRequest(req, decoder) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } ingress.SetNamespace(namespace) @@ -31,7 +39,7 @@ func mutateIngressDefaults(ctx context.Context, req admission.Request, version * tnt, err = capsuleingress.TenantFromIngress(ctx, c, ingress) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } if tnt == nil { @@ -50,9 +58,7 @@ func mutateIngressDefaults(ctx context.Context, req admission.Request, version * if ingressClassName := ingress.IngressClass(); ingressClassName != nil && *ingressClassName != allowed.Default { if ingressClass, err = utils.GetIngressClassByName(ctx, version, c, ingressClassName); err != nil && !k8serrors.IsNotFound(err) { - response := admission.Denied(caperrors.NewIngressClassError(*ingressClassName, err).Error()) - - return &response + return ad.Deny(caperrors.NewIngressClassError(*ingressClassName, err).Error()) } } else { mutate = true diff --git a/internal/webhook/defaults/pods.go b/internal/webhook/defaults/pods.go index e783ac17..0014e909 100644 --- a/internal/webhook/defaults/pods.go +++ b/internal/webhook/defaults/pods.go @@ -10,27 +10,33 @@ import ( corev1 "k8s.io/api/core/v1" schedulev1 "k8s.io/api/scheduling/v1" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/api" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/tenant" ) -func mutatePodDefaults(ctx context.Context, req admission.Request, c client.Client, decoder admission.Decoder, namespace string) *admission.Response { +func mutatePodDefaults( + ctx context.Context, + req admission.Request, + c client.Client, + decoder admission.Decoder, + namespace string, +) *admission.Response { var pod corev1.Pod if err := decoder.Decode(req, &pod); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } pod.SetNamespace(namespace) tnt, tErr := tenant.TenantByStatusNamespace(ctx, c, pod.Namespace) if tErr != nil { - return utils.ErroredResponse(tErr) + return ad.ErroredResponse(tErr) } else if tnt == nil { return nil } @@ -39,7 +45,7 @@ func mutatePodDefaults(ctx context.Context, req admission.Request, c client.Clie pcMutated, pcErr := handlePriorityClassDefault(ctx, c, tnt.Spec.PriorityClasses, &pod) if pcErr != nil { - return utils.ErroredResponse(pcErr) + return ad.ErroredResponse(pcErr) } rcMutated := handleRuntimeClassDefault(tnt.Spec.RuntimeClasses, &pod) @@ -50,10 +56,12 @@ func mutatePodDefaults(ctx context.Context, req admission.Request, c client.Clie var marshaled []byte if marshaled, err = json.Marshal(pod); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - return ptr.To(admission.PatchResponseFromRaw(req.Object.Raw, marshaled)) + resp := admission.PatchResponseFromRaw(req.Object.Raw, marshaled) + + return &resp } func handleRuntimeClassDefault(allowed *api.DefaultAllowedListSpec, pod *corev1.Pod) (mutated bool) { @@ -77,7 +85,12 @@ func handleRuntimeClassDefault(allowed *api.DefaultAllowedListSpec, pod *corev1. } } -func handlePriorityClassDefault(ctx context.Context, c client.Client, allowed *api.DefaultAllowedListSpec, pod *corev1.Pod) (mutated bool, err error) { +func handlePriorityClassDefault( + ctx context.Context, + c client.Reader, + allowed *api.DefaultAllowedListSpec, + pod *corev1.Pod, +) (mutated bool, err error) { if allowed == nil || allowed.Default == "" { return false, nil } diff --git a/internal/webhook/defaults/storage.go b/internal/webhook/defaults/storage.go index c48367c6..02576827 100644 --- a/internal/webhook/defaults/storage.go +++ b/internal/webhook/defaults/storage.go @@ -16,15 +16,22 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/webhook/utils" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/tenant" ) -func mutatePVCDefaults(ctx context.Context, req admission.Request, c client.Client, decoder admission.Decoder, namespace string) *admission.Response { +func mutatePVCDefaults( + ctx context.Context, + req admission.Request, + c client.Client, + decoder admission.Decoder, + namespace string, +) *admission.Response { var err error pvc := &corev1.PersistentVolumeClaim{} if err = decoder.Decode(req, pvc); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } pvc.SetNamespace(namespace) @@ -33,7 +40,7 @@ func mutatePVCDefaults(ctx context.Context, req admission.Request, c client.Clie tnt, err = tenant.TenantByStatusNamespace(ctx, c, pvc.Namespace) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } if tnt == nil { @@ -53,9 +60,7 @@ func mutatePVCDefaults(ctx context.Context, req admission.Request, c client.Clie if storageClassName := pvc.Spec.StorageClassName; storageClassName != nil && *storageClassName != allowed.Default { csc, err = utils.GetStorageClassByName(ctx, c, *storageClassName) if err != nil && !k8serrors.IsNotFound(err) { - response := admission.Denied(caperrors.NewStorageClassError(*storageClassName, err).Error()) - - return &response + return ad.Deny(caperrors.NewStorageClassError(*storageClassName, err).Error()) } } else { mutate = true @@ -69,7 +74,7 @@ func mutatePVCDefaults(ctx context.Context, req admission.Request, c client.Clie // Marshal Manifest marshaled, err := json.Marshal(pvc) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } response := admission.PatchResponseFromRaw(req.Object.Raw, marshaled) diff --git a/internal/webhook/dra/validate.go b/internal/webhook/dra/validate.go index b071ef1c..069773c7 100644 --- a/internal/webhook/dra/validate.go +++ b/internal/webhook/dra/validate.go @@ -16,6 +16,7 @@ import ( "github.com/projectcapsule/capsule/internal/webhook/utils" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" "github.com/projectcapsule/capsule/pkg/tenant" @@ -27,45 +28,69 @@ func DeviceClass() handlers.Handler { return &deviceClass{} } -func (h *deviceClass) OnCreate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *deviceClass) OnCreate( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { switch res := req.Kind.Kind; res { case "ResourceClaim": rc := &resources.ResourceClaim{} if err := decoder.Decode(req, rc); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - return h.validateResourceRequest(ctx, c, decoder, recorder, req, rc.Namespace, rc.Spec.Devices.Requests) + return h.validateResourceRequest(ctx, c, decoder, recorder, req, rc.Namespace, rc.Spec.Devices.Requests, rc) case "ResourceClaimTemplate": rct := &resources.ResourceClaimTemplate{} if err := decoder.Decode(req, rct); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - return h.validateResourceRequest(ctx, c, decoder, recorder, req, rct.Namespace, rct.Spec.Spec.Devices.Requests) + return h.validateResourceRequest(ctx, c, decoder, recorder, req, rct.Namespace, rct.Spec.Spec.Devices.Requests, rct) default: return nil } } } -func (h *deviceClass) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *deviceClass) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *deviceClass) OnUpdate(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *deviceClass) OnUpdate( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *deviceClass) validateResourceRequest(ctx context.Context, c client.Client, _ admission.Decoder, recorder events.EventRecorder, req admission.Request, namespace string, requests []resources.DeviceRequest) *admission.Response { +func (h *deviceClass) validateResourceRequest( + ctx context.Context, + c client.Client, + _ admission.Decoder, + recorder events.EventRecorder, + req admission.Request, + namespace string, + requests []resources.DeviceRequest, + obj client.Object, +) *admission.Response { tnt, err := tenant.TenantByStatusNamespace(ctx, c, namespace) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } if tnt == nil { @@ -86,11 +111,7 @@ func (h *deviceClass) validateResourceRequest(ctx context.Context, c client.Clie } if dc == nil { - recorder.Eventf(tnt, dc, corev1.EventTypeWarning, evt.ReasonMissingDeviceClass, evt.ActionValidationDenied, "%s %s/%s is missing DeviceClass", req.Kind.Kind, req.Namespace, req.Name) - - response := admission.Denied(caperrors.NewDeviceClassUndefined(*allowed).Error()) - - return &response + return ad.Deny(caperrors.NewDeviceClassUndefined(*allowed).Error()) } selector := allowed.SelectorMatch(dc) @@ -99,11 +120,9 @@ func (h *deviceClass) validateResourceRequest(ctx context.Context, c client.Clie case allowed.Match(dc.Name) || selector: return nil default: - recorder.Eventf(tnt, dc, corev1.EventTypeWarning, evt.ReasonForbiddenDeviceClass, evt.ActionValidationDenied, "%s %s/%s DeviceClass %s is forbidden for the current Tenant", req.Kind.Kind, req.Namespace, req.Name, &dc) + recorder.Eventf(obj, tnt, corev1.EventTypeWarning, evt.ReasonForbiddenDeviceClass, evt.ActionValidationDenied, "%s %s/%s DeviceClass %s is forbidden for the current Tenant", req.Kind.Kind, req.Namespace, req.Name, &dc) - response := admission.Denied(caperrors.NewDeviceClassForbidden(dc.Name, *allowed).Error()) - - return &response + return ad.Deny(caperrors.NewDeviceClassForbidden(dc.Name, *allowed).Error()) } } diff --git a/internal/webhook/gateway/validate_class.go b/internal/webhook/gateway/validate_class.go index e344e44d..36d4063e 100644 --- a/internal/webhook/gateway/validate_class.go +++ b/internal/webhook/gateway/validate_class.go @@ -17,6 +17,7 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/webhook/utils" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" @@ -32,35 +33,56 @@ func Class(configuration configuration.Configuration) handlers.Handler { } } -func (r *class) OnCreate(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (r *class) OnCreate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return r.validate(ctx, client, req, decoder, recorder) + return r.validate(ctx, c, req, decoder, recorder) } } -func (r *class) OnUpdate(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (r *class) OnUpdate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return r.validate(ctx, client, req, decoder, recorder) + return r.validate(ctx, c, req, decoder, recorder) } } -func (r *class) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (r *class) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (r *class) validate(ctx context.Context, client client.Client, req admission.Request, decoder admission.Decoder, recorder events.EventRecorder) *admission.Response { +func (r *class) validate( + ctx context.Context, + c client.Client, + req admission.Request, + decoder admission.Decoder, + recorder events.EventRecorder, +) *admission.Response { gatewayObj := &gatewayv1.Gateway{} if err := decoder.Decode(req, gatewayObj); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } var tnt *capsulev1beta2.Tenant - tnt, err := TenantFromGateway(ctx, client, gatewayObj) + tnt, err := TenantFromGateway(ctx, c, gatewayObj) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } if tnt == nil { @@ -73,23 +95,21 @@ func (r *class) validate(ctx context.Context, client client.Client, req admissio return nil } - gatewayClass, err := utils.GetGatewayClassClassByObjectName(ctx, client, gatewayObj.Spec.GatewayClassName) + gatewayClass, err := utils.GetGatewayClassClassByObjectName(ctx, c, gatewayObj.Spec.GatewayClassName) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } if gatewayClass == nil { - recorder.Eventf(tnt, gatewayClass, corev1.EventTypeWarning, evt.ReasonMissingGatewayClass, evt.ActionValidationDenied, "Gateway %s/%s is missing GatewayClass", req.Namespace, req.Name) + recorder.Eventf(gatewayObj, tnt, corev1.EventTypeWarning, evt.ReasonMissingGatewayClass, evt.ActionValidationDenied, "Gateway %s/%s is missing GatewayClass", req.Namespace, req.Name) - response := admission.Denied(caperrors.NewGatewayClassUndefined(*allowed).Error()) - - return &response + return ad.Deny(caperrors.NewGatewayClassUndefined(*allowed).Error()) } selector := false // Verify if the GatewayClass exists and matches the label selector/expression if len(allowed.MatchExpressions) > 0 || len(allowed.MatchLabels) > 0 { - gatewayClassObj, err := utils.GetGatewayClassClassByObjectName(ctx, client, gatewayObj.Spec.GatewayClassName) + gatewayClassObj, err := utils.GetGatewayClassClassByObjectName(ctx, c, gatewayObj.Spec.GatewayClassName) if err != nil && !k8serrors.IsNotFound(err) { response := admission.Errored(http.StatusInternalServerError, err) @@ -108,10 +128,8 @@ func (r *class) validate(ctx context.Context, client client.Client, req admissio case allowed.Match(gatewayClass.Name) || selector: return nil default: - recorder.Eventf(tnt, gatewayClass, corev1.EventTypeWarning, evt.ReasonForbiddenGatewayClass, evt.ActionValidationDenied, "Gateway %s/%s GatewayClass %s is forbidden for the current Tenant", req.Namespace, req.Name, &gatewayClass) + recorder.Eventf(gatewayObj, tnt, corev1.EventTypeWarning, evt.ReasonForbiddenGatewayClass, evt.ActionValidationDenied, "Gateway %s/%s GatewayClass %s is forbidden for the current Tenant", req.Namespace, req.Name, &gatewayClass) - response := admission.Denied(caperrors.NewGatewayClassForbidden(gatewayObj.Name, *allowed).Error()) - - return &response + return ad.Deny(caperrors.NewGatewayClassForbidden(gatewayObj.Name, *allowed).Error()) } } diff --git a/internal/webhook/misc/cordoning.go b/internal/webhook/generic/cordoning.go similarity index 50% rename from internal/webhook/misc/cordoning.go rename to internal/webhook/generic/cordoning.go index 1d24dc6e..9ea94f1e 100644 --- a/internal/webhook/misc/cordoning.go +++ b/internal/webhook/generic/cordoning.go @@ -1,7 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package misc +package generic import ( "context" @@ -11,6 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -22,31 +23,38 @@ func CordoningHandler(configuration configuration.Configuration) handlers.Handle } func (h *cordoningHandler) OnCreate( - c client.Client, + _ client.Client, + _ client.Reader, _ admission.Decoder, _ events.EventRecorder, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.cordonHandler(ctx, c, req) + return h.cordonHandler(req) } } -func (h *cordoningHandler) OnDelete(c client.Client, _ admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *cordoningHandler) OnDelete( + _ client.Client, + _ client.Reader, + _ admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.cordonHandler(ctx, c, req) + return h.cordonHandler(req) } } -func (h *cordoningHandler) OnUpdate(c client.Client, _ admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *cordoningHandler) OnUpdate( + _ client.Client, + _ client.Reader, + _ admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.cordonHandler(ctx, c, req) + return h.cordonHandler(req) } } -func (h *cordoningHandler) cordonHandler(ctx context.Context, c client.Client, req admission.Request) *admission.Response { - msg := fmt.Sprintf("The current namespace '%s' is cordoned. The attempted operation %s for %s/%s/%s/%s is not permitted during cordoning status.", req.Namespace, req.Operation, req.RequestKind.Group, req.RequestKind.Version, req.RequestKind.Kind, req.Name) - - response := admission.Denied(msg) - - return &response +func (h *cordoningHandler) cordonHandler(req admission.Request) *admission.Response { + return ad.Deny(fmt.Sprintf("The current namespace '%s' is cordoned. The attempted operation %s for %s/%s/%s/%s is not permitted during cordoning status.", req.Namespace, req.Operation, req.RequestKind.Group, req.RequestKind.Version, req.RequestKind.Kind, req.Name)) } diff --git a/internal/webhook/misc/custom_resource_quota.go b/internal/webhook/generic/custom_resource_quota.go similarity index 73% rename from internal/webhook/misc/custom_resource_quota.go rename to internal/webhook/generic/custom_resource_quota.go index af607ce3..f4fec307 100644 --- a/internal/webhook/misc/custom_resource_quota.go +++ b/internal/webhook/generic/custom_resource_quota.go @@ -1,7 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package misc +package generic import ( "context" @@ -16,8 +16,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" "github.com/projectcapsule/capsule/pkg/tenant" @@ -33,13 +33,18 @@ func ResourceCounterHandler(client client.Client) handlers.Handler { } } -func (r *resourceCounterHandler) OnCreate(clt client.Client, _ admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (r *resourceCounterHandler) OnCreate( + c client.Client, + reader client.Reader, + _ admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { var tntName string var err error - if tntName, err = r.getTenantName(ctx, clt, req); err != nil { - return utils.ErroredResponse(err) + if tntName, err = r.getTenantName(ctx, c, req); err != nil { + return ad.ErroredResponse(err) } if len(tntName) == 0 { @@ -53,7 +58,7 @@ func (r *resourceCounterHandler) OnCreate(clt client.Client, _ admission.Decoder var limit int64 err = retry.RetryOnConflict(retry.DefaultRetry, func() (retryErr error) { - if retryErr = clt.Get(ctx, types.NamespacedName{Name: tntName}, tnt); err != nil { + if retryErr = reader.Get(ctx, types.NamespacedName{Name: tntName}, tnt); err != nil { return retryErr } @@ -72,27 +77,32 @@ func (r *resourceCounterHandler) OnCreate(clt client.Client, _ admission.Decoder tnt.Annotations[capsulev1beta2.UsedAnnotationForResource(kgv)] = fmt.Sprintf("%d", used+1) - return clt.Update(ctx, tnt) + return c.Update(ctx, tnt) }) if err != nil { if errors.As(err, &caperrors.CustomResourceQuotaError{}) { recorder.Eventf(tnt, nil, corev1.EventTypeWarning, evt.ReasonOverprovision, evt.ActionValidationDenied, "Resource %s/%s in API group %s cannot be created, limit usage of %d has been reached", req.Namespace, req.Name, kgv, limit) } - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } return nil } } -func (r *resourceCounterHandler) OnDelete(clt client.Client, _ admission.Decoder, _ events.EventRecorder) handlers.Func { +func (r *resourceCounterHandler) OnDelete( + c client.Client, + reader client.Reader, + _ admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { var tntName string var err error - if tntName, err = r.getTenantName(ctx, clt, req); err != nil { - return utils.ErroredResponse(err) + if tntName, err = r.getTenantName(ctx, c, req); err != nil { + return ad.ErroredResponse(err) } if len(tntName) == 0 { @@ -103,7 +113,7 @@ func (r *resourceCounterHandler) OnDelete(clt client.Client, _ admission.Decoder err = retry.RetryOnConflict(retry.DefaultRetry, func() (retryErr error) { tnt := &capsulev1beta2.Tenant{} - if retryErr = clt.Get(ctx, types.NamespacedName{Name: tntName}, tnt); err != nil { + if retryErr = reader.Get(ctx, types.NamespacedName{Name: tntName}, tnt); err != nil { return retryErr } @@ -119,23 +129,32 @@ func (r *resourceCounterHandler) OnDelete(clt client.Client, _ admission.Decoder tnt.Annotations[capsulev1beta2.UsedAnnotationForResource(kgv)] = fmt.Sprintf("%d", used-1) - return clt.Update(ctx, tnt) + return c.Update(ctx, tnt) }) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } return nil } } -func (r *resourceCounterHandler) OnUpdate(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (r *resourceCounterHandler) OnUpdate( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (r *resourceCounterHandler) getTenantName(ctx context.Context, clt client.Client, req admission.Request) (string, error) { +func (r *resourceCounterHandler) getTenantName( + ctx context.Context, + clt client.Client, + req admission.Request, +) (string, error) { tnt, err := tenant.TenantByStatusNamespace(ctx, clt, req.Namespace) if err != nil { return "", err diff --git a/internal/webhook/generic/managed.go b/internal/webhook/generic/managed.go new file mode 100644 index 00000000..5cf7765e --- /dev/null +++ b/internal/webhook/generic/managed.go @@ -0,0 +1,73 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package generic + +import ( + "context" + + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" +) + +type managedValidatingHandler struct { + configuration configuration.Configuration +} + +func ManagedValidatingHandler(configuration configuration.Configuration) handlers.Handler { + return &managedValidatingHandler{ + configuration: configuration, + } +} + +func (h *managedValidatingHandler) OnCreate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return h.handle(ctx, req, c) + } +} + +func (h *managedValidatingHandler) OnDelete( + c client.Client, + _ client.Reader, + _ admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return h.handle(ctx, req, c) + } +} + +func (h *managedValidatingHandler) OnUpdate( + c client.Client, + _ client.Reader, + _ admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return h.handle(ctx, req, c) + } +} + +func (h *managedValidatingHandler) handle( + ctx context.Context, + req admission.Request, + c client.Client, +) *admission.Response { + user := handlers.ResolveAdmissionUser(ctx, c, req, h.configuration) + + if user.IsAdmin() { + return nil + } + + return ad.Deny("Labeling resources as controller managed can only be done by the controller or administrators") +} diff --git a/internal/webhook/misc/tenant_assignment.go b/internal/webhook/generic/metadata.go similarity index 62% rename from internal/webhook/misc/tenant_assignment.go rename to internal/webhook/generic/metadata.go index 279748a1..c69821ff 100644 --- a/internal/webhook/misc/tenant_assignment.go +++ b/internal/webhook/generic/metadata.go @@ -1,19 +1,20 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package misc +package generic import ( "context" admissionv1 "k8s.io/api/admission/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/api/meta" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" clt "github.com/projectcapsule/capsule/pkg/runtime/client" "github.com/projectcapsule/capsule/pkg/runtime/handlers" "github.com/projectcapsule/capsule/pkg/tenant" @@ -25,37 +26,61 @@ func TenantAssignmentHandler() handlers.Handler { return &tenantAssignmentHandler{} } -func (r *tenantAssignmentHandler) OnCreate(c client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (r *tenantAssignmentHandler) OnCreate( + _ client.Client, + reader client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return r.handle(ctx, c, decoder, req) + return r.handle(ctx, reader, decoder, req) } } -func (r *tenantAssignmentHandler) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (r *tenantAssignmentHandler) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (r *tenantAssignmentHandler) OnUpdate(c client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (r *tenantAssignmentHandler) OnUpdate( + _ client.Client, + reader client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return r.handle(ctx, c, decoder, req) + return r.handle(ctx, reader, decoder, req) } } -func (r *tenantAssignmentHandler) handle(ctx context.Context, c client.Client, decoder admission.Decoder, req admission.Request) *admission.Response { +func (r *tenantAssignmentHandler) handle( + ctx context.Context, + c client.Reader, + decoder admission.Decoder, + req admission.Request, +) *admission.Response { if req.Namespace == "" { return nil } obj := &metav1.PartialObjectMetadata{} if err := decoder.Decode(req, obj); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - tnt, err := tenant.GetTenantNameByStatusNamespace(ctx, c, req.Namespace) + tnt, err := tenant.GetTenantNameByNamespace(ctx, c, req.Namespace) if err != nil { - return utils.ErroredResponse(err) + if apierrors.IsNotFound(err) { + return nil + } + + return ad.ErroredResponse(err) } if tnt == "" { diff --git a/internal/webhook/generic/replications.go b/internal/webhook/generic/replications.go new file mode 100644 index 00000000..f49d340a --- /dev/null +++ b/internal/webhook/generic/replications.go @@ -0,0 +1,159 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package generic + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apiserver/pkg/authentication/serviceaccount" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" + "github.com/projectcapsule/capsule/pkg/runtime/gvk" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" + "github.com/projectcapsule/capsule/pkg/runtime/indexers/tenantresource" + "github.com/projectcapsule/capsule/pkg/tenant" +) + +type replicaHandler struct{} + +func ReplicaHandler() handlers.Handler { + return &replicaHandler{} +} + +func (h *replicaHandler) OnCreate( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} + +func (h *replicaHandler) OnDelete( + _ client.Client, + reader client.Reader, + _ admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return nil + } +} + +func (h *replicaHandler) OnUpdate( + c client.Client, + _ client.Reader, + _ admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return h.handler(ctx, c, req, recorder) + } +} + +func (h *replicaHandler) handler( + ctx context.Context, + c client.Client, + req admission.Request, + recorder events.EventRecorder, +) *admission.Response { + tnt, err := tenant.TenantByStatusNamespace(ctx, c, req.Namespace) + if err != nil { + return ad.ErroredResponse(err) + } + + if tnt == nil { + return nil + } + + // Checking if the object is managed by a TenantResource, local or global + ref := gvk.ResourceID{ + Group: req.Kind.Group, + Version: req.Kind.Version, + Kind: req.Kind.Kind, + Name: req.Name, + Namespace: req.Namespace, + } + + gvkKey := ref.GetGVKKey("") + + global := &capsulev1beta2.GlobalTenantResourceList{} + if err := c.List( + ctx, + global, + client.MatchingFieldsSelector{ + Selector: fields.OneTermEqualSelector(tenantresource.CreatedIndexerFieldName, gvkKey), + }, + ); err != nil { + return ad.ErroredResponse(err) + } + + if len(global.Items) > 0 { + for i := range global.Items { + if isAllowedServiceAccount(req.UserInfo.Username, global.Items[i].Status.ServiceAccount) { + return nil + } + } + + return ad.Deny( + fmt.Sprintf( + "resource %s is managed by a global capsule replication %s", + req.Name, + global.Items[0].GetName(), + ), + ) + } + + local := &capsulev1beta2.TenantResourceList{} + if err := c.List( + ctx, + local, + client.MatchingFieldsSelector{ + Selector: fields.OneTermEqualSelector(tenantresource.CreatedIndexerFieldName, gvkKey), + }, + ); err != nil { + return ad.ErroredResponse(err) + } + + if len(local.Items) > 0 { + for i := range local.Items { + if isAllowedServiceAccount(req.UserInfo.Username, local.Items[i].Status.ServiceAccount) { + return nil + } + } + + return ad.Deny( + fmt.Sprintf( + "resource %s is managed by a tenant capsule replication %s/%s", + req.Name, + local.Items[0].GetName(), + local.Items[0].GetNamespace(), + ), + ) + } + + return nil +} + +func isAllowedServiceAccount(username string, sa *meta.NamespacedRFC1123ObjectReferenceWithNamespace) bool { + if sa == nil { + return false + } + + ns, name, err := serviceaccount.SplitUsername(username) + if err != nil { + return false + } + + return name == sa.Name.String() && ns == sa.Namespace.String() +} diff --git a/internal/webhook/ingress/types.go b/internal/webhook/ingress/types.go index fba8b5d3..9f1a9bdc 100644 --- a/internal/webhook/ingress/types.go +++ b/internal/webhook/ingress/types.go @@ -10,6 +10,7 @@ import ( networkingv1 "k8s.io/api/networking/v1" networkingv1beta1 "k8s.io/api/networking/v1beta1" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -23,12 +24,17 @@ type Ingress interface { HostnamePathsPairs() map[string]sets.Set[string] SetIngressClass(string) SetNamespace(string) + GetClientObject() client.Object } type NetworkingV1 struct { *networkingv1.Ingress } +func (n NetworkingV1) GetClientObject() client.Object { + return n.Ingress +} + func (n NetworkingV1) Name() string { return n.GetName() } @@ -99,6 +105,10 @@ type NetworkingV1Beta1 struct { *networkingv1beta1.Ingress } +func (n NetworkingV1Beta1) GetClientObject() client.Object { + return n.Ingress +} + func (n NetworkingV1Beta1) Name() string { return n.GetName() } @@ -169,6 +179,10 @@ type Extension struct { *extensionsv1beta1.Ingress } +func (n Extension) GetClientObject() client.Object { + return n.Ingress +} + func (e Extension) Name() string { return e.GetName() } diff --git a/internal/webhook/ingress/validate_class.go b/internal/webhook/ingress/validate_class.go index 1f5c7d3d..ac17fe19 100644 --- a/internal/webhook/ingress/validate_class.go +++ b/internal/webhook/ingress/validate_class.go @@ -17,6 +17,7 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/webhook/utils" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" @@ -34,19 +35,34 @@ func Class(configuration configuration.Configuration, version *version.Version) } } -func (r *class) OnCreate(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (r *class) OnCreate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return r.validate(ctx, r.version, client, req, decoder, recorder) + return r.validate(ctx, r.version, c, req, decoder, recorder) } } -func (r *class) OnUpdate(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (r *class) OnUpdate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return r.validate(ctx, r.version, client, req, decoder, recorder) + return r.validate(ctx, r.version, c, req, decoder, recorder) } } -func (r *class) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (r *class) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } @@ -62,14 +78,14 @@ func (r *class) validate( ) *admission.Response { ingress, err := FromRequest(req, decoder) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } var tnt *capsulev1beta2.Tenant tnt, err = TenantFromIngress(ctx, client, ingress) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } if tnt == nil { @@ -85,11 +101,9 @@ func (r *class) validate( ingressClass := ingress.IngressClass() if ingressClass == nil { - recorder.Eventf(tnt, nil, corev1.EventTypeWarning, evt.ReasonMissingIngressClass, evt.ActionValidationDenied, "Ingress %s/%s is missing IngressClass", req.Namespace, req.Name) + recorder.Eventf(ingress.GetClientObject(), tnt, corev1.EventTypeWarning, evt.ReasonMissingIngressClass, evt.ActionValidationDenied, "Ingress %s/%s is missing IngressClass", req.Namespace, req.Name) - response := admission.Denied(caperrors.NewIngressClassUndefined(*allowed).Error()) - - return &response + return ad.Deny(caperrors.NewIngressClassUndefined(*allowed).Error()) } selector := false @@ -115,10 +129,8 @@ func (r *class) validate( case allowed.Match(*ingressClass) || selector: return nil default: - recorder.Eventf(tnt, nil, corev1.EventTypeWarning, evt.ReasonForbiddenIngressClass, evt.ActionValidationDenied, "Ingress %s/%s IngressClass %s is forbidden for the current Tenant", req.Namespace, req.Name, &ingressClass) + recorder.Eventf(ingress.GetClientObject(), tnt, corev1.EventTypeWarning, evt.ReasonForbiddenIngressClass, evt.ActionValidationDenied, "Ingress %s/%s IngressClass %s is forbidden for the current Tenant", req.Namespace, req.Name, &ingressClass) - response := admission.Denied(caperrors.NewIngressClassForbidden(*ingressClass, *allowed).Error()) - - return &response + return ad.Deny(caperrors.NewIngressClassForbidden(*ingressClass, *allowed).Error()) } } diff --git a/internal/webhook/ingress/validate_collision.go b/internal/webhook/ingress/validate_collision.go index 2e162337..03ae4591 100644 --- a/internal/webhook/ingress/validate_collision.go +++ b/internal/webhook/ingress/validate_collision.go @@ -18,9 +18,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/api" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" @@ -35,57 +35,81 @@ func Collision(configuration configuration.Configuration) handlers.Handler { return &collision{configuration: configuration} } -func (r *collision) OnCreate(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (r *collision) OnCreate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return r.validate(ctx, client, req, decoder, recorder) + return r.validate(ctx, c, req, decoder, recorder) } } -func (r *collision) OnUpdate(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (r *collision) OnUpdate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return r.validate(ctx, client, req, decoder, recorder) + return r.validate(ctx, c, req, decoder, recorder) } } -func (r *collision) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (r *collision) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (r *collision) validate(ctx context.Context, client client.Client, req admission.Request, decoder admission.Decoder, recorder events.EventRecorder) *admission.Response { +func (r *collision) validate( + ctx context.Context, + reader client.Client, + req admission.Request, + decoder admission.Decoder, + recorder events.EventRecorder, +) *admission.Response { ing, err := FromRequest(req, decoder) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - var tenant *capsulev1beta2.Tenant + var tnt *capsulev1beta2.Tenant - tenant, err = TenantFromIngress(ctx, client, ing) + tnt, err = TenantFromIngress(ctx, reader, ing) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - if tenant == nil || tenant.Spec.IngressOptions.HostnameCollisionScope == api.HostnameCollisionScopeDisabled { + if tnt == nil || tnt.Spec.IngressOptions.HostnameCollisionScope == api.HostnameCollisionScopeDisabled { return nil } - if err = r.validateCollision(ctx, client, ing, tenant.Spec.IngressOptions.HostnameCollisionScope); err == nil { + if err = r.validateCollision(ctx, reader, ing, tnt.Spec.IngressOptions.HostnameCollisionScope); err == nil { return nil } var collisionErr *caperrors.IngressHostnameCollisionError if errors.As(err, &collisionErr) { - recorder.Eventf(tenant, nil, corev1.EventTypeWarning, evt.ReasonIngressHostnameCollision, evt.ActionValidationDenied, "Ingress %s/%s hostname is colliding", ing.Namespace(), ing.Name()) + recorder.Eventf(ing.GetClientObject(), tnt, corev1.EventTypeWarning, evt.ReasonIngressHostnameCollision, evt.ActionValidationDenied, "Ingress %s/%s hostname is colliding", ing.Namespace(), ing.Name()) } - response := admission.Denied(err.Error()) - - return &response + return ad.Deny(err.Error()) } //nolint:gocognit,gocyclo,cyclop -func (r *collision) validateCollision(ctx context.Context, clt client.Client, ing Ingress, scope api.HostnameCollisionScope) error { +func (r *collision) validateCollision( + ctx context.Context, + reader client.Reader, + ing Ingress, + scope api.HostnameCollisionScope, +) error { for hostname, paths := range ing.HostnamePathsPairs() { for path := range paths { var ingressObjList client.ObjectList @@ -104,7 +128,7 @@ func (r *collision) validateCollision(ctx context.Context, clt client.Client, in switch scope { case api.HostnameCollisionScopeCluster: tenantList := &capsulev1beta2.TenantList{} - if err := clt.List(ctx, tenantList); err != nil { + if err := reader.List(ctx, tenantList); err != nil { return err } @@ -113,7 +137,7 @@ func (r *collision) validateCollision(ctx context.Context, clt client.Client, in } case api.HostnameCollisionScopeTenant: tenantList := &capsulev1beta2.TenantList{} - if err := clt.List(ctx, tenantList, client.MatchingFields{".status.namespaces": ing.Namespace()}); err != nil { + if err := reader.List(ctx, tenantList, client.MatchingFields{".status.namespaces": ing.Namespace()}); err != nil { return err } @@ -124,7 +148,7 @@ func (r *collision) validateCollision(ctx context.Context, clt client.Client, in namespaces.Insert(ing.Namespace()) } - if err := clt.List(ctx, ingressObjList, client.MatchingFields{ingress.HostPathPair: fmt.Sprintf("%s;%s", hostname, path)}); err != nil { + if err := reader.List(ctx, ingressObjList, client.MatchingFields{ingress.HostPathPair: fmt.Sprintf("%s;%s", hostname, path)}); err != nil { return err } diff --git a/internal/webhook/ingress/validate_hostnames.go b/internal/webhook/ingress/validate_hostnames.go index ce07ebe7..1190d8fe 100644 --- a/internal/webhook/ingress/validate_hostnames.go +++ b/internal/webhook/ingress/validate_hostnames.go @@ -15,8 +15,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" @@ -30,38 +30,59 @@ func Hostnames(configuration configuration.Configuration) handlers.Handler { return &hostnames{configuration: configuration} } -func (r *hostnames) OnCreate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (r *hostnames) OnCreate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { return r.validate(ctx, c, req, decoder, recorder) } } -func (r *hostnames) OnUpdate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (r *hostnames) OnUpdate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { return r.validate(ctx, c, req, decoder, recorder) } } -func (r *hostnames) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (r *hostnames) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (r *hostnames) validate(ctx context.Context, client client.Client, req admission.Request, decoder admission.Decoder, recorder events.EventRecorder) *admission.Response { +func (r *hostnames) validate( + ctx context.Context, + c client.Client, + req admission.Request, + decoder admission.Decoder, + recorder events.EventRecorder, +) *admission.Response { ingress, err := FromRequest(req, decoder) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - var tenant *capsulev1beta2.Tenant + var tnt *capsulev1beta2.Tenant - tenant, err = TenantFromIngress(ctx, client, ingress) + tnt, err = TenantFromIngress(ctx, c, ingress) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - if tenant == nil || tenant.Spec.IngressOptions.AllowedHostnames == nil { + if tnt == nil || tnt.Spec.IngressOptions.AllowedHostnames == nil { return nil } @@ -69,28 +90,28 @@ func (r *hostnames) validate(ctx context.Context, client client.Client, req admi for hostname := range ingress.HostnamePathsPairs() { if len(hostname) == 0 { - recorder.Eventf(tenant, nil, corev1.EventTypeWarning, evt.ReasonIngressHostnameEmpty, evt.ActionValidationDenied, "Ingress %s/%s hostname is empty", ingress.Namespace(), ingress.Name()) + recorder.Eventf(ingress.GetClientObject(), tnt, corev1.EventTypeWarning, evt.ReasonIngressHostnameEmpty, evt.ActionValidationDenied, "Ingress %s/%s hostname is empty", ingress.Namespace(), ingress.Name()) - return utils.ErroredResponse(caperrors.NewEmptyIngressHostname(*tenant.Spec.IngressOptions.AllowedHostnames)) + return ad.ErroredResponse(caperrors.NewEmptyIngressHostname(*tnt.Spec.IngressOptions.AllowedHostnames)) } hostnameList.Insert(hostname) } - if err = r.validateHostnames(*tenant, hostnameList); err == nil { + if err = r.validateHostnames(*tnt, hostnameList); err == nil { return nil } var hostnameNotValidErr *caperrors.IngressHostnameNotValidError if errors.As(err, &hostnameNotValidErr) { - recorder.Eventf(tenant, nil, corev1.EventTypeWarning, evt.ReasonIngressHostnameNotValid, evt.ActionValidationDenied, "Ingress %s/%s hostname is not valid", ingress.Namespace(), ingress.Name()) + recorder.Eventf(ingress.GetClientObject(), tnt, corev1.EventTypeWarning, evt.ReasonIngressHostnameNotValid, evt.ActionValidationDenied, "Ingress %s/%s hostname is not valid", ingress.Namespace(), ingress.Name()) response := admission.Denied(err.Error()) return &response } - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } func (r *hostnames) validateHostnames(tenant capsulev1beta2.Tenant, hostnames sets.Set[string]) error { diff --git a/internal/webhook/ingress/validate_wildcard.go b/internal/webhook/ingress/validate_wildcard.go index bc255b66..de706b57 100644 --- a/internal/webhook/ingress/validate_wildcard.go +++ b/internal/webhook/ingress/validate_wildcard.go @@ -14,9 +14,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" + indexer "github.com/projectcapsule/capsule/pkg/runtime/indexers/tenant" ) type wildcard struct{} @@ -25,29 +26,50 @@ func Wildcard() handlers.Handler { return &wildcard{} } -func (h *wildcard) OnCreate(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *wildcard) OnCreate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.validate(ctx, client, req, recorder, decoder) + return h.validate(ctx, c, req, recorder, decoder) } } -func (h *wildcard) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *wildcard) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *wildcard) OnUpdate(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *wildcard) OnUpdate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.validate(ctx, client, req, recorder, decoder) + return h.validate(ctx, c, req, recorder, decoder) } } -func (h *wildcard) validate(ctx context.Context, clt client.Client, req admission.Request, recorder events.EventRecorder, decoder admission.Decoder) *admission.Response { +func (h *wildcard) validate( + ctx context.Context, + c client.Client, + req admission.Request, + recorder events.EventRecorder, + decoder admission.Decoder, +) *admission.Response { tntList := &capsulev1beta2.TenantList{} - if err := clt.List(ctx, tntList, client.MatchingFields{".status.namespaces": req.Namespace}); err != nil { - return utils.ErroredResponse(err) + if err := c.List(ctx, tntList, client.MatchingFields{indexer.NamespaceIndexerFieldName: req.Namespace}); err != nil { + return ad.ErroredResponse(err) } // resource is not inside a Tenant namespace @@ -61,18 +83,16 @@ func (h *wildcard) validate(ctx context.Context, clt client.Client, req admissio // Retrieve ingress resource from request. ingress, err := FromRequest(req, decoder) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } // Loop over all the hosts present on the ingress. for host := range ingress.HostnamePathsPairs() { // Check if one of the host has wildcard. if strings.HasPrefix(host, "*") { // In case of wildcard, generate an event and then return. - recorder.Eventf(&tnt, nil, corev1.EventTypeWarning, evt.ReasonWildcardDenied, evt.ActionValidationDenied, "%s %s/%s cannot be %s", req.Kind.String(), req.Namespace, req.Name, strings.ToLower(string(req.Operation))) + recorder.Eventf(ingress.GetClientObject(), &tnt, corev1.EventTypeWarning, evt.ReasonWildcardDenied, evt.ActionValidationDenied, "%s %s/%s cannot be %s", req.Kind.String(), req.Namespace, req.Name, strings.ToLower(string(req.Operation))) - response := admission.Denied(fmt.Sprintf("Wildcard denied for tenant %s\n", tnt.GetName())) - - return &response + return ad.Deny(fmt.Sprintf("Wildcard denied for tenant %s\n", tnt.GetName())) } } } diff --git a/internal/webhook/misc/managed.go b/internal/webhook/misc/managed.go deleted file mode 100644 index 7ab361f3..00000000 --- a/internal/webhook/misc/managed.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package misc - -import ( - "context" - "fmt" - - "k8s.io/client-go/tools/events" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - "github.com/projectcapsule/capsule/pkg/runtime/handlers" -) - -type managedValidatingHandler struct{} - -func ManagedValidatingHandler() handlers.Handler { - return &managedValidatingHandler{} -} - -func (h *managedValidatingHandler) OnCreate(c client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { - return func(ctx context.Context, req admission.Request) *admission.Response { - return nil - } -} - -func (h *managedValidatingHandler) OnDelete(client client.Client, _ admission.Decoder, recorder events.EventRecorder) handlers.Func { - return func(ctx context.Context, req admission.Request) *admission.Response { - response := admission.Denied(fmt.Sprintf("resource %s is managed by capsule and can not by modified by capsule users", req.Name)) - - return &response - } -} - -func (h *managedValidatingHandler) OnUpdate(client client.Client, _ admission.Decoder, recorder events.EventRecorder) handlers.Func { - return func(ctx context.Context, req admission.Request) *admission.Response { - response := admission.Denied(fmt.Sprintf("resource %s is managed by capsule and can not by modified by capsule users", req.Name)) - - return &response - } -} diff --git a/internal/webhook/namespace/mutation/ownerreference.go b/internal/webhook/namespace/mutation/assignment.go similarity index 57% rename from internal/webhook/namespace/mutation/ownerreference.go rename to internal/webhook/namespace/mutation/assignment.go index ceae8380..343b9db2 100644 --- a/internal/webhook/namespace/mutation/ownerreference.go +++ b/internal/webhook/namespace/mutation/assignment.go @@ -5,10 +5,7 @@ package mutation import ( "context" - "encoding/json" - "net/http" - authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" @@ -19,31 +16,44 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/api/meta" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" "github.com/projectcapsule/capsule/pkg/tenant" + "github.com/projectcapsule/capsule/pkg/users" ) type ownerReferenceHandler struct { cfg configuration.Configuration } -func OwnerReferenceHandler(cfg configuration.Configuration) handlers.TypedHandler[*corev1.Namespace] { +func OwnerReferenceHandler(cfg configuration.Configuration) handlers.TypedHandlerWithUser[*corev1.Namespace] { return &ownerReferenceHandler{ cfg: cfg, } } -func (h *ownerReferenceHandler) OnCreate(c client.Client, ns *corev1.Namespace, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *ownerReferenceHandler) OnCreate( + c client.Client, + reader client.Reader, + user users.AdmissionUser, + ns *corev1.Namespace, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - tnt, errResponse := utils.GetNamespaceTenant(ctx, c, ns, req, h.cfg, recorder) + tnt, errResponse := utils.GetNamespaceTenant(ctx, reader, c, ns, user, h.cfg, recorder) if errResponse != nil { return errResponse } if tnt == nil { - response := admission.Denied("Unable to assign namespace to tenant. Please use " + meta.TenantLabel + " label when creating a namespace") + response := admission.Denied( + "Unable to assign namespace to tenant. Please use " + + meta.TenantLabel + + " label when creating a namespace", + ) return &response } @@ -54,83 +64,76 @@ func (h *ownerReferenceHandler) OnCreate(c client.Client, ns *corev1.Namespace, ns.SetLabels(labels) if err := assignToTenant(c, tnt, ns, recorder); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - marshaled, err := json.Marshal(ns) - if err != nil { - response := admission.Errored(http.StatusInternalServerError, err) - - return &response - } - - response := admission.PatchResponseFromRaw(req.Object.Raw, marshaled) - - return &response + return nil } } -func (h *ownerReferenceHandler) OnDelete(client.Client, *corev1.Namespace, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *ownerReferenceHandler) OnDelete( + client.Client, + client.Reader, + users.AdmissionUser, + *corev1.Namespace, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *ownerReferenceHandler) OnUpdate(c client.Client, newNs *corev1.Namespace, oldNs *corev1.Namespace, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *ownerReferenceHandler) OnUpdate( + c client.Client, + reader client.Reader, + user users.AdmissionUser, + newNs *corev1.Namespace, + oldNs *corev1.Namespace, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - tnt, err := resolveTenantForNamespaceUpdate(ctx, c, h.cfg, oldNs, newNs, req.UserInfo) + tnt, err := resolveTenantForNamespaceUpdate(ctx, reader, user, h.cfg, oldNs, newNs) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } if tnt == nil { return nil } - if err := assignToTenant(c, tnt, oldNs, recorder); err != nil { - return utils.ErroredResponse(err) - } - - var refs []metav1.OwnerReference - - for _, ref := range oldNs.OwnerReferences { - if tenant.IsTenantOwnerReferenceForTenant(ref, tnt) { - refs = append(refs, ref) - } - } + refs := make([]metav1.OwnerReference, 0, len(newNs.OwnerReferences)) for _, ref := range newNs.OwnerReferences { - if !tenant.IsTenantOwnerReference(ref) { - refs = append(refs, ref) + if tenant.IsTenantOwnerReference(ref) && !tenant.IsTenantOwnerReferenceForTenant(ref, tnt) { + continue } + + refs = append(refs, ref) } newNs.OwnerReferences = refs + if err := assignToTenant(c, tnt, newNs, recorder); err != nil { + return ad.ErroredResponse(err) + } + labels := newNs.GetLabels() - tenant.AddNamespaceNameLabels(labels, oldNs) + tenant.AddNamespaceNameLabels(labels, newNs) tenant.AddTenantNameLabel(labels, tnt) newNs.SetLabels(labels) - marshaled, err := json.Marshal(newNs) - if err != nil { - response := admission.Errored(http.StatusInternalServerError, err) - - return &response - } - - response := admission.PatchResponseFromRaw(req.Object.Raw, marshaled) - - return &response + return nil } } func resolveTenantForNamespaceUpdate( ctx context.Context, - c client.Client, + c client.Reader, + user users.AdmissionUser, cfg configuration.Configuration, oldNs, newNs *corev1.Namespace, - userInfo authenticationv1.UserInfo, ) (*capsulev1beta2.Tenant, error) { // 1) try old ownerRefs if tnt, err := tenant.GetTenantByOwnerreferences(ctx, c, oldNs.OwnerReferences); err != nil { @@ -146,8 +149,13 @@ func resolveTenantForNamespaceUpdate( return tnt, nil } - // 3) fall back to labels + user - return tenant.GetTenantByLabelsAndUser(ctx, c, cfg, newNs, userInfo) + // 3) Controller/admin is allowed to resolve by label only. + if user.IsAdmin() { + return tenant.GetTenantByLabels(ctx, c, newNs) + } + + // 4) fall back to labels + user + return tenant.GetTenantByLabelsAndUser(ctx, c, cfg, newNs, user) } func assignToTenant( @@ -166,12 +174,12 @@ func assignToTenant( } if err := controllerutil.SetOwnerReference(tnt, ns, c.Scheme()); err != nil { - recorder.Eventf(ns, tnt, corev1.EventTypeWarning, evt.ReasonNamespaceHijack, evt.ActionValidationDenied, "Namespace %s cannot be assigned to the desired tenant %s", ns.GetName(), tnt.GetName()) + recorder.Eventf(ns, nil, corev1.EventTypeWarning, evt.ReasonNamespaceHijack, evt.ActionValidationDenied, "Namespace %s cannot be assigned to the desired tenant %s", ns.GetName(), tnt.GetName()) return err } - recorder.Eventf(ns, tnt, corev1.EventTypeNormal, evt.ReasonTenantAssigned, evt.ActionValidationDenied, "Namespace %s has been assigned to the desired tenant %s", ns.GetName(), tnt.GetName()) + recorder.Eventf(ns, nil, corev1.EventTypeNormal, evt.ReasonTenantAssigned, evt.ActionMutated, "Namespace %s has been assigned to the desired tenant %s", ns.GetName(), tnt.GetName()) return nil } diff --git a/internal/webhook/namespace/mutation/cordoning.go b/internal/webhook/namespace/mutation/cordoning.go deleted file mode 100644 index 4ce5bea5..00000000 --- a/internal/webhook/namespace/mutation/cordoning.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package mutation - -import ( - "context" - "encoding/json" - "net/http" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/events" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api/meta" - "github.com/projectcapsule/capsule/pkg/runtime/configuration" - "github.com/projectcapsule/capsule/pkg/runtime/handlers" - capsuleutils "github.com/projectcapsule/capsule/pkg/utils" -) - -type cordoningLabelHandler struct { - cfg configuration.Configuration -} - -func CordoningLabelHandler(cfg configuration.Configuration) handlers.TypedHandler[*corev1.Namespace] { - return &cordoningLabelHandler{ - cfg: cfg, - } -} - -func (h *cordoningLabelHandler) OnCreate(client.Client, *corev1.Namespace, admission.Decoder, events.EventRecorder) handlers.Func { - return func(context.Context, admission.Request) *admission.Response { - return nil - } -} - -func (h *cordoningLabelHandler) OnDelete(client.Client, *corev1.Namespace, admission.Decoder, events.EventRecorder) handlers.Func { - return func(context.Context, admission.Request) *admission.Response { - return nil - } -} - -func (h *cordoningLabelHandler) OnUpdate( - c client.Client, - ns *corev1.Namespace, - old *corev1.Namespace, - decoder admission.Decoder, - _ events.EventRecorder, -) handlers.Func { - return func(ctx context.Context, req admission.Request) *admission.Response { - return h.handle(ctx, c, req, ns) - } -} - -func (h *cordoningLabelHandler) handle( - ctx context.Context, - c client.Client, - req admission.Request, - ns *corev1.Namespace, -) *admission.Response { - tnt := &capsulev1beta2.Tenant{} - - ln, err := capsuleutils.GetTypeLabel(tnt) - if err != nil { - response := admission.Errored(http.StatusInternalServerError, err) - - return &response - } - - if label, ok := ns.Labels[ln]; ok { - if err = c.Get(ctx, types.NamespacedName{Name: label}, tnt); err != nil { - response := admission.Errored(http.StatusInternalServerError, err) - - return &response - } - } - - condition := tnt.Status.Conditions.GetConditionByType(meta.CordonedCondition) - if condition == nil { - return nil - } - - if condition.Status != metav1.ConditionTrue { - return nil - } - - labels := ns.GetLabels() - if _, ok := labels[meta.CordonedLabel]; ok { - return nil - } - - ns.Labels[meta.CordonedLabel] = meta.ValueTrue - - marshaled, err := json.Marshal(ns) - if err != nil { - response := admission.Errored(http.StatusInternalServerError, err) - - return &response - } - - response := admission.PatchResponseFromRaw(req.Object.Raw, marshaled) - - return &response -} diff --git a/internal/webhook/namespace/mutation/guard.go b/internal/webhook/namespace/mutation/guard.go new file mode 100644 index 00000000..b3cdd003 --- /dev/null +++ b/internal/webhook/namespace/mutation/guard.go @@ -0,0 +1,121 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package mutation + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + evt "github.com/projectcapsule/capsule/pkg/runtime/events" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" + "github.com/projectcapsule/capsule/pkg/tenant" + "github.com/projectcapsule/capsule/pkg/users" +) + +type namespacePatchGuardHandler struct { + cfg configuration.Configuration +} + +func NamespacePatchGuardHandler(cfg configuration.Configuration) handlers.TypedHandlerWithUser[*corev1.Namespace] { + return &namespacePatchGuardHandler{cfg: cfg} +} + +func (h *namespacePatchGuardHandler) OnCreate( + client.Client, + client.Reader, + users.AdmissionUser, + *corev1.Namespace, + admission.Decoder, + events.EventRecorder, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} + +func (h *namespacePatchGuardHandler) OnDelete( + client.Client, + client.Reader, + users.AdmissionUser, + *corev1.Namespace, + admission.Decoder, + events.EventRecorder, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} + +func (h *namespacePatchGuardHandler) OnUpdate( + _ client.Client, + reader client.Reader, + user users.AdmissionUser, + newNs *corev1.Namespace, + oldNs *corev1.Namespace, + _ admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + if user.IsAdmin() { + return nil + } + + oldTenant, err := tenant.ResolveNamespaceTenant(ctx, reader, oldNs) + if err != nil { + return ad.ErroredResponse(err) + } + + newTenant, err := tenant.ResolveNamespaceTenant(ctx, reader, newNs) + if err != nil { + return ad.ErroredResponse(err) + } + + switch { + case oldTenant == nil && newTenant == nil: + return denyNamespacePatch(oldNs, recorder, "namespace is not owned by any tenant") + + case oldTenant == nil && newTenant != nil: + return denyNamespacePatch(oldNs, recorder, "namespace can not be patched into a tenant") + + case oldTenant != nil && newTenant == nil: + return denyNamespacePatch(oldNs, recorder, "namespace can not remove tenant ownership") + + case oldTenant.GetName() != newTenant.GetName() || oldTenant.GetUID() != newTenant.GetUID(): + return denyNamespacePatch(oldNs, recorder, "namespace can not be migrated between tenants") + } + + if !tenant.NamespaceIsOwned(ctx, reader, h.cfg, oldNs, oldTenant, user) { + return denyNamespacePatch(oldNs, recorder, "denied patch request for this namespace") + } + + return nil + } +} + +func denyNamespacePatch( + ns *corev1.Namespace, + recorder events.EventRecorder, + message string, +) *admission.Response { + if ns != nil { + recorder.Eventf( + ns, + nil, + corev1.EventTypeWarning, + "NamespacePatch", + evt.ActionValidationDenied, + "Namespace %s can not be patched: %s", + ns.GetName(), + message, + ) + } + + return ad.Deny(message) +} diff --git a/internal/webhook/namespace/mutation/handler.go b/internal/webhook/namespace/mutation/handler.go index 35f51720..ca2401cf 100644 --- a/internal/webhook/namespace/mutation/handler.go +++ b/internal/webhook/namespace/mutation/handler.go @@ -5,21 +5,20 @@ package mutation import ( "context" + "encoding/json" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "github.com/projectcapsule/capsule/internal/webhook/utils" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" - evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" "github.com/projectcapsule/capsule/pkg/tenant" - "github.com/projectcapsule/capsule/pkg/users" ) -func NamespaceHandler(configuration configuration.Configuration, handlers ...handlers.TypedHandler[*corev1.Namespace]) handlers.Handler { +func NamespaceHandler(configuration configuration.Configuration, handlers ...handlers.TypedHandlerWithUser[*corev1.Namespace]) handlers.Handler { return &handler{ cfg: configuration, handlers: handlers, @@ -28,98 +27,121 @@ func NamespaceHandler(configuration configuration.Configuration, handlers ...han type handler struct { cfg configuration.Configuration - handlers []handlers.TypedHandler[*corev1.Namespace] + handlers []handlers.TypedHandlerWithUser[*corev1.Namespace] } -func (h *handler) OnCreate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *handler) OnCreate( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - userIsAdmin := users.IsAdminUser(req, h.cfg.Administrators()) + user := handlers.ResolveAdmissionUser(ctx, c, req, h.cfg) - if !userIsAdmin && !users.IsCapsuleUser(ctx, c, h.cfg, req.UserInfo.Username, req.UserInfo.Groups) { + if !user.IsAdmin() && !user.IsCapsule() { return nil } ns := &corev1.Namespace{} if err := decoder.Decode(req, ns); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - tnt, err := tenant.GetTenantByLabels(ctx, c, ns) + tnt, err := tenant.GetTenantByLabels(ctx, reader, ns) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - if tnt == nil && userIsAdmin { + if tnt == nil && user.IsAdmin() { return nil } for _, hndl := range h.handlers { - if response := hndl.OnCreate(c, ns, decoder, recorder)(ctx, req); response != nil { + response := hndl.OnCreate(c, reader, user, ns, decoder, recorder)(ctx, req) + + if response == nil { + continue + } + + if !response.Allowed { return response } } - return nil + marshaled, err := json.Marshal(ns) + if err != nil { + return ad.ErroredResponse(err) + } + + response := admission.PatchResponseFromRaw(req.Object.Raw, marshaled) + if len(response.Patches) == 0 { + allowed := admission.Allowed("") + + return &allowed + } + + return &response } } -func (h *handler) OnDelete(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { - return func(context.Context, admission.Request) *admission.Response { - return nil - } -} - -func (h *handler) OnUpdate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *handler) OnUpdate( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - userIsAdmin := users.IsAdminUser(req, h.cfg.Administrators()) + user := handlers.ResolveAdmissionUser(ctx, c, req, h.cfg) - if !userIsAdmin && !users.IsCapsuleUser(ctx, c, h.cfg, req.UserInfo.Username, req.UserInfo.Groups) { + if !user.IsAdmin() && !user.IsCapsule() { return nil } ns := &corev1.Namespace{} if err := decoder.Decode(req, ns); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } oldNs := &corev1.Namespace{} if err := decoder.DecodeRaw(req.OldObject, oldNs); err != nil { - return utils.ErroredResponse(err) - } - - tnt, err := tenant.GetTenantByOwnerreferences(ctx, c, oldNs.OwnerReferences) - if err != nil { - return utils.ErroredResponse(err) - } - - //nolint:nestif - if userIsAdmin { - if tnt == nil { - tnt, err = tenant.GetTenantByLabels(ctx, c, ns) - if err != nil { - return utils.ErroredResponse(err) - } - - if tnt == nil { - return nil - } - } - } else { - if owned := tenant.NamespaceIsOwned(ctx, c, h.cfg, oldNs, tnt, req.UserInfo); !owned { - recorder.Eventf(tnt, oldNs, corev1.EventTypeWarning, "NamespacePatch", evt.ActionValidationDenied, "Namespace %s can not be patched", oldNs.GetName()) - - response := admission.Denied("Denied patch request for this namespace") - - return &response - } + return ad.ErroredResponse(err) } for _, hndl := range h.handlers { - if response := hndl.OnUpdate(c, ns, oldNs, decoder, recorder)(ctx, req); response != nil { + response := hndl.OnUpdate(c, reader, user, ns, oldNs, decoder, recorder)(ctx, req) + if response == nil { + continue + } + + if !response.Allowed { return response } } + marshaled, err := json.Marshal(ns) + if err != nil { + return ad.ErroredResponse(err) + } + + response := admission.PatchResponseFromRaw(req.Object.Raw, marshaled) + if len(response.Patches) == 0 { + allowed := admission.Allowed("") + + return &allowed + } + + return &response + } +} + +func (h *handler) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { return nil } } diff --git a/internal/webhook/namespace/mutation/metadata.go b/internal/webhook/namespace/mutation/metadata.go index 04dd227e..291f9d17 100644 --- a/internal/webhook/namespace/mutation/metadata.go +++ b/internal/webhook/namespace/mutation/metadata.go @@ -5,51 +5,59 @@ package mutation import ( "context" - "encoding/json" "maps" - "net/http" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/webhook/utils" + "github.com/projectcapsule/capsule/pkg/api/meta" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/handlers" "github.com/projectcapsule/capsule/pkg/tenant" + "github.com/projectcapsule/capsule/pkg/users" ) type metadataHandler struct { cfg configuration.Configuration } -func MetadataHandler(cfg configuration.Configuration) handlers.TypedHandler[*corev1.Namespace] { +func MetadataHandler(cfg configuration.Configuration) handlers.TypedHandlerWithUser[*corev1.Namespace] { return &metadataHandler{ cfg: cfg, } } -func (h *metadataHandler) OnCreate(client client.Client, ns *corev1.Namespace, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *metadataHandler) OnCreate( + c client.Client, + reader client.Reader, + user users.AdmissionUser, + ns *corev1.Namespace, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - tnt, errResponse := utils.GetNamespaceTenant(ctx, client, ns, req, h.cfg, recorder) + tnt, errResponse := utils.GetNamespaceTenant(ctx, reader, c, ns, user, h.cfg, recorder) if errResponse != nil { return errResponse } if tnt == nil { - response := admission.Denied("Unable to assign namespace to tenant.") - - return &response + return ad.Deny("Unable to assign namespace to tenant.") } labels, annotations, err := tenant.BuildNamespaceMetadataForTenant(ns, tnt) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } managedMetadataOnly := tnt.Spec.NamespaceOptions != nil && tnt.Spec.NamespaceOptions.ManagedMetadataOnly - if managedMetadataOnly { + if !managedMetadataOnly { labels = mergeStringMap(ns.GetLabels(), labels) annotations = mergeStringMap(ns.GetAnnotations(), annotations) } @@ -60,85 +68,139 @@ func (h *metadataHandler) OnCreate(client client.Client, ns *corev1.Namespace, d ns.SetLabels(labels) ns.SetAnnotations(annotations) - marshaled, err := json.Marshal(ns) - if err != nil { - response := admission.Errored(http.StatusInternalServerError, err) - - return &response + if response := h.handleCordoning(tnt, ns); response != nil { + return response } - response := admission.PatchResponseFromRaw(req.Object.Raw, marshaled) - - return &response + return nil } } -func (h *metadataHandler) OnDelete(client.Client, *corev1.Namespace, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *metadataHandler) OnDelete( + client.Client, + client.Reader, + users.AdmissionUser, + *corev1.Namespace, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *metadataHandler) OnUpdate(c client.Client, newNs *corev1.Namespace, oldNs *corev1.Namespace, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *metadataHandler) OnUpdate( + c client.Client, + reader client.Reader, + user users.AdmissionUser, + newNs *corev1.Namespace, + oldNs *corev1.Namespace, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - tnt, errResponse := utils.GetNamespaceTenant(ctx, c, oldNs, req, h.cfg, recorder) + tnt, errResponse := h.resolveTenantForUpdate(ctx, reader, c, oldNs, newNs, user, recorder) if errResponse != nil { return errResponse } if tnt == nil { - response := admission.Denied("Unable to assign namespace to tenant.") - - return &response - } - - o, err := json.Marshal(newNs.DeepCopy()) - if err != nil { - response := admission.Errored(http.StatusInternalServerError, err) - - return &response + return ad.Deny("Unable to assign namespace to tenant.") } labels, annotations, err := tenant.BuildNamespaceMetadataForTenant(newNs, tnt) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - managedMetadataOnly := tnt.Spec.NamespaceOptions != nil && tnt.Spec.NamespaceOptions.ManagedMetadataOnly - if managedMetadataOnly { + managedMetadataOnly := tnt.Spec.NamespaceOptions != nil && + tnt.Spec.NamespaceOptions.ManagedMetadataOnly + + if !managedMetadataOnly { labels = mergeStringMap(newNs.GetLabels(), labels) annotations = mergeStringMap(newNs.GetAnnotations(), annotations) } - tenant.AddNamespaceNameLabels(labels, oldNs) + tenant.AddNamespaceNameLabels(labels, newNs) tenant.AddTenantNameLabel(labels, tnt) newNs.SetLabels(labels) newNs.SetAnnotations(annotations) - obj, err := json.Marshal(newNs) - if err != nil { - response := admission.Errored(http.StatusInternalServerError, err) - - return &response - } - - response := admission.PatchResponseFromRaw(o, obj) - - return &response + return nil } } func mergeStringMap(dst, src map[string]string) map[string]string { - if len(src) == 0 { - return dst + out := maps.Clone(dst) + if out == nil { + out = map[string]string{} } - if dst == nil { - return maps.Clone(src) - } + maps.Copy(out, src) - maps.Copy(dst, src) + return out +} - return dst +func (h *metadataHandler) handleCordoning( + tnt *capsulev1beta2.Tenant, + ns *corev1.Namespace, +) *admission.Response { + condition := tnt.Status.Conditions.GetConditionByType(meta.CordonedCondition) + if condition == nil { + return nil + } + + if condition.Status != metav1.ConditionTrue { + return nil + } + + labels := ns.GetLabels() + if labels == nil { + labels = map[string]string{} + } + + if _, ok := labels[meta.CordonedLabel]; ok { + return nil + } + + labels[meta.CordonedLabel] = meta.ValueTrue + ns.SetLabels(labels) + + return nil +} + +func (h *metadataHandler) resolveTenantForUpdate( + ctx context.Context, + reader client.Reader, + cache client.Client, + oldNs *corev1.Namespace, + newNs *corev1.Namespace, + user users.AdmissionUser, + recorder events.EventRecorder, +) (*capsulev1beta2.Tenant, *admission.Response) { + if user.IsAdmin() { + tnt, err := tenant.GetTenantByLabels(ctx, reader, newNs) + if err != nil { + return nil, ad.ErroredResponse(err) + } + + if tnt != nil { + return tnt, nil + } + + tnt, err = tenant.GetTenantByLabels(ctx, reader, oldNs) + if err != nil { + return nil, ad.ErroredResponse(err) + } + + return tnt, nil + } + + tnt, errResponse := utils.GetNamespaceTenant(ctx, reader, cache, oldNs, user, h.cfg, recorder) + if errResponse != nil { + return nil, errResponse + } + + return tnt, nil } diff --git a/internal/webhook/namespace/validation/freezed.go b/internal/webhook/namespace/validation/cordoning.go similarity index 56% rename from internal/webhook/namespace/validation/freezed.go rename to internal/webhook/namespace/validation/cordoning.go index 2c382917..0e4e1a13 100644 --- a/internal/webhook/namespace/validation/freezed.go +++ b/internal/webhook/namespace/validation/cordoning.go @@ -12,75 +12,76 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" "github.com/projectcapsule/capsule/pkg/users" ) -type freezedHandler struct { +type cordoningHandler struct { cfg configuration.Configuration } -func FreezeHandler(configuration configuration.Configuration) handlers.TypedHandlerWithTenant[*corev1.Namespace] { - return &freezedHandler{cfg: configuration} +func CordoningHandler(configuration configuration.Configuration) handlers.TypedHandlerWithTenantUser[*corev1.Namespace] { + return &cordoningHandler{cfg: configuration} } -func (h *freezedHandler) OnCreate( +func (h *cordoningHandler) OnCreate( c client.Client, + _ client.Reader, + user users.AdmissionUser, ns *corev1.Namespace, - decoder admission.Decoder, + _ admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - if tnt.Spec.Cordoned { - recorder.Eventf(tnt, ns, corev1.EventTypeWarning, evt.ReasonCordoning, evt.ActionValidationDenied, "Namespace %s cannot be attached, the current Tenant is freezed", ns.GetName()) + if tnt.Spec.Cordoned && user.IsCapsule() { + recorder.Eventf(ns, nil, corev1.EventTypeWarning, evt.ReasonCordoning, evt.ActionValidationDenied, "Namespace %s cannot be attached, the current Tenant is cordoned", ns.GetName()) - response := admission.Denied("the selected Tenant is freezed") - - return &response + return ad.Deny("the selected Tenant is cordoned") } return nil } } -func (h *freezedHandler) OnDelete( +func (h *cordoningHandler) OnDelete( c client.Client, + _ client.Reader, + user users.AdmissionUser, ns *corev1.Namespace, - decoder admission.Decoder, + _ admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - if tnt.Spec.Cordoned && users.IsCapsuleUser(ctx, c, h.cfg, req.UserInfo.Username, req.UserInfo.Groups) { - recorder.Eventf(tnt, ns, corev1.EventTypeWarning, "TenantFreezed", "Denied", "Namespace %s cannot be deleted, the current Tenant is freezed", req.Name) + if tnt.Spec.Cordoned && user.IsCapsule() { + recorder.Eventf(ns, tnt, corev1.EventTypeWarning, "TenantFreezed", "Denied", "Namespace %s cannot be deleted, the current Tenant is cordoned", req.Name) - response := admission.Denied("the selected Tenant is freezed") - - return &response + return ad.Deny("the selected Tenant is cordoned") } return nil } } -func (h *freezedHandler) OnUpdate( +func (h *cordoningHandler) OnUpdate( c client.Client, + _ client.Reader, + user users.AdmissionUser, ns *corev1.Namespace, old *corev1.Namespace, - decoder admission.Decoder, + _ admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - if tnt.Spec.Cordoned && users.IsCapsuleUser(ctx, c, h.cfg, req.UserInfo.Username, req.UserInfo.Groups) { - recorder.Eventf(tnt, ns, corev1.EventTypeWarning, "TenantFreezed", "Denied", "Namespace %s cannot be updated, the current Tenant is freezed", ns.GetName()) + if tnt.Spec.Cordoned && user.IsCapsule() { + recorder.Eventf(ns, tnt, corev1.EventTypeWarning, "TenantFreezed", "Denied", "Namespace %s cannot be updated, the current Tenant is cordoned", ns.GetName()) - response := admission.Denied("the selected Tenant is freezed") - - return &response + return ad.Deny("the selected Tenant is cordoned") } return nil diff --git a/internal/webhook/namespace/validation/handler.go b/internal/webhook/namespace/validation/handler.go index 44ae03f6..c62e14c3 100644 --- a/internal/webhook/namespace/validation/handler.go +++ b/internal/webhook/namespace/validation/handler.go @@ -8,20 +8,20 @@ import ( "fmt" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" - "github.com/projectcapsule/capsule/pkg/api/meta" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" + evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" "github.com/projectcapsule/capsule/pkg/tenant" - "github.com/projectcapsule/capsule/pkg/users" ) -func NamespaceHandler(configuration configuration.Configuration, hndlers ...handlers.TypedHandlerWithTenant[*corev1.Namespace]) handlers.Handler { +func NamespaceHandler(configuration configuration.Configuration, hndlers ...handlers.TypedHandlerWithTenantUser[*corev1.Namespace]) handlers.Handler { return &handler{ cfg: configuration, handlers: hndlers, @@ -30,25 +30,76 @@ func NamespaceHandler(configuration configuration.Configuration, hndlers ...hand type handler struct { cfg configuration.Configuration - handlers []handlers.TypedHandlerWithTenant[*corev1.Namespace] + handlers []handlers.TypedHandlerWithTenantUser[*corev1.Namespace] } -func (h *handler) OnCreate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *handler) OnCreate( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - userIsAdmin := users.IsAdminUser(req, h.cfg.Administrators()) - - if !userIsAdmin && !users.IsCapsuleUser(ctx, c, h.cfg, req.UserInfo.Username, req.UserInfo.Groups) { - return nil - } + user := handlers.ResolveAdmissionUser(ctx, c, req, h.cfg) ns := &corev1.Namespace{} if err := decoder.Decode(req, ns); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - tnt, err := h.verifyReference(ctx, c, ns) + if !user.IsAdmin() && !user.IsCapsule() && !tenant.HasTenantReference(ns) { + return nil + } + + tnt, err := tenant.ResolveNamespaceTenant(ctx, reader, ns) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) + } + + if !user.IsAdmin() && !user.IsCapsule() && tnt != nil { + return ad.Deny("only tenant owners can create tenant-owned namespaces") + } + + if tnt == nil { + return nil + } + + if terminating := h.rejectOnTermination( + ctx, + c, + ns, + tnt, + ); terminating != nil { + return terminating + } + + for _, hndl := range h.handlers { + if response := hndl.OnCreate(c, reader, user, ns, decoder, recorder, tnt)(ctx, req); response != nil { + return response + } + } + + return nil + } +} + +func (h *handler) OnDelete( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + user := handlers.ResolveAdmissionUser(ctx, c, req, h.cfg) + + oldNs := &corev1.Namespace{} + if err := decoder.DecodeRaw(req.OldObject, oldNs); err != nil { + return ad.ErroredResponse(err) + } + + tnt, err := tenant.ResolveNamespaceTenant(ctx, reader, oldNs) + if err != nil { + return ad.ErroredResponse(err) } if tnt == nil { @@ -56,7 +107,7 @@ func (h *handler) OnCreate(c client.Client, decoder admission.Decoder, recorder } for _, hndl := range h.handlers { - if response := hndl.OnCreate(c, ns, decoder, recorder, tnt)(ctx, req); response != nil { + if response := hndl.OnDelete(c, reader, user, oldNs, decoder, recorder, tnt)(ctx, req); response != nil { return response } } @@ -65,52 +116,88 @@ func (h *handler) OnCreate(c client.Client, decoder admission.Decoder, recorder } } -func (h *handler) OnDelete(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *handler) OnUpdate( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return nil - } -} - -func (h *handler) OnUpdate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { - return func(ctx context.Context, req admission.Request) *admission.Response { - userIsAdmin := users.IsAdminUser(req, h.cfg.Administrators()) - - if !userIsAdmin && !users.IsCapsuleUser(ctx, c, h.cfg, req.UserInfo.Username, req.UserInfo.Groups) { - return nil - } + user := handlers.ResolveAdmissionUser(ctx, c, req, h.cfg) ns := &corev1.Namespace{} if err := decoder.Decode(req, ns); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } oldNs := &corev1.Namespace{} if err := decoder.DecodeRaw(req.OldObject, oldNs); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - oldTenant, err := h.verifyReference(ctx, c, oldNs) + oldHasTenantReference := tenant.HasTenantReference(oldNs) + newHasTenantReference := tenant.HasTenantReference(ns) + + if !user.IsAdmin() { + switch { + case !oldHasTenantReference && newHasTenantReference: + return ad.Deny("namespace can not be patched into a tenant") + case oldHasTenantReference && !newHasTenantReference: + return ad.Deny("namespace can not remove tenant ownership") + case !oldHasTenantReference && !newHasTenantReference: + return nil + } + } + + oldTenant, err := tenant.ResolveNamespaceTenant(ctx, reader, oldNs) if err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } - if oldTenant == nil { + newTenant, err := tenant.ResolveNamespaceTenant(ctx, reader, ns) + if err != nil { + return ad.ErroredResponse(err) + } + + if !user.IsAdmin() { + if oldTenant == nil || newTenant == nil { + return ad.Deny("namespace tenant ownership is incomplete") + } + + if oldTenant.GetName() != newTenant.GetName() || oldTenant.GetUID() != newTenant.GetUID() { + return ad.Deny("namespace can not be migrated between tenants") + } + + if user.IsCapsule() && !tenant.NamespaceIsOwned(ctx, c, h.cfg, oldNs, oldTenant, user) { + recorder.Eventf( + oldNs, + nil, + corev1.EventTypeWarning, + "NamespacePatch", + evt.ActionValidationDenied, + "Namespace %s can not be patched", + oldNs.GetName(), + ) + + return ad.Deny("denied patch request for this namespace") + } + } + + if terminating := h.rejectOnTermination(ctx, c, ns, newTenant); terminating != nil { + return terminating + } + + tnt := newTenant + if !user.IsAdmin() { + tnt = oldTenant + } + + if tnt == nil { return nil } - newTenant, err := h.verifyReference(ctx, c, ns) - if err != nil { - return utils.ErroredResponse(err) - } - - if newTenant.GetName() != oldTenant.GetName() { - err := fmt.Errorf("namespace can not be migrated between tenants") - - return utils.ErroredResponse(err) - } - for _, hndl := range h.handlers { - if response := hndl.OnUpdate(c, ns, oldNs, decoder, recorder, oldTenant)(ctx, req); response != nil { + if response := hndl.OnUpdate(c, reader, user, ns, oldNs, decoder, recorder, tnt)(ctx, req); response != nil { return response } } @@ -119,28 +206,30 @@ func (h *handler) OnUpdate(c client.Client, decoder admission.Decoder, recorder } } -func (h *handler) verifyReference( +func (h *handler) rejectOnTermination( ctx context.Context, - c client.Client, + c client.Reader, ns *corev1.Namespace, -) (*capsulev1beta2.Tenant, error) { - tenantByOwnerreference, err := tenant.GetTenantByOwnerreferences(ctx, c, ns.OwnerReferences) - if err != nil { - return nil, err + t *capsulev1beta2.Tenant, +) *admission.Response { + tnt := &capsulev1beta2.Tenant{} + + _ = c.Get(ctx, types.NamespacedName{Name: t.GetName()}, tnt) + + if tnt.DeletionTimestamp == nil { + return nil } - name := "" - if tenantByOwnerreference != nil { - name = tenantByOwnerreference.GetName() + instance := tnt.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ + Name: ns.GetName(), + UID: ns.GetUID(), + }) + + if instance != nil { + return nil } - if name != ns.Labels[meta.TenantLabel] { - return nil, fmt.Errorf( - "namespace label %q does not match owner reference %q", - ns.Labels[meta.TenantLabel], - name, - ) - } + err := fmt.Errorf("tenant is terminating and does not accept new namespaces") - return tenantByOwnerreference, nil + return ad.ErroredResponse(err) } diff --git a/internal/webhook/namespace/validation/patch.go b/internal/webhook/namespace/validation/patch.go deleted file mode 100644 index 8e4a4dc2..00000000 --- a/internal/webhook/namespace/validation/patch.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package validation - -import ( - "context" - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/tools/events" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/runtime/configuration" - evt "github.com/projectcapsule/capsule/pkg/runtime/events" - "github.com/projectcapsule/capsule/pkg/runtime/handlers" - "github.com/projectcapsule/capsule/pkg/users" -) - -type patchHandler struct { - cfg configuration.Configuration -} - -func PatchHandler(configuration configuration.Configuration) handlers.TypedHandlerWithTenant[*corev1.Namespace] { - return &patchHandler{cfg: configuration} -} - -func (h *patchHandler) OnCreate( - client.Client, - *corev1.Namespace, - admission.Decoder, - events.EventRecorder, - *capsulev1beta2.Tenant, -) handlers.Func { - return func(context.Context, admission.Request) *admission.Response { - return nil - } -} - -func (h *patchHandler) OnDelete( - client.Client, - *corev1.Namespace, - admission.Decoder, - events.EventRecorder, - *capsulev1beta2.Tenant, -) handlers.Func { - return func(context.Context, admission.Request) *admission.Response { - return nil - } -} - -func (h *patchHandler) OnUpdate( - c client.Client, - ns *corev1.Namespace, - old *corev1.Namespace, - decoder admission.Decoder, - recorder events.EventRecorder, - tnt *capsulev1beta2.Tenant, -) handlers.Func { - return func(ctx context.Context, req admission.Request) *admission.Response { - e := fmt.Sprintf("namespace/%s can not be patched", ns.Name) - - if ok := users.IsTenantOwnerByStatus(ctx, c, h.cfg, tnt, req.UserInfo); ok { - return nil - } - - recorder.Eventf(tnt, ns, corev1.EventTypeWarning, evt.ReasonNamespaceHijack, evt.ActionValidationDenied, e) - response := admission.Denied(e) - - return &response - } -} diff --git a/internal/webhook/namespace/validation/prefix.go b/internal/webhook/namespace/validation/prefix.go index ff9bb770..51eb6222 100644 --- a/internal/webhook/namespace/validation/prefix.go +++ b/internal/webhook/namespace/validation/prefix.go @@ -14,23 +14,27 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" + "github.com/projectcapsule/capsule/pkg/users" ) type prefixHandler struct { cfg configuration.Configuration } -func PrefixHandler(configuration configuration.Configuration) handlers.TypedHandlerWithTenant[*corev1.Namespace] { +func PrefixHandler(configuration configuration.Configuration) handlers.TypedHandlerWithTenantUser[*corev1.Namespace] { return &prefixHandler{ cfg: configuration, } } func (h *prefixHandler) OnCreate( - c client.Client, + _ client.Client, + _ client.Reader, + _ users.AdmissionUser, ns *corev1.Namespace, decoder admission.Decoder, recorder events.EventRecorder, @@ -38,25 +42,43 @@ func (h *prefixHandler) OnCreate( ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { if exp, _ := h.cfg.ProtectedNamespaceRegexp(); exp != nil { - if matched := exp.MatchString(ns.GetName()); matched { - response := admission.Denied(fmt.Sprintf("Creating namespaces with name matching %s regexp is not allowed; please, reach out to the system administrators", exp.String())) - - return &response + if exp.MatchString(ns.GetName()) { + return ad.Deny( + fmt.Sprintf( + "Creating namespaces with name matching %s regexp is not allowed; please, reach out to the system administrators", + exp.String(), + ), + ) } } - if h.cfg.ForceTenantPrefix() { - if tnt.Spec.ForceTenantPrefix != nil && !*tnt.Spec.ForceTenantPrefix { - return nil - } + enforcePrefix := h.cfg.ForceTenantPrefix() + if tnt.Spec.ForceTenantPrefix != nil { + enforcePrefix = *tnt.Spec.ForceTenantPrefix + } - if e := fmt.Sprintf("%s-%s", tnt.GetName(), ns.GetName()); !strings.HasPrefix(ns.GetName(), fmt.Sprintf("%s-", tnt.GetName())) { - recorder.Eventf(tnt, ns, corev1.EventTypeWarning, evt.ReasonInvalidTenantPrefix, evt.ActionValidationDenied, "Namespace %s does not match the expected prefix for the current Tenant", ns.GetName()) + if !enforcePrefix { + return nil + } - response := admission.Denied(fmt.Sprintf("The namespace doesn't match the tenant prefix, expected %s", e)) + expectedPrefix := tnt.GetName() + "-" + if !strings.HasPrefix(ns.GetName(), expectedPrefix) { + recorder.Eventf( + ns, + nil, + corev1.EventTypeWarning, + evt.ReasonInvalidTenantPrefix, + evt.ActionValidationDenied, + "Namespace %s does not match the expected prefix for the current Tenant", + ns.GetName(), + ) - return &response - } + return ad.Deny( + fmt.Sprintf( + "The namespace doesn't match the tenant prefix, expected prefix %q", + expectedPrefix, + ), + ) } return nil @@ -65,6 +87,8 @@ func (h *prefixHandler) OnCreate( func (h *prefixHandler) OnUpdate( client.Client, + client.Reader, + users.AdmissionUser, *corev1.Namespace, *corev1.Namespace, admission.Decoder, @@ -78,6 +102,8 @@ func (h *prefixHandler) OnUpdate( func (h *prefixHandler) OnDelete( client.Client, + client.Reader, + users.AdmissionUser, *corev1.Namespace, admission.Decoder, events.EventRecorder, diff --git a/internal/webhook/namespace/validation/quota.go b/internal/webhook/namespace/validation/quota.go index d0271302..f3323cb0 100644 --- a/internal/webhook/namespace/validation/quota.go +++ b/internal/webhook/namespace/validation/quota.go @@ -14,30 +14,36 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" + "github.com/projectcapsule/capsule/pkg/users" ) type quotaHandler struct{} -func QuotaHandler() handlers.TypedHandlerWithTenant[*corev1.Namespace] { +func QuotaHandler() handlers.TypedHandlerWithTenantUser[*corev1.Namespace] { return "aHandler{} } func (h *quotaHandler) OnCreate( - c client.Client, + _ client.Client, + reader client.Reader, + _ users.AdmissionUser, ns *corev1.Namespace, - decoder admission.Decoder, + _ admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.handle(ctx, c, recorder, ns, tnt) + return h.handle(ctx, reader, recorder, ns, tnt) } } func (h *quotaHandler) OnDelete( client.Client, + client.Reader, + users.AdmissionUser, *corev1.Namespace, admission.Decoder, events.EventRecorder, @@ -49,21 +55,23 @@ func (h *quotaHandler) OnDelete( } func (h *quotaHandler) OnUpdate( - c client.Client, + _ client.Client, + reader client.Reader, + _ users.AdmissionUser, ns *corev1.Namespace, _ *corev1.Namespace, - decoder admission.Decoder, + _ admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.handle(ctx, c, recorder, ns, tnt) + return h.handle(ctx, reader, recorder, ns, tnt) } } func (h *quotaHandler) handle( ctx context.Context, - c client.Client, + c client.Reader, recorder events.EventRecorder, ns *corev1.Namespace, tnt *capsulev1beta2.Tenant, @@ -77,11 +85,9 @@ func (h *quotaHandler) handle( return nil } - recorder.Eventf(tnt, ns, corev1.EventTypeWarning, evt.ReasonOverprovision, evt.ActionValidationDenied, "Namespace %s cannot be attached, quota exceeded for the current Tenant", ns.GetName()) + recorder.Eventf(ns, nil, corev1.EventTypeWarning, evt.ReasonOverprovision, evt.ActionValidationDenied, "Namespace %s cannot be attached, quota exceeded for the current Tenant", ns.GetName()) - response := admission.Denied(caperrors.NewNamespaceQuotaExceededError().Error()) - - return &response + return ad.Deny(caperrors.NewNamespaceQuotaExceededError().Error()) } return nil diff --git a/internal/webhook/namespace/validation/required_metadata.go b/internal/webhook/namespace/validation/required_metadata.go index 6fd7fcd4..ad6608fa 100644 --- a/internal/webhook/namespace/validation/required_metadata.go +++ b/internal/webhook/namespace/validation/required_metadata.go @@ -14,17 +14,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" + "github.com/projectcapsule/capsule/pkg/users" ) type requiredMetadataHandler struct{} -func RequiredMetadataHandler() handlers.TypedHandlerWithTenant[*corev1.Namespace] { +func RequiredMetadataHandler() handlers.TypedHandlerWithTenantUser[*corev1.Namespace] { return &requiredMetadataHandler{} } func (h *requiredMetadataHandler) OnCreate( _ client.Client, + _ client.Reader, + _ users.AdmissionUser, ns *corev1.Namespace, _ admission.Decoder, _ events.EventRecorder, @@ -60,6 +64,8 @@ func (h *requiredMetadataHandler) OnCreate( func (h *requiredMetadataHandler) OnUpdate( _ client.Client, + _ client.Reader, + _ users.AdmissionUser, newNs *corev1.Namespace, oldNs *corev1.Namespace, _ admission.Decoder, @@ -98,6 +104,8 @@ func (h *requiredMetadataHandler) OnUpdate( func (h *requiredMetadataHandler) OnDelete( client.Client, + client.Reader, + users.AdmissionUser, *corev1.Namespace, admission.Decoder, events.EventRecorder, @@ -110,22 +118,16 @@ func validateRequiredMapCreate(kind string, required map[string]string, actual m for key, exp := range required { val, ok := actual[key] if !ok { - resp := admission.Denied(fmt.Sprintf("required %s %q not present", kind, key)) - - return &resp + return ad.Deny(fmt.Sprintf("required %s %q not present", kind, key)) } re, reErr := regexp.Compile(exp) if reErr != nil { - resp := admission.Denied(fmt.Sprintf("invalid required %s regex for %q: %q: %v", kind, key, exp, reErr)) - - return &resp + return ad.Deny(fmt.Sprintf("invalid required %s regex for %q: %q: %v", kind, key, exp, reErr)) } if !re.MatchString(val) { - resp := admission.Denied(fmt.Sprintf("required %s %q value %q does not match regex %q", kind, key, val, exp)) - - return &resp + return ad.Deny(fmt.Sprintf("required %s %q value %q does not match regex %q", kind, key, val, exp)) } } @@ -147,22 +149,16 @@ func validateRequiredMapUpdate(kind string, required map[string]string, newMap, } if !newOK { - resp := admission.Denied(fmt.Sprintf("required %s %q not present", kind, key)) - - return &resp + return ad.Deny(fmt.Sprintf("required %s %q not present", kind, key)) } re, reErr := regexp.Compile(exp) if reErr != nil { - resp := admission.Denied(fmt.Sprintf("invalid required %s regex for %q: %q: %v", kind, key, exp, reErr)) - - return &resp + return ad.Deny(fmt.Sprintf("invalid required %s regex for %q: %q: %v", kind, key, exp, reErr)) } if !re.MatchString(valNew) { - resp := admission.Denied(fmt.Sprintf("required %s %q value %q does not match regex %q", mismatchKind, key, valNew, exp)) - - return &resp + return ad.Deny(fmt.Sprintf("required %s %q value %q does not match regex %q", mismatchKind, key, valNew, exp)) } } diff --git a/internal/webhook/namespace/validation/user_metadata.go b/internal/webhook/namespace/validation/user_metadata.go index b4dcd0e8..6786307e 100644 --- a/internal/webhook/namespace/validation/user_metadata.go +++ b/internal/webhook/namespace/validation/user_metadata.go @@ -14,41 +14,45 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" + "github.com/projectcapsule/capsule/pkg/users" ) type userMetadataHandler struct{} -func UserMetadataHandler() handlers.TypedHandlerWithTenant[*corev1.Namespace] { +func UserMetadataHandler() handlers.TypedHandlerWithTenantUser[*corev1.Namespace] { return &userMetadataHandler{} } func (h *userMetadataHandler) OnCreate( - c client.Client, + _ client.Client, + _ client.Reader, + _ users.AdmissionUser, ns *corev1.Namespace, - decoder admission.Decoder, + _ admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { + ns.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Namespace")) + if tnt.Spec.NamespaceOptions != nil { err := api.ValidateForbidden(ns.Annotations, tnt.Spec.NamespaceOptions.ForbiddenAnnotations) if err != nil { err = errors.Wrap(err, "namespace annotations validation failed") - recorder.Eventf(tnt, ns, corev1.EventTypeWarning, evt.ReasonForbiddenAnnotation, evt.ActionValidationDenied, err.Error()) - response := admission.Denied(err.Error()) + recorder.Eventf(ns, ns, corev1.EventTypeWarning, evt.ReasonForbiddenAnnotation, evt.ActionValidationDenied, err.Error()) - return &response + return ad.Deny(err.Error()) } err = api.ValidateForbidden(ns.Labels, tnt.Spec.NamespaceOptions.ForbiddenLabels) if err != nil { err = errors.Wrap(err, "namespace labels validation failed") - recorder.Eventf(tnt, ns, corev1.EventTypeWarning, evt.ReasonForbiddenLabel, evt.ActionValidationDenied, err.Error()) - response := admission.Denied(err.Error()) + recorder.Eventf(ns, ns, corev1.EventTypeWarning, evt.ReasonForbiddenLabel, evt.ActionValidationDenied, err.Error()) - return &response + return ad.Deny(err.Error()) } } @@ -57,10 +61,12 @@ func (h *userMetadataHandler) OnCreate( } func (h *userMetadataHandler) OnUpdate( - client client.Client, + _ client.Client, + _ client.Reader, + _ users.AdmissionUser, newNs *corev1.Namespace, oldNs *corev1.Namespace, - decoder admission.Decoder, + _ admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, ) handlers.Func { @@ -68,19 +74,19 @@ func (h *userMetadataHandler) OnUpdate( if len(tnt.Spec.NodeSelector) > 0 { v, ok := newNs.GetAnnotations()["scheduler.alpha.kubernetes.io/node-selector"] if !ok { - response := admission.Denied("the node-selector annotation is enforced, cannot be removed") + msg := "the node-selector annotation is enforced, cannot be removed" - recorder.Eventf(tnt, oldNs, corev1.EventTypeWarning, "ForbiddenNodeSelectorDeletion", "Denied", string(response.Result.Reason)) + recorder.Eventf(oldNs, oldNs, corev1.EventTypeWarning, "ForbiddenNodeSelectorDeletion", "Denied", msg) - return &response + return ad.Deny(msg) } if v != oldNs.GetAnnotations()["scheduler.alpha.kubernetes.io/node-selector"] { - response := admission.Denied("the node-selector annotation is enforced, cannot be updated") + msg := "the node-selector annotation is enforced, cannot be updated" - recorder.Eventf(tnt, oldNs, corev1.EventTypeWarning, "ForbiddenNodeSelectorUpdate", "Denied", string(response.Result.Reason)) + recorder.Eventf(oldNs, oldNs, corev1.EventTypeWarning, "ForbiddenNodeSelectorUpdate", "Denied", msg) - return &response + return ad.Deny(msg) } } @@ -128,19 +134,17 @@ func (h *userMetadataHandler) OnUpdate( err := api.ValidateForbidden(annotations, tnt.Spec.NamespaceOptions.ForbiddenAnnotations) if err != nil { err = errors.Wrap(err, "namespace annotations validation failed") - recorder.Eventf(tnt, oldNs, corev1.EventTypeWarning, evt.ReasonForbiddenAnnotation, evt.ActionValidationDenied, err.Error()) - response := admission.Denied(err.Error()) + recorder.Eventf(oldNs, oldNs, corev1.EventTypeWarning, evt.ReasonForbiddenAnnotation, evt.ActionValidationDenied, err.Error()) - return &response + return ad.Deny(err.Error()) } err = api.ValidateForbidden(labels, tnt.Spec.NamespaceOptions.ForbiddenLabels) if err != nil { err = errors.Wrap(err, "namespace labels validation failed") - recorder.Eventf(tnt, oldNs, corev1.EventTypeWarning, evt.ReasonForbiddenLabel, evt.ActionValidationDenied, err.Error()) - response := admission.Denied(err.Error()) + recorder.Eventf(oldNs, oldNs, corev1.EventTypeWarning, evt.ReasonForbiddenLabel, evt.ActionValidationDenied, err.Error()) - return &response + return ad.Deny(err.Error()) } } @@ -150,6 +154,8 @@ func (h *userMetadataHandler) OnUpdate( func (h *userMetadataHandler) OnDelete( client.Client, + client.Reader, + users.AdmissionUser, *corev1.Namespace, admission.Decoder, events.EventRecorder, diff --git a/internal/webhook/node/user_metadata.go b/internal/webhook/node/user_metadata.go index 20933dd8..fbfe25b5 100644 --- a/internal/webhook/node/user_metadata.go +++ b/internal/webhook/node/user_metadata.go @@ -13,11 +13,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "github.com/projectcapsule/capsule/internal/webhook/utils" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" + caputils "github.com/projectcapsule/capsule/pkg/utils" ) type userMetadataHandler struct { @@ -32,21 +33,36 @@ func UserMetadataHandler(configuration configuration.Configuration, ver *version } } -func (r *userMetadataHandler) OnCreate(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (r *userMetadataHandler) OnCreate( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (r *userMetadataHandler) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (r *userMetadataHandler) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (r *userMetadataHandler) OnUpdate(_ client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (r *userMetadataHandler) OnUpdate( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(_ context.Context, req admission.Request) *admission.Response { - nodeWebhookSupported, _ := utils.NodeWebhookSupported(r.version) + nodeWebhookSupported, _ := caputils.NodeWebhookSupported(r.version) if !nodeWebhookSupported { return nil @@ -54,12 +70,12 @@ func (r *userMetadataHandler) OnUpdate(_ client.Client, decoder admission.Decode oldNode := &corev1.Node{} if err := decoder.DecodeRaw(req.OldObject, oldNode); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } newNode := &corev1.Node{} if err := decoder.Decode(req, newNode); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } if r.configuration.ForbiddenUserNodeLabels() != nil { @@ -67,11 +83,9 @@ func (r *userMetadataHandler) OnUpdate(_ client.Client, decoder admission.Decode newNodeForbiddenLabels := r.getForbiddenNodeLabels(newNode) if !reflect.DeepEqual(oldNodeForbiddenLabels, newNodeForbiddenLabels) { - recorder.Eventf(newNode, oldNode, corev1.EventTypeWarning, evt.ReasonForbiddenLabel, evt.ActionValidationDenied, "Denied modifying forbidden labels on node") + recorder.Eventf(newNode, nil, corev1.EventTypeWarning, evt.ReasonForbiddenLabel, evt.ActionValidationDenied, "Denied modifying forbidden labels on node") - response := admission.Denied(caperrors.NewNodeLabelForbiddenError(r.configuration.ForbiddenUserNodeLabels()).Error()) - - return &response + return ad.Deny(caperrors.NewNodeLabelForbiddenError(r.configuration.ForbiddenUserNodeLabels()).Error()) } } @@ -80,11 +94,9 @@ func (r *userMetadataHandler) OnUpdate(_ client.Client, decoder admission.Decode newNodeForbiddenAnnotations := r.getForbiddenNodeAnnotations(newNode) if !reflect.DeepEqual(oldNodeForbiddenAnnotations, newNodeForbiddenAnnotations) { - recorder.Eventf(newNode, oldNode, corev1.EventTypeWarning, evt.ReasonForbiddenLabel, evt.ActionValidationDenied, "Denied modifying forbidden annotations on node") + recorder.Eventf(newNode, nil, corev1.EventTypeWarning, evt.ReasonForbiddenLabel, evt.ActionValidationDenied, "Denied modifying forbidden annotations on node") - response := admission.Denied(caperrors.NewNodeAnnotationForbiddenError(r.configuration.ForbiddenUserNodeAnnotations()).Error()) - - return &response + return ad.Deny(caperrors.NewNodeAnnotationForbiddenError(r.configuration.ForbiddenUserNodeAnnotations()).Error()) } } diff --git a/internal/webhook/pod/containerregistry_legacy.go b/internal/webhook/pod/containerregistry_legacy.go index 29b3244f..5508c064 100644 --- a/internal/webhook/pod/containerregistry_legacy.go +++ b/internal/webhook/pod/containerregistry_legacy.go @@ -12,7 +12,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" @@ -29,12 +31,13 @@ func ContainerRegistryLegacy(configuration configuration.Configuration) handlers } func (h *containerRegistryLegacyHandler) OnCreate( - c client.Client, + _ client.Client, + _ client.Reader, pod *corev1.Pod, - decoder admission.Decoder, + _ admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, - _ *capsulev1beta2.NamespaceRuleBody, + _ *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { return h.validate(req, pod, tnt, recorder) @@ -42,13 +45,14 @@ func (h *containerRegistryLegacyHandler) OnCreate( } func (h *containerRegistryLegacyHandler) OnUpdate( - c client.Client, + _ client.Client, + _ client.Reader, old *corev1.Pod, pod *corev1.Pod, - decoder admission.Decoder, + _ admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, - _ *capsulev1beta2.NamespaceRuleBody, + _ *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { return h.validate(req, pod, tnt, recorder) @@ -57,11 +61,12 @@ func (h *containerRegistryLegacyHandler) OnUpdate( func (h *containerRegistryLegacyHandler) OnDelete( client.Client, + client.Reader, *corev1.Pod, admission.Decoder, events.EventRecorder, *capsulev1beta2.Tenant, - *capsulev1beta2.NamespaceRuleBody, + *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil @@ -122,9 +127,7 @@ func (h *containerRegistryLegacyHandler) verifyContainerRegistry( ) //nolint:staticcheck - response := admission.Denied(caperrors.NewContainerRegistryForbidden(image, *tnt.Spec.ContainerRegistries).Error()) - - return &response + return ad.Deny(caperrors.NewContainerRegistryForbidden(image, *tnt.Spec.ContainerRegistries).Error()) } //nolint:staticcheck @@ -144,9 +147,7 @@ func (h *containerRegistryLegacyHandler) verifyContainerRegistry( ) //nolint:staticcheck - response := admission.Denied(caperrors.NewContainerRegistryForbidden(reg.FQCI(), *tnt.Spec.ContainerRegistries).Error()) - - return &response + return ad.Deny(caperrors.NewContainerRegistryForbidden(reg.FQCI(), *tnt.Spec.ContainerRegistries).Error()) } return nil diff --git a/internal/webhook/pod/imagepullpolicy.go b/internal/webhook/pod/imagepullpolicy.go index d412c358..b7a267d6 100644 --- a/internal/webhook/pod/imagepullpolicy.go +++ b/internal/webhook/pod/imagepullpolicy.go @@ -12,7 +12,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -24,12 +26,13 @@ func ImagePullPolicy() handlers.TypedHandlerWithTenantWithRuleset[*corev1.Pod] { } func (h *imagePullPolicy) OnCreate( - c client.Client, + _ client.Client, + _ client.Reader, pod *corev1.Pod, - decoder admission.Decoder, + _ admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, - _ *capsulev1beta2.NamespaceRuleBody, + _ *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { return h.validate(req, pod, tnt, recorder) @@ -37,13 +40,14 @@ func (h *imagePullPolicy) OnCreate( } func (h *imagePullPolicy) OnUpdate( - c client.Client, + _ client.Client, + _ client.Reader, old *corev1.Pod, pod *corev1.Pod, - decoder admission.Decoder, + _ admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, - _ *capsulev1beta2.NamespaceRuleBody, + _ *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { return h.validate(req, pod, tnt, recorder) @@ -52,11 +56,12 @@ func (h *imagePullPolicy) OnUpdate( func (h *imagePullPolicy) OnDelete( client.Client, + client.Reader, *corev1.Pod, admission.Decoder, events.EventRecorder, *capsulev1beta2.Tenant, - *capsulev1beta2.NamespaceRuleBody, + *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil @@ -114,9 +119,7 @@ func (h *imagePullPolicy) verifyPullPolicy( "PullPolicy %s is forbidden for the tenant %s", usedPullPolicy, tnt.GetName(), ) - response := admission.Denied(caperrors.NewImagePullPolicyForbidden(usedPullPolicy, container, policy.AllowedPullPolicies()).Error()) - - return &response + return ad.Deny(caperrors.NewImagePullPolicyForbidden(usedPullPolicy, container, policy.AllowedPullPolicies()).Error()) } return nil diff --git a/internal/webhook/pod/priorityclass.go b/internal/webhook/pod/priorityclass.go index e7ea9de5..da2c0209 100644 --- a/internal/webhook/pod/priorityclass.go +++ b/internal/webhook/pod/priorityclass.go @@ -14,7 +14,9 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/webhook/utils" + "github.com/projectcapsule/capsule/pkg/api" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -26,12 +28,13 @@ func PriorityClass() handlers.TypedHandlerWithTenantWithRuleset[*corev1.Pod] { } func (h *priorityClass) OnCreate( - c client.Client, + _ client.Client, + reader client.Reader, pod *corev1.Pod, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, - _ *capsulev1beta2.NamespaceRuleBody, + _ *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { allowed := tnt.Spec.PriorityClasses @@ -51,7 +54,7 @@ func (h *priorityClass) OnCreate( // Verify if the StorageClass exists and matches the label selector/expression if len(allowed.MatchExpressions) > 0 || len(allowed.MatchLabels) > 0 { - priorityClassObj, err := utils.GetPriorityClassByName(ctx, c, priorityClassName) + priorityClassObj, err := utils.GetPriorityClassByName(ctx, reader, priorityClassName) if err != nil { response := admission.Errored(http.StatusInternalServerError, err) @@ -80,21 +83,20 @@ func (h *priorityClass) OnCreate( "Using Priority Class %s is forbidden for the tenant %s", priorityClassName, tnt.GetName(), ) - response := admission.Denied(caperrors.NewPodPriorityClassForbidden(priorityClassName, *allowed).Error()) - - return &response + return ad.Deny(caperrors.NewPodPriorityClassForbidden(priorityClassName, *allowed).Error()) } } } func (h *priorityClass) OnUpdate( client.Client, + client.Reader, *corev1.Pod, *corev1.Pod, admission.Decoder, events.EventRecorder, *capsulev1beta2.Tenant, - *capsulev1beta2.NamespaceRuleBody, + *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil @@ -103,11 +105,12 @@ func (h *priorityClass) OnUpdate( func (h *priorityClass) OnDelete( client.Client, + client.Reader, *corev1.Pod, admission.Decoder, events.EventRecorder, *capsulev1beta2.Tenant, - *capsulev1beta2.NamespaceRuleBody, + *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil diff --git a/internal/webhook/pod/registry.go b/internal/webhook/pod/registry.go index f66afe97..68a93735 100644 --- a/internal/webhook/pod/registry.go +++ b/internal/webhook/pod/registry.go @@ -18,6 +18,7 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/cache" "github.com/projectcapsule/capsule/pkg/api" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" @@ -36,12 +37,13 @@ func ContainerRegistry(configuration configuration.Configuration, cache *cache.R } func (h *registryHandler) OnCreate( - c client.Client, + _ client.Client, + _ client.Reader, pod *corev1.Pod, - decoder admission.Decoder, + _ admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, - rule *capsulev1beta2.NamespaceRuleBody, + rule *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { return h.validate(req, pod, tnt, recorder, rule) @@ -49,13 +51,14 @@ func (h *registryHandler) OnCreate( } func (h *registryHandler) OnUpdate( - c client.Client, + _ client.Client, + _ client.Reader, old *corev1.Pod, pod *corev1.Pod, - decoder admission.Decoder, + _ admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, - rule *capsulev1beta2.NamespaceRuleBody, + rule *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { return h.validate(req, pod, tnt, recorder, rule) @@ -64,11 +67,12 @@ func (h *registryHandler) OnUpdate( func (h *registryHandler) OnDelete( client.Client, + client.Reader, *corev1.Pod, admission.Decoder, events.EventRecorder, *capsulev1beta2.Tenant, - *capsulev1beta2.NamespaceRuleBody, + *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil @@ -80,7 +84,7 @@ func (h *registryHandler) validate( pod *corev1.Pod, tnt *capsulev1beta2.Tenant, recorder events.EventRecorder, - rule *capsulev1beta2.NamespaceRuleBody, + rule *api.NamespaceRuleBodyNamespace, ) *admission.Response { if rule == nil || len(rule.Enforce.Registries) == 0 { resp := admission.Allowed("no registry rules") @@ -162,9 +166,9 @@ func (h *registryHandler) validateVolumes( ref := strings.TrimSpace(v.Image.Reference) if ref == "" { - resp := admission.Denied(fmt.Sprintf("volume %q has empty image.reference", v.Name)) - - return &resp + return ad.Deny( + fmt.Sprintf("volume %q has empty image.reference", v.Name), + ) } if resp := h.verifyOCIReference( @@ -236,8 +240,6 @@ func (h *registryHandler) verifyOCIReference( if ref == "" { msg := fmt.Sprintf("%s has empty reference", where) - resp := admission.Denied(msg) - recorder.Eventf( pod, tnt, @@ -247,7 +249,7 @@ func (h *registryHandler) verifyOCIReference( msg, ) - return &resp + return ad.Deny(msg) } // Match rules against the FULL OCI reference string. @@ -256,8 +258,6 @@ func (h *registryHandler) verifyOCIReference( if !cfg.allowed { msg := fmt.Sprintf("%s reference %q is not allowed", where, ref) - resp := admission.Denied(msg) - recorder.Eventf( pod, tnt, @@ -267,7 +267,7 @@ func (h *registryHandler) verifyOCIReference( msg, ) - return &resp + return ad.Deny(msg) } // No defaulting: enforce only if restricted; empty pullPolicy is rejected under restriction. @@ -280,8 +280,6 @@ func (h *registryHandler) verifyOCIReference( where, ref, allowed, ) - resp := admission.Denied(msg) - recorder.Eventf( pod, tnt, @@ -291,7 +289,7 @@ func (h *registryHandler) verifyOCIReference( msg, ) - return &resp + return ad.Deny(msg) } if _, ok := cfg.allowedPolicy[pullPolicy]; !ok { @@ -300,8 +298,6 @@ func (h *registryHandler) verifyOCIReference( where, ref, pullPolicy, allowed, ) - resp := admission.Denied(msg) - recorder.Eventf( pod, tnt, @@ -311,7 +307,7 @@ func (h *registryHandler) verifyOCIReference( msg, ) - return &resp + return ad.Deny(msg) } } diff --git a/internal/webhook/pod/runtimeclass.go b/internal/webhook/pod/runtimeclass.go index 141052a2..9508a07f 100644 --- a/internal/webhook/pod/runtimeclass.go +++ b/internal/webhook/pod/runtimeclass.go @@ -15,7 +15,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -27,26 +29,28 @@ func RuntimeClass() handlers.TypedHandlerWithTenantWithRuleset[*corev1.Pod] { } func (h *runtimeClass) OnCreate( - c client.Client, + _ client.Client, + reader client.Reader, pod *corev1.Pod, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, - _ *capsulev1beta2.NamespaceRuleBody, + _ *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.validate(ctx, c, recorder, req, pod, tnt) + return h.validate(ctx, reader, recorder, req, pod, tnt) } } func (h *runtimeClass) OnUpdate( client.Client, + client.Reader, *corev1.Pod, *corev1.Pod, admission.Decoder, events.EventRecorder, *capsulev1beta2.Tenant, - *capsulev1beta2.NamespaceRuleBody, + *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil @@ -55,18 +59,19 @@ func (h *runtimeClass) OnUpdate( func (h *runtimeClass) OnDelete( client.Client, + client.Reader, *corev1.Pod, admission.Decoder, events.EventRecorder, *capsulev1beta2.Tenant, - *capsulev1beta2.NamespaceRuleBody, + *api.NamespaceRuleBodyNamespace, ) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *runtimeClass) class(ctx context.Context, c client.Client, name string) (client.Object, error) { +func (h *runtimeClass) class(ctx context.Context, c client.Reader, name string) (client.Object, error) { if len(name) == 0 { return nil, nil } @@ -81,7 +86,7 @@ func (h *runtimeClass) class(ctx context.Context, c client.Client, name string) func (h *runtimeClass) validate( ctx context.Context, - c client.Client, + c client.Reader, recorder events.EventRecorder, req admission.Request, pod *corev1.Pod, @@ -110,17 +115,15 @@ func (h *runtimeClass) validate( return nil case !allowed.MatchSelectByName(class): recorder.Eventf( - tnt, pod, + tnt, corev1.EventTypeWarning, evt.ReasonForbiddenRuntimeClass, evt.ActionValidationDenied, "Using Runtime Class %s is forbidden for the tenant %s", runtimeClassName, tnt.GetName(), ) - response := admission.Denied(caperrors.NewPodRuntimeClassForbidden(runtimeClassName, *allowed).Error()) - - return &response + return ad.Deny(caperrors.NewPodRuntimeClassForbidden(runtimeClassName, *allowed).Error()) default: return nil } diff --git a/internal/webhook/pvc/pv.go b/internal/webhook/pvc/pv.go deleted file mode 100644 index 4b2c2fe4..00000000 --- a/internal/webhook/pvc/pv.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package pvc - -import ( - "context" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/events" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" - caperrors "github.com/projectcapsule/capsule/pkg/api/errors" - "github.com/projectcapsule/capsule/pkg/api/meta" - evt "github.com/projectcapsule/capsule/pkg/runtime/events" - "github.com/projectcapsule/capsule/pkg/runtime/handlers" -) - -type pv struct{} - -func PersistentVolumeReuse() handlers.TypedHandlerWithTenant[*corev1.PersistentVolumeClaim] { - return &pv{} -} - -func (h pv) OnCreate( - c client.Client, - pvc *corev1.PersistentVolumeClaim, - decoder admission.Decoder, - recorder events.EventRecorder, - tnt *capsulev1beta2.Tenant, -) handlers.Func { - return func(ctx context.Context, req admission.Request) *admission.Response { - pvObj, err := h.handle(ctx, c, pvc, tnt.Name) - if err == nil { - return nil - } - - var related runtime.Object - if pvObj != nil { - related = pvObj - } else { - related = tnt - } - - caperrors.RecordTypedErrorEvent(recorder, pvc, related, err) - - return utils.ErroredResponse(err) - } -} - -func (h pv) OnUpdate( - client.Client, - *corev1.PersistentVolumeClaim, - *corev1.PersistentVolumeClaim, - admission.Decoder, - events.EventRecorder, - *capsulev1beta2.Tenant, -) handlers.Func { - return func(ctx context.Context, req admission.Request) *admission.Response { - return nil - } -} - -func (h pv) OnDelete( - client.Client, - *corev1.PersistentVolumeClaim, - admission.Decoder, - events.EventRecorder, - *capsulev1beta2.Tenant, -) handlers.Func { - return func(context.Context, admission.Request) *admission.Response { - return nil - } -} - -func (h pv) handle( - ctx context.Context, - c client.Client, - pvc *corev1.PersistentVolumeClaim, - tenantName string, -) (*corev1.PersistentVolume, error) { - if pvc.Spec.Selector != nil { - return nil, caperrors.NewPVSelectorError(evt.ActionValidationDenied) - } - - if pvc.Spec.VolumeName == "" { - return nil, nil - } - - pv := &corev1.PersistentVolume{} - if err := c.Get(ctx, types.NamespacedName{Name: pvc.Spec.VolumeName}, pv); err != nil { - if errors.IsNotFound(err) { - return nil, caperrors.NewPvNotFoundError( - pvc.Spec.VolumeName, - evt.ActionValidationDenied, - ) - } - - return nil, err - } - - labels := pv.GetLabels() - - value, ok := labels[meta.TenantLabel] - if !ok { - return pv, caperrors.NewMissingTenantPVLabelsError( - pv.GetName(), - evt.ActionValidationDenied, - ) - } - - if value != tenantName { - return pv, caperrors.NewCrossTenantPVMountError(pv.GetName(), evt.ActionValidationDenied) - } - - return pv, nil -} diff --git a/internal/webhook/pvc/pvc_mutating_volume.go b/internal/webhook/pvc/pvc_mutating_volume.go new file mode 100644 index 00000000..7b9a0ee0 --- /dev/null +++ b/internal/webhook/pvc/pvc_mutating_volume.go @@ -0,0 +1,147 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package pvc + +import ( + "context" + "encoding/json" + "net/http" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" +) + +type persistentVolumeMutatingVolume struct{} + +func PersistentVolumeMutatingVolume() handlers.TypedHandlerWithTenant[*corev1.PersistentVolumeClaim] { + return &persistentVolumeMutatingVolume{} +} + +func (h persistentVolumeMutatingVolume) OnCreate( + _ client.Client, + _ client.Reader, + pvc *corev1.PersistentVolumeClaim, + _ admission.Decoder, + recorder events.EventRecorder, + tnt *capsulev1beta2.Tenant, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + // Kubernetes does not dynamically provision PVs for PVCs with a non-empty selector. + // Therefore, only mutate PVCs that already opted into static binding semantics: + // - either by setting spec.selector + // - or by pre-binding through spec.volumeName + if pvc.Spec.Selector == nil && pvc.Spec.VolumeName == "" { + return nil + } + + pvc.Spec.Selector = addTenantSelectorExpression(pvc.Spec.Selector, tnt.Name) + + marshaled, err := json.Marshal(pvc) + if err != nil { + response := admission.Errored(http.StatusInternalServerError, err) + + return &response + } + + response := admission.PatchResponseFromRaw(req.Object.Raw, marshaled) + + return &response + } +} + +func (h persistentVolumeMutatingVolume) OnUpdate( + _ client.Client, + _ client.Reader, + oldPVC *corev1.PersistentVolumeClaim, + newPVC *corev1.PersistentVolumeClaim, + _ admission.Decoder, + recorder events.EventRecorder, + tnt *capsulev1beta2.Tenant, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + if newPVC == nil || tnt == nil { + return nil + } + + // Avoid mutating normal dynamically provisioned PVCs. + // + // Only canonicalize tenant selector if the PVC already participates in + // static binding semantics. + if newPVC.Spec.Selector == nil { + return nil + } + + newPVC.Spec.Selector = addTenantSelectorExpression(newPVC.Spec.Selector, tnt.Name) + + marshaled, err := json.Marshal(newPVC) + if err != nil { + response := admission.Errored(http.StatusInternalServerError, err) + + return &response + } + + response := admission.PatchResponseFromRaw(req.Object.Raw, marshaled) + + return &response + } +} + +func (h persistentVolumeMutatingVolume) OnDelete( + client.Client, + client.Reader, + *corev1.PersistentVolumeClaim, + admission.Decoder, + events.EventRecorder, + *capsulev1beta2.Tenant, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} + +func addTenantSelectorExpression( + selector *metav1.LabelSelector, + tenantName string, +) *metav1.LabelSelector { + if selector == nil { + selector = &metav1.LabelSelector{} + } + + // Remove tenant label from MatchLabels to avoid conflicting requirements. + if selector.MatchLabels != nil { + delete(selector.MatchLabels, meta.TenantLabel) + + if len(selector.MatchLabels) == 0 { + selector.MatchLabels = nil + } + } + + // Remove any existing tenant expression, regardless of operator or value. + matchExpressions := make([]metav1.LabelSelectorRequirement, 0, len(selector.MatchExpressions)) + + for _, expression := range selector.MatchExpressions { + if expression.Key == meta.TenantLabel { + continue + } + + matchExpressions = append(matchExpressions, expression) + } + + matchExpressions = append(matchExpressions, metav1.LabelSelectorRequirement{ + Key: meta.TenantLabel, + Operator: metav1.LabelSelectorOpIn, + Values: []string{tenantName}, + }) + + selector.MatchExpressions = matchExpressions + + return selector +} diff --git a/internal/webhook/pvc/validating.go b/internal/webhook/pvc/pvc_validating_class.go similarity index 72% rename from internal/webhook/pvc/validating.go rename to internal/webhook/pvc/pvc_validating_class.go index 108d3459..30006406 100644 --- a/internal/webhook/pvc/validating.go +++ b/internal/webhook/pvc/pvc_validating_class.go @@ -8,26 +8,28 @@ import ( "net/http" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/webhook/utils" - caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) -type validating struct{} +type persistentVolumeValidatingClass struct{} -func Validating() handlers.TypedHandlerWithTenant[*corev1.PersistentVolumeClaim] { - return &validating{} +func PersistentVolumeValidatingClass() handlers.TypedHandlerWithTenant[*corev1.PersistentVolumeClaim] { + return &persistentVolumeValidatingClass{} } -func (h *validating) OnCreate( - c client.Client, +func (h *persistentVolumeValidatingClass) OnCreate( + _ client.Client, + reader client.Reader, pvc *corev1.PersistentVolumeClaim, decoder admission.Decoder, recorder events.EventRecorder, @@ -52,17 +54,15 @@ func (h *validating) OnCreate( "Requires a StorageClass", ) - response := admission.Denied(caperrors.NewStorageClassNotValid(*tnt.Spec.StorageClasses).Error()) - - return &response + return ad.Deny(errors.NewStorageClassNotValid(*tnt.Spec.StorageClasses).Error()) } selector := false // Verify if the StorageClass exists and matches the label selector/expression if len(allowed.MatchExpressions) > 0 || len(allowed.MatchLabels) > 0 { - storageClassObj, err := utils.GetStorageClassByName(ctx, c, *storageClass) - if err != nil && !errors.IsNotFound(err) { + storageClassObj, err := utils.GetStorageClassByName(ctx, reader, *storageClass) + if err != nil && !apierrors.IsNotFound(err) { response := admission.Errored(http.StatusInternalServerError, err) return &response @@ -88,15 +88,14 @@ func (h *validating) OnCreate( evt.ActionValidationDenied, "StorageClass %s is forbidden for the Tenant %s", *storageClass, tnt.GetName()) - response := admission.Denied(caperrors.NewStorageClassForbidden(*pvc.Spec.StorageClassName, *tnt.Spec.StorageClasses).Error()) - - return &response + return ad.Deny(errors.NewStorageClassForbidden(*pvc.Spec.StorageClassName, *tnt.Spec.StorageClasses).Error()) } } } -func (h *validating) OnUpdate( +func (h *persistentVolumeValidatingClass) OnUpdate( client.Client, + client.Reader, *corev1.PersistentVolumeClaim, *corev1.PersistentVolumeClaim, admission.Decoder, @@ -108,8 +107,9 @@ func (h *validating) OnUpdate( } } -func (h *validating) OnDelete( +func (h *persistentVolumeValidatingClass) OnDelete( client.Client, + client.Reader, *corev1.PersistentVolumeClaim, admission.Decoder, events.EventRecorder, diff --git a/internal/webhook/pvc/pvc_validating_volume.go b/internal/webhook/pvc/pvc_validating_volume.go new file mode 100644 index 00000000..9fad9b16 --- /dev/null +++ b/internal/webhook/pvc/pvc_validating_volume.go @@ -0,0 +1,158 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package pvc + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/errors" + "github.com/projectcapsule/capsule/pkg/api/meta" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" + evt "github.com/projectcapsule/capsule/pkg/runtime/events" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" +) + +type persistentVolumeValidatingVolume struct{} + +func PersistentVolumeValidatingVolume() handlers.TypedHandlerWithTenant[*corev1.PersistentVolumeClaim] { + return &persistentVolumeValidatingVolume{} +} + +func (h persistentVolumeValidatingVolume) OnCreate( + _ client.Client, + reader client.Reader, + pvc *corev1.PersistentVolumeClaim, + decoder admission.Decoder, + recorder events.EventRecorder, + tnt *capsulev1beta2.Tenant, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + if err := validatePVCSelector(pvc, tnt); err != nil { + return ad.ErroredResponse(err) + } + + return validatePVCVolumeName(ctx, reader, pvc, tnt) + } +} + +func (h persistentVolumeValidatingVolume) OnUpdate( + _ client.Client, + reader client.Reader, + oldPVC *corev1.PersistentVolumeClaim, + newPVC *corev1.PersistentVolumeClaim, + decoder admission.Decoder, + recorder events.EventRecorder, + tnt *capsulev1beta2.Tenant, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + if err := validatePVCSelector(newPVC, tnt); err != nil { + return ad.ErroredResponse(err) + } + + return validatePVCVolumeName(ctx, reader, newPVC, tnt) + } +} + +func (h persistentVolumeValidatingVolume) OnDelete( + client.Client, + client.Reader, + *corev1.PersistentVolumeClaim, + admission.Decoder, + events.EventRecorder, + *capsulev1beta2.Tenant, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} + +func validatePVCSelector( + pvc *corev1.PersistentVolumeClaim, + tnt *capsulev1beta2.Tenant, +) error { + if pvc == nil || tnt == nil || pvc.Spec.Selector == nil { + return nil + } + + for _, expression := range pvc.Spec.Selector.MatchExpressions { + if expression.Key != meta.TenantLabel { + continue + } + + if expression.Operator != metav1.LabelSelectorOpIn { + return fmt.Errorf( + "PVC selector expression for %q must use operator %q", + meta.TenantLabel, + metav1.LabelSelectorOpIn, + ) + } + + if len(expression.Values) != 1 || expression.Values[0] != tnt.Name { + return fmt.Errorf( + "PVC selector expression for %q must contain only tenant %q", + meta.TenantLabel, + tnt.Name, + ) + } + + return nil + } + + return fmt.Errorf( + "PVC selector must include tenant selector expression %q In [%q]", + meta.TenantLabel, + tnt.Name, + ) +} + +func validatePVCVolumeName( + ctx context.Context, + c client.Reader, + pvc *corev1.PersistentVolumeClaim, + tnt *capsulev1beta2.Tenant, +) *admission.Response { + if pvc == nil || tnt == nil { + return nil + } + + // The PVC hasn't any volumeName pre-claimed, it can be skipped. + if pvc.Spec.VolumeName == "" { + return nil + } + + // Checking if the PV is labelled with the Tenant name. + pv := corev1.PersistentVolume{} + if err := c.Get(ctx, types.NamespacedName{Name: pvc.Spec.VolumeName}, &pv); err != nil { + if apierrors.IsNotFound(err) { + err = fmt.Errorf("cannot create a PVC referring to a not yet existing PV") + } + + return ad.ErroredResponse(err) + } + + if pv.GetLabels() == nil { + return ad.Deny(errors.NewMissingTenantPVLabelsError(pv.GetName(), evt.ActionValidationDenied).Error()) + } + + value, ok := pv.GetLabels()[meta.TenantLabel] + if !ok { + return ad.Deny(errors.NewMissingTenantPVLabelsError(pv.GetName(), evt.ActionValidationDenied).Error()) + } + + if value != tnt.Name { + return ad.Deny(errors.NewCrossTenantPVMountError(pv.GetName(), evt.ActionValidationDenied).Error()) + } + + return nil +} diff --git a/internal/webhook/resourcepool/claim_mutating.go b/internal/webhook/resourcepool/claim_mutating.go index cd097ad4..6be53207 100644 --- a/internal/webhook/resourcepool/claim_mutating.go +++ b/internal/webhook/resourcepool/claim_mutating.go @@ -15,8 +15,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/api/meta" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -28,21 +28,36 @@ func ClaimMutationHandler(log logr.Logger) handlers.Handler { return &claimMutationHandler{log: log} } -func (h *claimMutationHandler) OnUpdate(c client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (h *claimMutationHandler) OnUpdate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.handle(ctx, req, decoder, c, h.handleReleaseAnnotation) + return h.handle(ctx, c, req, decoder, h.handleReleaseAnnotation) } } -func (h *claimMutationHandler) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *claimMutationHandler) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *claimMutationHandler) OnCreate(c client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (h *claimMutationHandler) OnCreate( + c client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.handle(ctx, req, decoder, c, func(claim *capsulev1beta2.ResourcePoolClaim) { + return h.handle(ctx, c, req, decoder, func(claim *capsulev1beta2.ResourcePoolClaim) { meta.ReleaseAnnotationRemove(claim) }) } @@ -50,15 +65,15 @@ func (h *claimMutationHandler) OnCreate(c client.Client, decoder admission.Decod func (h *claimMutationHandler) handle( ctx context.Context, + c client.Client, req admission.Request, decoder admission.Decoder, - c client.Client, annoHandler func(c *capsulev1beta2.ResourcePoolClaim), ) *admission.Response { claim := &capsulev1beta2.ResourcePoolClaim{} if err := decoder.Decode(req, claim); err != nil { - return utils.ErroredResponse(fmt.Errorf("failed to decode new object: %w", err)) + return ad.ErroredResponse(fmt.Errorf("failed to decode new object: %w", err)) } annoHandler(claim) diff --git a/internal/webhook/resourcepool/claim_validating.go b/internal/webhook/resourcepool/claim_validating.go index 13cf3356..7d70f99d 100644 --- a/internal/webhook/resourcepool/claim_validating.go +++ b/internal/webhook/resourcepool/claim_validating.go @@ -14,7 +14,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -26,48 +26,59 @@ func ClaimValidationHandler(log logr.Logger) handlers.Handler { return &claimValidationHandler{log: log} } -func (h *claimValidationHandler) OnCreate(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *claimValidationHandler) OnCreate( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *claimValidationHandler) OnDelete(_ client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (h *claimValidationHandler) OnDelete( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(_ context.Context, req admission.Request) *admission.Response { claim := &capsulev1beta2.ResourcePoolClaim{} if err := decoder.DecodeRaw(req.OldObject, claim); err != nil { - return utils.ErroredResponse(fmt.Errorf("failed to decode old object: %w", err)) + return ad.ErroredResponse(fmt.Errorf("failed to decode old object: %w", err)) } if claim.IsBoundInResourcePool() { - response := admission.Denied(fmt.Sprintf("cannot delete the pool while claim is used in resourcepool %s", claim.Status.Pool.Name)) - - return &response + return ad.Deny(fmt.Sprintf("cannot delete the pool while claim is used in resourcepool %s", claim.Status.Pool.Name)) } return nil } } -func (h *claimValidationHandler) OnUpdate(_ client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (h *claimValidationHandler) OnUpdate( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(_ context.Context, req admission.Request) *admission.Response { oldClaim := &capsulev1beta2.ResourcePoolClaim{} newClaim := &capsulev1beta2.ResourcePoolClaim{} if err := decoder.DecodeRaw(req.OldObject, oldClaim); err != nil { - return utils.ErroredResponse(fmt.Errorf("failed to decode old object: %w", err)) + return ad.ErroredResponse(fmt.Errorf("failed to decode old object: %w", err)) } if err := decoder.Decode(req, newClaim); err != nil { - return utils.ErroredResponse(fmt.Errorf("failed to decode new object: %w", err)) + return ad.ErroredResponse(fmt.Errorf("failed to decode new object: %w", err)) } if oldClaim.IsBoundInResourcePool() { if oldClaim.Spec.Pool != newClaim.Spec.Pool || !reflect.DeepEqual(oldClaim.Spec.ResourceClaims, newClaim.Spec.ResourceClaims) { - response := admission.Denied(fmt.Sprintf("cannot change the requested resources while claim is allocated to a resourcepool %s", oldClaim.Status.Pool.Name)) - - return &response + return ad.Deny(fmt.Sprintf("cannot change the requested resources while claim is allocated to a resourcepool %s", oldClaim.Status.Pool.Name)) } } diff --git a/internal/webhook/resourcepool/pool_mutating.go b/internal/webhook/resourcepool/pool_mutating.go index a36248b1..471c00c8 100644 --- a/internal/webhook/resourcepool/pool_mutating.go +++ b/internal/webhook/resourcepool/pool_mutating.go @@ -17,7 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -29,19 +29,34 @@ func PoolMutationHandler(log logr.Logger) handlers.Handler { return &poolMutationHandler{log: log} } -func (h *poolMutationHandler) OnCreate(_ client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (h *poolMutationHandler) OnCreate( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(_ context.Context, req admission.Request) *admission.Response { return h.handle(req, decoder) } } -func (h *poolMutationHandler) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *poolMutationHandler) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *poolMutationHandler) OnUpdate(_ client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (h *poolMutationHandler) OnUpdate( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(_ context.Context, req admission.Request) *admission.Response { return h.handle(req, decoder) } @@ -53,7 +68,7 @@ func (h *poolMutationHandler) handle( ) *admission.Response { pool := &capsulev1beta2.ResourcePool{} if err := decoder.Decode(req, pool); err != nil { - return utils.ErroredResponse(fmt.Errorf("failed to decode object: %w", err)) + return ad.ErroredResponse(fmt.Errorf("failed to decode object: %w", err)) } // Correctly set the defaults diff --git a/internal/webhook/resourcepool/pool_validation.go b/internal/webhook/resourcepool/pool_validation.go index 221d5ece..b8b5f51c 100644 --- a/internal/webhook/resourcepool/pool_validation.go +++ b/internal/webhook/resourcepool/pool_validation.go @@ -15,7 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -27,28 +27,43 @@ func PoolValidationHandler(log logr.Logger) handlers.Handler { return &poolValidationHandler{log: log} } -func (h *poolValidationHandler) OnCreate(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *poolValidationHandler) OnCreate( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *poolValidationHandler) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *poolValidationHandler) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *poolValidationHandler) OnUpdate(_ client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (h *poolValidationHandler) OnUpdate( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(_ context.Context, req admission.Request) *admission.Response { oldPool := &capsulev1beta2.ResourcePool{} if err := decoder.DecodeRaw(req.OldObject, oldPool); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } pool := &capsulev1beta2.ResourcePool{} if err := decoder.Decode(req, pool); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } // Verify if resource decrease is allowed or no @@ -64,24 +79,23 @@ func (h *poolValidationHandler) OnUpdate(_ client.Client, decoder admission.Deco continue } - response := admission.Denied(fmt.Sprintf( - "can not remove resource %s as it is still being allocated. Remove corresponding claims or keep the resources in the pool", - resourceName, - )) - - return &response + return ad.Deny( + fmt.Sprintf( + "can not remove resource %s as it is still being allocated. Remove corresponding claims or keep the resources in the pool", + resourceName, + ), + ) } if allocation.Cmp(qt) < 0 { - response := admission.Denied( + return ad.Deny( fmt.Sprintf( "can not reduce %s usage to %s because quantity %s is claimed . Remove corresponding claims or keep the resources in the pool", resourceName, allocation.String(), qt.String(), - )) - - return &response + ), + ) } } } diff --git a/internal/webhook/route/cordoning.go b/internal/webhook/route/cordoning.go index 13e143e5..e4b84d57 100644 --- a/internal/webhook/route/cordoning.go +++ b/internal/webhook/route/cordoning.go @@ -14,7 +14,7 @@ func Cordoning(handlers ...handlers.Handler) handlers.Webhook { } func (w cordoning) GetPath() string { - return "/misc/cordoning" + return "/generic/cordoning" } func (w cordoning) GetHandlers() []handlers.Handler { diff --git a/internal/webhook/route/customquota.go b/internal/webhook/route/customquota.go new file mode 100644 index 00000000..7c12efe4 --- /dev/null +++ b/internal/webhook/route/customquota.go @@ -0,0 +1,54 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package route + +import "github.com/projectcapsule/capsule/pkg/runtime/handlers" + +type customQuotaValidation struct { + handlers []handlers.Handler +} + +func CustomQuotaValidation(handler ...handlers.Handler) handlers.Webhook { + return &customQuotaValidation{handlers: handler} +} + +func (w *customQuotaValidation) GetHandlers() []handlers.Handler { + return w.handlers +} + +func (w *customQuotaValidation) GetPath() string { + return "/custom-quotas/namespaced/validating" +} + +type globalCustomQuotaValidation struct { + handlers []handlers.Handler +} + +func GlobalCustomQuotaValidation(handler ...handlers.Handler) handlers.Webhook { + return &globalCustomQuotaValidation{handlers: handler} +} + +func (w *globalCustomQuotaValidation) GetHandlers() []handlers.Handler { + return w.handlers +} + +func (w *globalCustomQuotaValidation) GetPath() string { + return "/custom-quotas/cluster/validating" +} + +type customQuotasCalculation struct { + handlers []handlers.Handler +} + +func CalculationCustomQuotas(handler ...handlers.Handler) handlers.Webhook { + return &customQuotasCalculation{handlers: handler} +} + +func (w *customQuotasCalculation) GetHandlers() []handlers.Handler { + return w.handlers +} + +func (w *customQuotasCalculation) GetPath() string { + return "/custom-quotas/calculations" +} diff --git a/internal/webhook/route/generic.go b/internal/webhook/route/generic.go new file mode 100644 index 00000000..371ae792 --- /dev/null +++ b/internal/webhook/route/generic.go @@ -0,0 +1,76 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package route + +import ( + "github.com/projectcapsule/capsule/internal/webhook/generic" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" +) + +type replicasResourcesHandler struct{} + +func GenericReplicasHandler() handlers.Webhook { + return &replicasResourcesHandler{} +} + +func (w *replicasResourcesHandler) GetHandlers() []handlers.Handler { + return []handlers.Handler{ + generic.ReplicaHandler(), + } +} + +func (w *replicasResourcesHandler) GetPath() string { + return "/generic/replications" +} + +type genericCustomResourcesHandler struct { + handlers []handlers.Handler +} + +func GenericCustomResources(handlers ...handlers.Handler) handlers.Webhook { + return &genericCustomResourcesHandler{handlers: handlers} +} + +func (w *genericCustomResourcesHandler) GetHandlers() []handlers.Handler { + return w.handlers +} + +func (w *genericCustomResourcesHandler) GetPath() string { + return "/generic/customresources" +} + +type genericMetadataAssignment struct { + handlers []handlers.Handler +} + +func GenericTenantAssignment(handlers ...handlers.Handler) handlers.Webhook { + return &genericMetadataAssignment{handlers: handlers} +} + +func (w genericMetadataAssignment) GetPath() string { + return "/generic/metadata" +} + +func (w genericMetadataAssignment) GetHandlers() []handlers.Handler { + return w.handlers +} + +type miscManagedValidation struct { + configuration configuration.Configuration +} + +func GenericManagedHandler(cfg configuration.Configuration) handlers.Webhook { + return &miscManagedValidation{configuration: cfg} +} + +func (t miscManagedValidation) GetPath() string { + return "/generic/managed" +} + +func (t miscManagedValidation) GetHandlers() []handlers.Handler { + return []handlers.Handler{ + generic.ManagedValidatingHandler(t.configuration), + } +} diff --git a/internal/webhook/route/misc.go b/internal/webhook/route/misc.go deleted file mode 100644 index da357fcf..00000000 --- a/internal/webhook/route/misc.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package route - -import "github.com/projectcapsule/capsule/pkg/runtime/handlers" - -type miscCustomResourcesHandler struct { - handlers []handlers.Handler -} - -func MiscCustomResources(handlers ...handlers.Handler) handlers.Webhook { - return &miscCustomResourcesHandler{handlers: handlers} -} - -func (w *miscCustomResourcesHandler) GetHandlers() []handlers.Handler { - return w.handlers -} - -func (w *miscCustomResourcesHandler) GetPath() string { - return "/misc/customresources" -} - -type miscTenantAssignment struct { - handlers []handlers.Handler -} - -func MiscTenantAssignment(handlers ...handlers.Handler) handlers.Webhook { - return &miscTenantAssignment{handlers: handlers} -} - -func (w miscTenantAssignment) GetPath() string { - return "/misc/tenant-label" -} - -func (w miscTenantAssignment) GetHandlers() []handlers.Handler { - return w.handlers -} - -type miscManagedValidation struct { - handlers []handlers.Handler -} - -func MiscManagedValidation(handlers ...handlers.Handler) handlers.Webhook { - return &miscManagedValidation{handlers: handlers} -} - -func (t miscManagedValidation) GetPath() string { - return "/misc/managed" -} - -func (t miscManagedValidation) GetHandlers() []handlers.Handler { - return t.handlers -} diff --git a/internal/webhook/route/pvc.go b/internal/webhook/route/pvc.go index ec2899e7..59ed9e8c 100644 --- a/internal/webhook/route/pvc.go +++ b/internal/webhook/route/pvc.go @@ -3,20 +3,38 @@ package route -import "github.com/projectcapsule/capsule/pkg/runtime/handlers" +import ( + "github.com/projectcapsule/capsule/pkg/runtime/handlers" +) -type pvc struct { +type pvcValidating struct { handlers []handlers.Handler } -func PVC(handler ...handlers.Handler) handlers.Webhook { - return &pvc{handlers: handler} +func PVCValidating(handler ...handlers.Handler) handlers.Webhook { + return &pvcValidating{handlers: handler} } -func (w *pvc) GetHandlers() []handlers.Handler { +func (w *pvcValidating) GetHandlers() []handlers.Handler { return w.handlers } -func (w *pvc) GetPath() string { +func (pvcValidating) GetPath() string { return "/persistentvolumeclaims/validating" } + +type pvcMutating struct { + handlers []handlers.Handler +} + +func PVCMutating(handler ...handlers.Handler) handlers.Webhook { + return &pvcMutating{handlers: handler} +} + +func (w *pvcMutating) GetHandlers() []handlers.Handler { + return w.handlers +} + +func (pvcMutating) GetPath() string { + return "/persistentvolumeclaims/mutating" +} diff --git a/internal/webhook/route/tenantresource_objs.go b/internal/webhook/route/tenantresource_objs.go deleted file mode 100644 index 2b7cc417..00000000 --- a/internal/webhook/route/tenantresource_objs.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package route - -import "github.com/projectcapsule/capsule/pkg/runtime/handlers" - -type tntResourceObjs struct { - handlers []handlers.Handler -} - -func TenantResourceObjects(handlers ...handlers.Handler) handlers.Webhook { - return &tntResourceObjs{handlers: handlers} -} - -func (t tntResourceObjs) GetPath() string { - return "/tenantresource-objects" -} - -func (t tntResourceObjs) GetHandlers() []handlers.Handler { - return t.handlers -} diff --git a/internal/webhook/router.go b/internal/webhook/router.go index c1f61f37..094aba52 100644 --- a/internal/webhook/router.go +++ b/internal/webhook/router.go @@ -25,6 +25,7 @@ func Register(manager controllerruntime.Manager, webhookList ...handlers.Webhook server.Register(wh.GetPath(), &webhook.Admission{ Handler: &handlerRouter{ client: manager.GetClient(), + reader: manager.GetAPIReader(), decoder: admission.NewDecoder(manager.GetScheme()), recorder: recorder, handlers: wh.GetHandlers(), @@ -37,6 +38,7 @@ func Register(manager controllerruntime.Manager, webhookList ...handlers.Webhook type handlerRouter struct { client client.Client + reader client.Reader decoder admission.Decoder recorder events.EventRecorder @@ -47,19 +49,19 @@ func (r *handlerRouter) Handle(ctx context.Context, req admission.Request) admis switch req.Operation { case admissionv1.Create: for _, h := range r.handlers { - if response := h.OnCreate(r.client, r.decoder, r.recorder)(ctx, req); response != nil { + if response := h.OnCreate(r.client, r.reader, r.decoder, r.recorder)(ctx, req); response != nil { return *response } } case admissionv1.Update: for _, h := range r.handlers { - if response := h.OnUpdate(r.client, r.decoder, r.recorder)(ctx, req); response != nil { + if response := h.OnUpdate(r.client, r.reader, r.decoder, r.recorder)(ctx, req); response != nil { return *response } } case admissionv1.Delete: for _, h := range r.handlers { - if response := h.OnDelete(r.client, r.decoder, r.recorder)(ctx, req); response != nil { + if response := h.OnDelete(r.client, r.reader, r.decoder, r.recorder)(ctx, req); response != nil { return *response } } diff --git a/internal/webhook/service/validating.go b/internal/webhook/service/validating.go index e3031cd0..d38cf211 100644 --- a/internal/webhook/service/validating.go +++ b/internal/webhook/service/validating.go @@ -17,6 +17,7 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" caperrors "github.com/projectcapsule/capsule/pkg/api/errors" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -28,7 +29,8 @@ func Validating() handlers.TypedHandlerWithTenant[*corev1.Service] { } func (h *validating) OnCreate( - c client.Client, + _ client.Client, + _ client.Reader, svc *corev1.Service, decoder admission.Decoder, recorder events.EventRecorder, @@ -40,7 +42,8 @@ func (h *validating) OnCreate( } func (h *validating) OnUpdate( - c client.Client, + _ client.Client, + _ client.Reader, old *corev1.Service, svc *corev1.Service, decoder admission.Decoder, @@ -54,6 +57,7 @@ func (h *validating) OnUpdate( func (h *validating) OnDelete( client.Client, + client.Reader, *corev1.Service, admission.Decoder, events.EventRecorder, @@ -80,9 +84,7 @@ func (h *validating) handle( "Cannot be type of NodePort for the Tenant %s", tnt.GetName(), ) - response := admission.Denied(caperrors.NewNodePortDisabledError().Error()) - - return &response + return ad.Deny(caperrors.NewExternalNameDisabledError().Error()) } if svc.Spec.Type == corev1.ServiceTypeExternalName && tnt.Spec.ServiceOptions != nil && tnt.Spec.ServiceOptions.AllowedServices != nil && !*tnt.Spec.ServiceOptions.AllowedServices.ExternalName { @@ -95,24 +97,20 @@ func (h *validating) handle( "Cannot be type of ExternalName for the Tenant %s", tnt.GetName(), ) - response := admission.Denied(caperrors.NewExternalNameDisabledError().Error()) - - return &response + return ad.Deny(caperrors.NewExternalNameDisabledError().Error()) } if svc.Spec.Type == corev1.ServiceTypeLoadBalancer && tnt.Spec.ServiceOptions != nil && tnt.Spec.ServiceOptions.AllowedServices != nil && !*tnt.Spec.ServiceOptions.AllowedServices.LoadBalancer { recorder.Eventf( - tnt, svc, + tnt, corev1.EventTypeWarning, evt.ReasonForbiddenLoadBalancer, evt.ActionValidationDenied, "Cannot be type of LoadBalancer for the Tenant %s", tnt.GetName(), ) - response := admission.Denied(caperrors.NewLoadBalancerDisabled().Error()) - - return &response + return ad.Deny(caperrors.NewLoadBalancerDisabled().Error()) } if tnt.Spec.ServiceOptions != nil { @@ -129,9 +127,7 @@ func (h *validating) handle( err.Error(), ) - response := admission.Denied(err.Error()) - - return &response + return ad.Deny(err.Error()) } err = api.ValidateForbidden(svc.Labels, tnt.Spec.ServiceOptions.ForbiddenLabels) @@ -147,9 +143,7 @@ func (h *validating) handle( err.Error(), ) - response := admission.Denied(err.Error()) - - return &response + return ad.Deny(err.Error()) } } @@ -186,9 +180,7 @@ func (h *validating) handle( "External IP %s is forbidden for the Tenant %s", ip.String(), tnt.GetName(), ) - response := admission.Denied(caperrors.NewExternalServiceIPForbidden(tnt.Spec.ServiceOptions.ExternalServiceIPs.Allowed).Error()) - - return &response + return ad.Deny(caperrors.NewExternalServiceIPForbidden(tnt.Spec.ServiceOptions.ExternalServiceIPs.Allowed).Error()) } } diff --git a/internal/webhook/serviceaccounts/handler.go b/internal/webhook/serviceaccounts/handler.go index 328b6e47..186fe9cc 100644 --- a/internal/webhook/serviceaccounts/handler.go +++ b/internal/webhook/serviceaccounts/handler.go @@ -6,14 +6,16 @@ package serviceaccounts import ( corev1 "k8s.io/api/core/v1" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) -func Handler(handler ...handlers.TypedHandlerWithTenant[*corev1.ServiceAccount]) handlers.Handler { - return &handlers.TypedTenantHandler[*corev1.ServiceAccount]{ +func Handler(cfg configuration.Configuration, handler ...handlers.TypedHandlerWithTenantUser[*corev1.ServiceAccount]) handlers.Handler { + return &handlers.TypedTenantWithUserHandler[*corev1.ServiceAccount]{ Factory: func() *corev1.ServiceAccount { return &corev1.ServiceAccount{} }, - Handlers: handler, + Handlers: handler, + Configuration: cfg, } } diff --git a/internal/webhook/serviceaccounts/owner_promotion.go b/internal/webhook/serviceaccounts/owner_promotion.go new file mode 100644 index 00000000..f3c9b271 --- /dev/null +++ b/internal/webhook/serviceaccounts/owner_promotion.go @@ -0,0 +1,113 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package serviceaccounts + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + evt "github.com/projectcapsule/capsule/pkg/runtime/events" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" + "github.com/projectcapsule/capsule/pkg/users" +) + +type ownerPromotion struct { + cfg configuration.Configuration +} + +func OwnerPromotion(cfg configuration.Configuration) handlers.TypedHandlerWithTenantUser[*corev1.ServiceAccount] { + return &ownerPromotion{cfg: cfg} +} + +func (h *ownerPromotion) OnCreate( + _ client.Client, + _ client.Reader, + user users.AdmissionUser, + sa *corev1.ServiceAccount, + decoder admission.Decoder, + recorder events.EventRecorder, + tnt *capsulev1beta2.Tenant, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return h.handle(user, recorder, sa, tnt) + } +} + +func (h *ownerPromotion) OnUpdate( + _ client.Client, + _ client.Reader, + user users.AdmissionUser, + old *corev1.ServiceAccount, + sa *corev1.ServiceAccount, + decoder admission.Decoder, + recorder events.EventRecorder, + tnt *capsulev1beta2.Tenant, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return h.handle(user, recorder, sa, tnt) + } +} + +func (h *ownerPromotion) OnDelete( + client.Client, + client.Reader, + users.AdmissionUser, + *corev1.ServiceAccount, + admission.Decoder, + events.EventRecorder, + *capsulev1beta2.Tenant, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} + +func (h *ownerPromotion) handle( + user users.AdmissionUser, + recorder events.EventRecorder, + sa *corev1.ServiceAccount, + tnt *capsulev1beta2.Tenant, +) *admission.Response { + _, hasOwnerPromotion := sa.Labels[meta.OwnerPromotionLabel] + if !hasOwnerPromotion { + return nil + } + + if !h.cfg.AllowServiceAccountPromotion() { + return ad.Deny("service account owner promotion is disabled. Contact your system administrators") + } + + if !tnt.Spec.Permissions.AllowOwnerPromotion { + return ad.Deny("service account owner promotion is disabled for this tenant. Contact your system administrators") + } + + // We don't want to allow promoted serviceaccounts to promote other serviceaccounts + if ok := users.IsTenantOwnerByStatus(tnt, user); ok { + return nil + } + + msg := fmt.Sprintf("%s not allowed to promote serviceaccount to tenant owner", user.Username) + + recorder.Eventf( + sa, + tnt, + corev1.EventTypeWarning, + evt.ReasonPromotionDenied, + evt.ActionValidationDenied, + msg, + ) + + response := admission.Denied(msg) + + return &response +} diff --git a/internal/webhook/serviceaccounts/validating.go b/internal/webhook/serviceaccounts/promotion.go similarity index 68% rename from internal/webhook/serviceaccounts/validating.go rename to internal/webhook/serviceaccounts/promotion.go index 4325ff56..da70aa7b 100644 --- a/internal/webhook/serviceaccounts/validating.go +++ b/internal/webhook/serviceaccounts/promotion.go @@ -14,34 +14,39 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api/meta" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" "github.com/projectcapsule/capsule/pkg/users" ) -type validating struct { +type promotion struct { cfg configuration.Configuration } -func Validating(cfg configuration.Configuration) handlers.TypedHandlerWithTenant[*corev1.ServiceAccount] { - return &validating{cfg: cfg} +func Promotion(cfg configuration.Configuration) handlers.TypedHandlerWithTenantUser[*corev1.ServiceAccount] { + return &promotion{cfg: cfg} } -func (h *validating) OnCreate( - c client.Client, +func (h *promotion) OnCreate( + _ client.Client, + _ client.Reader, + user users.AdmissionUser, sa *corev1.ServiceAccount, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.handle(ctx, c, req, recorder, sa, tnt) + return h.handle(user, recorder, sa, tnt) } } -func (h *validating) OnUpdate( - c client.Client, +func (h *promotion) OnUpdate( + _ client.Client, + _ client.Reader, + user users.AdmissionUser, old *corev1.ServiceAccount, sa *corev1.ServiceAccount, decoder admission.Decoder, @@ -49,12 +54,14 @@ func (h *validating) OnUpdate( tnt *capsulev1beta2.Tenant, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { - return h.handle(ctx, c, req, recorder, sa, tnt) + return h.handle(user, recorder, sa, tnt) } } -func (h *validating) OnDelete( +func (h *promotion) OnDelete( client.Client, + client.Reader, + users.AdmissionUser, *corev1.ServiceAccount, admission.Decoder, events.EventRecorder, @@ -65,33 +72,27 @@ func (h *validating) OnDelete( } } -func (h *validating) handle( - ctx context.Context, - c client.Client, - req admission.Request, +func (h *promotion) handle( + user users.AdmissionUser, recorder events.EventRecorder, sa *corev1.ServiceAccount, tnt *capsulev1beta2.Tenant, ) *admission.Response { - _, hasOwnerPromotion := sa.Labels[meta.OwnerPromotionLabel] - if !hasOwnerPromotion { + _, hasPromotion := sa.Labels[meta.ServiceAccountPromotionLabel] + if !hasPromotion { return nil } if !h.cfg.AllowServiceAccountPromotion() { - response := admission.Denied( - "service account owner promotion is disabled. Contact your system administrators", - ) - - return &response + return ad.Deny("service account promotion is disabled. Contact cluster administrators") } // We don't want to allow promoted serviceaccounts to promote other serviceaccounts - if ok := users.IsTenantOwnerByStatus(ctx, c, h.cfg, tnt, req.UserInfo); ok { + if ok := users.IsTenantOwnerByStatus(tnt, user); ok { return nil } - msg := fmt.Sprintf("%s not allowed to promote serviceaccount to tenant owner", req.UserInfo.Username) + msg := fmt.Sprintf("%s not allowed to promote serviceaccount to tenant owner", user.Username) recorder.Eventf( sa, diff --git a/internal/webhook/tenant/mutation/metadata.go b/internal/webhook/tenant/mutation/metadata.go index 922bc649..b4df39e0 100644 --- a/internal/webhook/tenant/mutation/metadata.go +++ b/internal/webhook/tenant/mutation/metadata.go @@ -13,8 +13,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/api/meta" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -24,19 +24,34 @@ func MetaHandler() handlers.Handler { return &metaHandler{} } -func (h *metaHandler) OnCreate(_ client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (h *metaHandler) OnCreate( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(_ context.Context, req admission.Request) *admission.Response { return h.handle(decoder, req) } } -func (h *metaHandler) OnUpdate(_ client.Client, decoder admission.Decoder, _ events.EventRecorder) handlers.Func { +func (h *metaHandler) OnUpdate( + _ client.Client, + _ client.Reader, + decoder admission.Decoder, + _ events.EventRecorder, +) handlers.Func { return func(_ context.Context, req admission.Request) *admission.Response { return h.handle(decoder, req) } } -func (h *metaHandler) OnDelete(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *metaHandler) OnDelete( + client.Client, + client.Reader, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } @@ -45,7 +60,7 @@ func (h *metaHandler) OnDelete(client.Client, admission.Decoder, events.EventRec func (h *metaHandler) handle(decoder admission.Decoder, req admission.Request) *admission.Response { tenant := &capsulev1beta2.Tenant{} if err := decoder.Decode(req, tenant); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } labels := tenant.GetLabels() diff --git a/internal/webhook/tenant/validation/containerregistry_regex.go b/internal/webhook/tenant/validation/containerregistry_regex.go index 73fd64e3..805fb6f7 100644 --- a/internal/webhook/tenant/validation/containerregistry_regex.go +++ b/internal/webhook/tenant/validation/containerregistry_regex.go @@ -13,6 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -24,6 +25,7 @@ func ContainerRegistryRegexHandler() handlers.TypedHandler[*capsulev1beta2.Tenan func (h *containerRegistryRegexHandler) OnCreate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, decoder admission.Decoder, _ events.EventRecorder, @@ -39,6 +41,7 @@ func (h *containerRegistryRegexHandler) OnCreate( func (h *containerRegistryRegexHandler) OnDelete( client.Client, + client.Reader, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder, @@ -50,6 +53,7 @@ func (h *containerRegistryRegexHandler) OnDelete( func (h *containerRegistryRegexHandler) OnUpdate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, old *capsulev1beta2.Tenant, decoder admission.Decoder, @@ -71,9 +75,7 @@ func (h *containerRegistryRegexHandler) validate( ) *admission.Response { if tnt.Spec.ContainerRegistries != nil && len(tnt.Spec.ContainerRegistries.Regex) > 0 { if _, err := regexp.Compile(tnt.Spec.ContainerRegistries.Regex); err != nil { - response := admission.Denied("unable to compile containerRegistries allowedRegex") - - return &response + return ad.Deny("unable to compile containerRegistries allowedRegex") } } diff --git a/internal/webhook/tenant/validation/forbidden_annotations_regex.go b/internal/webhook/tenant/validation/forbidden_annotations_regex.go index b839954c..a24c97fd 100644 --- a/internal/webhook/tenant/validation/forbidden_annotations_regex.go +++ b/internal/webhook/tenant/validation/forbidden_annotations_regex.go @@ -13,6 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -24,6 +25,7 @@ func ForbiddenAnnotationsRegexHandler() handlers.TypedHandler[*capsulev1beta2.Te func (h *forbiddenAnnotationsRegexHandler) OnCreate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, _ admission.Decoder, _ events.EventRecorder, @@ -39,6 +41,7 @@ func (h *forbiddenAnnotationsRegexHandler) OnCreate( func (h *forbiddenAnnotationsRegexHandler) OnDelete( client.Client, + client.Reader, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder, @@ -50,6 +53,7 @@ func (h *forbiddenAnnotationsRegexHandler) OnDelete( func (h *forbiddenAnnotationsRegexHandler) OnUpdate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, old *capsulev1beta2.Tenant, _ admission.Decoder, @@ -76,9 +80,7 @@ func (h *forbiddenAnnotationsRegexHandler) validate(tnt *capsulev1beta2.Tenant, for scope, annotation := range annotationsToCheck { if _, err := regexp.Compile(tnt.Spec.NamespaceOptions.ForbiddenLabels.Regex); err != nil { - response := admission.Denied(fmt.Sprintf("unable to compile %s regex for forbidden %s", annotation, scope)) - - return &response + return ad.Deny(fmt.Sprintf("unable to compile %s regex for forbidden %s", annotation, scope)) } } diff --git a/internal/webhook/tenant/validation/freezed_emitter.go b/internal/webhook/tenant/validation/freezed_emitter.go index ff74b365..69f4dd6b 100644 --- a/internal/webhook/tenant/validation/freezed_emitter.go +++ b/internal/webhook/tenant/validation/freezed_emitter.go @@ -22,13 +22,25 @@ func FreezedEmitter() handlers.TypedHandler[*capsulev1beta2.Tenant] { return &freezedEmitterHandler{} } -func (h *freezedEmitterHandler) OnCreate(client.Client, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *freezedEmitterHandler) OnCreate( + client.Client, + client.Reader, + *capsulev1beta2.Tenant, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } } -func (h *freezedEmitterHandler) OnDelete(client.Client, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *freezedEmitterHandler) OnDelete( + client.Client, + client.Reader, + *capsulev1beta2.Tenant, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } @@ -36,6 +48,7 @@ func (h *freezedEmitterHandler) OnDelete(client.Client, *capsulev1beta2.Tenant, func (h *freezedEmitterHandler) OnUpdate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, old *capsulev1beta2.Tenant, decoder admission.Decoder, @@ -44,9 +57,9 @@ func (h *freezedEmitterHandler) OnUpdate( return func(_ context.Context, req admission.Request) *admission.Response { switch { case !old.Spec.Cordoned && tnt.Spec.Cordoned: - recorder.Eventf(tnt, tnt, corev1.EventTypeNormal, evt.ReasonCordoning, evt.ActionCordoned, "Tenant has been cordoned", "") + recorder.Eventf(tnt, nil, corev1.EventTypeNormal, evt.ReasonCordoning, evt.ActionCordoned, "Tenant has been cordoned") case old.Spec.Cordoned && !tnt.Spec.Cordoned: - recorder.Eventf(tnt, tnt, corev1.EventTypeNormal, evt.ReasonCordoning, evt.ActionUncordoned, "Tenant has been uncordoned", "") + recorder.Eventf(tnt, nil, corev1.EventTypeNormal, evt.ReasonCordoning, evt.ActionUncordoned, "Tenant has been uncordoned") } return nil diff --git a/internal/webhook/tenant/validation/handler.go b/internal/webhook/tenant/validation/handler.go index 86588acb..76c4b14a 100644 --- a/internal/webhook/tenant/validation/handler.go +++ b/internal/webhook/tenant/validation/handler.go @@ -11,7 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -28,15 +28,20 @@ type handler struct { handlers []handlers.TypedHandler[*capsulev1beta2.Tenant] } -func (h *handler) OnCreate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *handler) OnCreate( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { tnt := &capsulev1beta2.Tenant{} if err := decoder.Decode(req, tnt); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } for _, hndl := range h.handlers { - if response := hndl.OnCreate(c, tnt, decoder, recorder)(ctx, req); response != nil { + if response := hndl.OnCreate(c, reader, tnt, decoder, recorder)(ctx, req); response != nil { return response } } @@ -45,15 +50,20 @@ func (h *handler) OnCreate(c client.Client, decoder admission.Decoder, recorder } } -func (h *handler) OnDelete(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *handler) OnDelete( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { tnt := &capsulev1beta2.Tenant{} if err := decoder.DecodeRaw(req.OldObject, tnt); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } for _, hndl := range h.handlers { - if response := hndl.OnDelete(c, tnt, decoder, recorder)(ctx, req); response != nil { + if response := hndl.OnDelete(c, reader, tnt, decoder, recorder)(ctx, req); response != nil { return response } } @@ -62,20 +72,25 @@ func (h *handler) OnDelete(c client.Client, decoder admission.Decoder, recorder } } -func (h *handler) OnUpdate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) handlers.Func { +func (h *handler) OnUpdate( + c client.Client, + reader client.Reader, + decoder admission.Decoder, + recorder events.EventRecorder, +) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { tnt := &capsulev1beta2.Tenant{} if err := decoder.Decode(req, tnt); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } old := &capsulev1beta2.Tenant{} if err := decoder.DecodeRaw(req.OldObject, old); err != nil { - return utils.ErroredResponse(err) + return ad.ErroredResponse(err) } for _, hndl := range h.handlers { - if response := hndl.OnUpdate(c, tnt, old, decoder, recorder)(ctx, req); response != nil { + if response := hndl.OnUpdate(c, reader, tnt, old, decoder, recorder)(ctx, req); response != nil { return response } } diff --git a/internal/webhook/tenant/validation/hostname_regex.go b/internal/webhook/tenant/validation/hostname_regex.go index 1f0c2eb8..d4fbe8ed 100644 --- a/internal/webhook/tenant/validation/hostname_regex.go +++ b/internal/webhook/tenant/validation/hostname_regex.go @@ -13,6 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -24,6 +25,7 @@ func HostnameRegexHandler() handlers.TypedHandler[*capsulev1beta2.Tenant] { func (h *hostnameRegexHandler) OnCreate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, decoder admission.Decoder, _ events.EventRecorder, @@ -37,7 +39,13 @@ func (h *hostnameRegexHandler) OnCreate( } } -func (h *hostnameRegexHandler) OnDelete(client.Client, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *hostnameRegexHandler) OnDelete( + client.Client, + client.Reader, + *capsulev1beta2.Tenant, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } @@ -45,6 +53,7 @@ func (h *hostnameRegexHandler) OnDelete(client.Client, *capsulev1beta2.Tenant, a func (h *hostnameRegexHandler) OnUpdate( _ client.Client, + _ client.Reader, old *capsulev1beta2.Tenant, tnt *capsulev1beta2.Tenant, decoder admission.Decoder, @@ -66,9 +75,7 @@ func (h *hostnameRegexHandler) validate( ) *admission.Response { if tnt.Spec.IngressOptions.AllowedHostnames != nil && len(tnt.Spec.IngressOptions.AllowedHostnames.Regex) > 0 { if _, err := regexp.Compile(tnt.Spec.IngressOptions.AllowedHostnames.Regex); err != nil { - response := admission.Denied("unable to compile allowedHostnames allowedRegex") - - return &response + return ad.Deny("unable to compile allowedHostnames allowedRegex") } } diff --git a/internal/webhook/tenant/validation/ingressclass_regex.go b/internal/webhook/tenant/validation/ingressclass_regex.go index 9539e901..f4e45b39 100644 --- a/internal/webhook/tenant/validation/ingressclass_regex.go +++ b/internal/webhook/tenant/validation/ingressclass_regex.go @@ -13,6 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -24,6 +25,7 @@ func IngressClassRegexHandler() handlers.TypedHandler[*capsulev1beta2.Tenant] { func (h *ingressClassRegexHandler) OnCreate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, _ admission.Decoder, _ events.EventRecorder, @@ -39,6 +41,7 @@ func (h *ingressClassRegexHandler) OnCreate( func (h *ingressClassRegexHandler) OnDelete( client.Client, + client.Reader, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder, @@ -50,6 +53,7 @@ func (h *ingressClassRegexHandler) OnDelete( func (h *ingressClassRegexHandler) OnUpdate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, old *capsulev1beta2.Tenant, decoder admission.Decoder, @@ -68,9 +72,7 @@ func (h *ingressClassRegexHandler) validate(tnt *capsulev1beta2.Tenant, req admi //nolint:staticcheck if tnt.Spec.IngressOptions.AllowedClasses != nil && len(tnt.Spec.IngressOptions.AllowedClasses.Regex) > 0 { if _, err := regexp.Compile(tnt.Spec.IngressOptions.AllowedClasses.Regex); err != nil { - response := admission.Denied("unable to compile ingressClasses allowedRegex") - - return &response + return ad.Deny("unable to compile ingressClasses allowedRegex") } } diff --git a/internal/webhook/tenant/validation/name.go b/internal/webhook/tenant/validation/name.go index 79a3ed47..30c0ee37 100644 --- a/internal/webhook/tenant/validation/name.go +++ b/internal/webhook/tenant/validation/name.go @@ -12,6 +12,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -23,6 +24,7 @@ func NameHandler() handlers.TypedHandler[*capsulev1beta2.Tenant] { func (h *nameHandler) OnCreate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, decoder admission.Decoder, _ events.EventRecorder, @@ -30,9 +32,7 @@ func (h *nameHandler) OnCreate( return func(_ context.Context, req admission.Request) *admission.Response { matched, _ := regexp.MatchString(`[a-z0-9]([-a-z0-9]*[a-z0-9])?`, tnt.GetName()) if !matched { - response := admission.Denied("tenant name has forbidden characters") - - return &response + return ad.Deny("tenant name has forbidden characters") } return nil @@ -41,6 +41,7 @@ func (h *nameHandler) OnCreate( func (h *nameHandler) OnDelete( client.Client, + client.Reader, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder, @@ -52,6 +53,7 @@ func (h *nameHandler) OnDelete( func (h *nameHandler) OnUpdate( client.Client, + client.Reader, *capsulev1beta2.Tenant, *capsulev1beta2.Tenant, admission.Decoder, diff --git a/internal/webhook/tenant/validation/namespace_metadata.go b/internal/webhook/tenant/validation/namespace_metadata.go new file mode 100644 index 00000000..511d5dba --- /dev/null +++ b/internal/webhook/tenant/validation/namespace_metadata.go @@ -0,0 +1,171 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package validation + +import ( + "context" + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" + "github.com/projectcapsule/capsule/pkg/template" +) + +type namespaceMetadataHandler struct{} + +func NamespaceMetadataHandler() handlers.TypedHandler[*capsulev1beta2.Tenant] { + return &namespaceMetadataHandler{} +} + +func (h *namespaceMetadataHandler) OnCreate( + _ client.Client, + _ client.Reader, + tnt *capsulev1beta2.Tenant, + _ admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return func(_ context.Context, _ admission.Request) *admission.Response { + return validateTenantNamespaceMetadata(tnt) + } +} + +func (h *namespaceMetadataHandler) OnDelete( + client.Client, + client.Reader, + *capsulev1beta2.Tenant, + admission.Decoder, + events.EventRecorder, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} + +func (h *namespaceMetadataHandler) OnUpdate( + _ client.Client, + _ client.Reader, + newTnt *capsulev1beta2.Tenant, + _ *capsulev1beta2.Tenant, + _ admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { + return validateTenantNamespaceMetadata(newTnt) + } +} + +func validateTenantNamespaceMetadata(tnt *capsulev1beta2.Tenant) *admission.Response { + if tnt == nil { + return nil + } + + if tnt.Spec.NamespaceOptions == nil { + return nil + } + + errs := make([]string, 0, 1+len(tnt.Spec.NamespaceOptions.AdditionalMetadataList)) + + errs = append( + errs, + validateAdditionalMetadata( + "spec.namespaceOptions.additionalMetadata", + //nolint:staticcheck + tnt.Spec.NamespaceOptions.AdditionalMetadata, + )..., + ) + + for i, item := range tnt.Spec.NamespaceOptions.AdditionalMetadataList { + errs = append( + errs, + validateAdditionalMetadata( + fmt.Sprintf("spec.namespaceOptions.additionalMetadataList[%d].additionalMetadata", i), + &api.AdditionalMetadataSpec{ + Labels: item.Labels, + Annotations: item.Annotations, + }, + )..., + ) + } + + if len(errs) > 0 { + return ad.Deny(strings.Join(errs, "; ")) + } + + return nil +} + +func validateAdditionalMetadata( + fieldPath string, + metadata *api.AdditionalMetadataSpec, +) []string { + if metadata == nil { + return nil + } + + errs := make([]string, 0, len(metadata.Labels)*2+len(metadata.Annotations)*2) + + errs = append(errs, validateLabelMap(fieldPath+".labels", metadata.Labels)...) + errs = append(errs, validateAnnotationMap(fieldPath+".annotations", metadata.Annotations)...) + + return errs +} + +func validateLabelMap(fieldPath string, labels map[string]string) []string { + errs := make([]string, 0, len(labels)*2) + + for key, value := range labels { + errs = append( + errs, + template.ValidateKubernetesStringOrAllowedTemplates( + fmt.Sprintf("%s[%q].key", fieldPath, key), + key, + validation.IsQualifiedName, + )..., + ) + + errs = append( + errs, + template.ValidateKubernetesStringOrAllowedTemplates( + fmt.Sprintf("%s[%q].value", fieldPath, key), + value, + validation.IsValidLabelValue, + )..., + ) + } + + return errs +} + +func validateAnnotationMap(fieldPath string, annotations map[string]string) []string { + errs := make([]string, 0, len(annotations)*2) + + for key, value := range annotations { + errs = append( + errs, + template.ValidateKubernetesStringOrAllowedTemplates( + fmt.Sprintf("%s[%q].key", fieldPath, key), + key, + validation.IsQualifiedName, + )..., + ) + + errs = append( + errs, + template.ValidateAllowedTemplatesOnly( + fmt.Sprintf("%s[%q].value", fieldPath, key), + value, + )..., + ) + } + + return errs +} diff --git a/internal/webhook/tenant/validation/protected.go b/internal/webhook/tenant/validation/protected.go index 18c877c0..db2682f8 100644 --- a/internal/webhook/tenant/validation/protected.go +++ b/internal/webhook/tenant/validation/protected.go @@ -11,6 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -22,6 +23,7 @@ func ProtectedHandler() handlers.TypedHandler[*capsulev1beta2.Tenant] { func (h *protectedHandler) OnCreate( client.Client, + client.Reader, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder, @@ -32,23 +34,29 @@ func (h *protectedHandler) OnCreate( } func (h *protectedHandler) OnDelete( - c client.Client, + _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, _ admission.Decoder, _ events.EventRecorder, ) handlers.Func { return func(ctx context.Context, req admission.Request) *admission.Response { if tnt.Spec.PreventDeletion { - response := admission.Denied("tenant is protected and cannot be deleted") - - return &response + return ad.Deny("tenant is protected and cannot be deleted") } return nil } } -func (h *protectedHandler) OnUpdate(client.Client, *capsulev1beta2.Tenant, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *protectedHandler) OnUpdate( + client.Client, + client.Reader, + *capsulev1beta2.Tenant, + *capsulev1beta2.Tenant, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } diff --git a/internal/webhook/tenant/validation/racing_namespaces.go b/internal/webhook/tenant/validation/racing_namespaces.go new file mode 100644 index 00000000..b0bb063b --- /dev/null +++ b/internal/webhook/tenant/validation/racing_namespaces.go @@ -0,0 +1,85 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package validation + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" + "github.com/projectcapsule/capsule/pkg/runtime/handlers" + namespaceindex "github.com/projectcapsule/capsule/pkg/runtime/indexers/namespace" +) + +type remainingNamespaceHandler struct{} + +func RemainingNamespaceHandler() handlers.TypedHandler[*capsulev1beta2.Tenant] { + return &remainingNamespaceHandler{} +} + +func (h *remainingNamespaceHandler) OnCreate( + client.Client, + client.Reader, + *capsulev1beta2.Tenant, + admission.Decoder, + events.EventRecorder, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} + +// This happens when a tenant has not yet reconciled it's namespaces but is deleted +// and in the meantime a new namespace was created referencing the same tenant. +func (h *remainingNamespaceHandler) OnDelete( + c client.Client, + _ client.Reader, + tnt *capsulev1beta2.Tenant, + _ admission.Decoder, + _ events.EventRecorder, +) handlers.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + list := &corev1.NamespaceList{} + + err := c.List(ctx, list, client.MatchingFields{namespaceindex.OwnerReferenceIndex: tnt.GetName()}) + if err != nil { + return ad.ErroredResponse(err) + } + + if len(list.Items) == 0 { + return nil + } + + for _, ns := range list.Items { + instance := tnt.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ + Name: ns.GetName(), + UID: ns.GetUID(), + }) + + if instance == nil { + return ad.Deny("tenant has remaining namespace referencing it (" + ns.GetName() + ")") + } + } + + return nil + } +} + +func (h *remainingNamespaceHandler) OnUpdate( + client.Client, + client.Reader, + *capsulev1beta2.Tenant, + *capsulev1beta2.Tenant, + admission.Decoder, + events.EventRecorder, +) handlers.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} diff --git a/internal/webhook/tenant/validation/required_metdata_regex.go b/internal/webhook/tenant/validation/required_metdata_regex.go index 7b172421..84b6638b 100644 --- a/internal/webhook/tenant/validation/required_metdata_regex.go +++ b/internal/webhook/tenant/validation/required_metdata_regex.go @@ -12,6 +12,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" "github.com/projectcapsule/capsule/pkg/utils" ) @@ -24,6 +25,7 @@ func RequiredMetadataHandler() handlers.TypedHandler[*capsulev1beta2.Tenant] { func (h *requiredMetadataHandler) OnCreate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, _ admission.Decoder, _ events.EventRecorder, @@ -39,6 +41,7 @@ func (h *requiredMetadataHandler) OnCreate( func (h *requiredMetadataHandler) OnDelete( client.Client, + client.Reader, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder, @@ -50,6 +53,7 @@ func (h *requiredMetadataHandler) OnDelete( func (h *requiredMetadataHandler) OnUpdate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, old *capsulev1beta2.Tenant, decoder admission.Decoder, @@ -76,17 +80,13 @@ func (h *requiredMetadataHandler) validate(tnt *capsulev1beta2.Tenant, req admis for _, exp := range tnt.Spec.NamespaceOptions.RequiredMetadata.Labels { if _, err := regexp.Compile(exp); err != nil { - response := admission.Denied("unable to compile required label") - - return &response + return ad.Deny("unable to compile required label") } } for _, exp := range tnt.Spec.NamespaceOptions.RequiredMetadata.Annotations { if _, err := regexp.Compile(exp); err != nil { - response := admission.Denied("unable to compile required annotation") - - return &response + return ad.Deny("unable to compile required annotation") } } diff --git a/internal/webhook/tenant/validation/rolebindings_regex.go b/internal/webhook/tenant/validation/rolebindings_regex.go index 111d9590..e28bc4de 100644 --- a/internal/webhook/tenant/validation/rolebindings_regex.go +++ b/internal/webhook/tenant/validation/rolebindings_regex.go @@ -15,6 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -26,6 +27,7 @@ func RoleBindingRegexHandler() handlers.TypedHandler[*capsulev1beta2.Tenant] { func (h *rbRegexHandler) OnCreate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, decoder admission.Decoder, _ events.EventRecorder, @@ -37,6 +39,7 @@ func (h *rbRegexHandler) OnCreate( func (h *rbRegexHandler) OnDelete( client.Client, + client.Reader, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder, @@ -48,6 +51,7 @@ func (h *rbRegexHandler) OnDelete( func (h *rbRegexHandler) OnUpdate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, old *capsulev1beta2.Tenant, decoder admission.Decoder, @@ -65,9 +69,7 @@ func (h *rbRegexHandler) validate(tnt *capsulev1beta2.Tenant, decoder admission. if subject.Kind == rbacv1.ServiceAccountKind { err := validation.IsDNS1123Subdomain(subject.Name) if len(err) > 0 { - response := admission.Denied(fmt.Sprintf("Subject Name '%v' for binding '%v' is invalid. %v", subject.Name, binding.ClusterRoleName, strings.Join(err, ", "))) - - return &response + return ad.Deny(fmt.Sprintf("Subject Name '%v' for binding '%v' is invalid. %v", subject.Name, binding.ClusterRoleName, strings.Join(err, ", "))) } } } diff --git a/internal/webhook/tenant/validation/rule_validator.go b/internal/webhook/tenant/validation/rule_validator.go index 1f39bd9f..ea449cb2 100644 --- a/internal/webhook/tenant/validation/rule_validator.go +++ b/internal/webhook/tenant/validation/rule_validator.go @@ -14,6 +14,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -25,6 +26,7 @@ func RuleHandler() handlers.TypedHandler[*capsulev1beta2.Tenant] { func (h *RuleValidationHandler) OnCreate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, decoder admission.Decoder, _ events.EventRecorder, @@ -38,7 +40,13 @@ func (h *RuleValidationHandler) OnCreate( } } -func (h *RuleValidationHandler) OnDelete(client.Client, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *RuleValidationHandler) OnDelete( + client.Client, + client.Reader, + *capsulev1beta2.Tenant, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } @@ -46,6 +54,7 @@ func (h *RuleValidationHandler) OnDelete(client.Client, *capsulev1beta2.Tenant, func (h *RuleValidationHandler) OnUpdate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, old *capsulev1beta2.Tenant, decoder admission.Decoder, @@ -74,22 +83,18 @@ func ValidateRule(tnt *capsulev1beta2.Tenant, req admission.Request) *admission. // Validate NamespaceSelector (if provided) if rule.NamespaceSelector != nil { if _, err := metav1.LabelSelectorAsSelector(rule.NamespaceSelector); err != nil { - resp := admission.Denied( + return ad.Deny( fmt.Sprintf("rules[%d].namespaceSelector is invalid: %v", i, err), ) - - return &resp } } // Validate Registries for _, r := range rule.Enforce.Registries { if _, err := regexp.Compile(r.Registry); err != nil { - resp := admission.Denied( + return ad.Deny( fmt.Sprintf("unable to compile regex %q: %v", r.Registry, err), ) - - return &resp } } } diff --git a/internal/webhook/tenant/validation/serviceaccount_format.go b/internal/webhook/tenant/validation/serviceaccount_format.go index 9ca72f3b..d92dfdf7 100644 --- a/internal/webhook/tenant/validation/serviceaccount_format.go +++ b/internal/webhook/tenant/validation/serviceaccount_format.go @@ -13,6 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -26,6 +27,7 @@ func ServiceAccountNameHandler() handlers.TypedHandler[*capsulev1beta2.Tenant] { func (h *saNameHandler) OnCreate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, _ admission.Decoder, _ events.EventRecorder, @@ -35,7 +37,13 @@ func (h *saNameHandler) OnCreate( } } -func (h *saNameHandler) OnDelete(client.Client, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *saNameHandler) OnDelete( + client.Client, + client.Reader, + *capsulev1beta2.Tenant, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } @@ -43,6 +51,7 @@ func (h *saNameHandler) OnDelete(client.Client, *capsulev1beta2.Tenant, admissio func (h *saNameHandler) OnUpdate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, old *capsulev1beta2.Tenant, _ admission.Decoder, @@ -60,9 +69,7 @@ func (h *saNameHandler) validateServiceAccountName(tnt *capsulev1beta2.Tenant, r } if !compiler.MatchString(owner.Name) { - response := admission.Denied(fmt.Sprintf("owner name %s is not a valid Service Account name ", owner.Name)) - - return &response + return ad.Deny(fmt.Sprintf("owner name %s is not a valid Service Account name ", owner.Name)) } } diff --git a/internal/webhook/tenant/validation/storageclass_regex.go b/internal/webhook/tenant/validation/storageclass_regex.go index 0fe3fb5c..5af457ac 100644 --- a/internal/webhook/tenant/validation/storageclass_regex.go +++ b/internal/webhook/tenant/validation/storageclass_regex.go @@ -13,6 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -24,6 +25,7 @@ func StorageClassRegexHandler() handlers.TypedHandler[*capsulev1beta2.Tenant] { func (h *storageClassRegexHandler) OnCreate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, decoder admission.Decoder, _ events.EventRecorder, @@ -39,6 +41,7 @@ func (h *storageClassRegexHandler) OnCreate( func (h *storageClassRegexHandler) OnDelete( client.Client, + client.Reader, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder, @@ -50,6 +53,7 @@ func (h *storageClassRegexHandler) OnDelete( func (h *storageClassRegexHandler) OnUpdate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, old *capsulev1beta2.Tenant, _ admission.Decoder, @@ -68,9 +72,7 @@ func (h *storageClassRegexHandler) validate(tnt *capsulev1beta2.Tenant, req admi //nolint:staticcheck if tnt.Spec.StorageClasses != nil && len(tnt.Spec.StorageClasses.Regex) > 0 { if _, err := regexp.Compile(tnt.Spec.StorageClasses.Regex); err != nil { - response := admission.Denied("unable to compile storageClasses allowedRegex") - - return &response + return ad.Deny("unable to compile storageClasses allowedRegex") } } diff --git a/internal/webhook/tenant/validation/warnings.go b/internal/webhook/tenant/validation/warnings.go index ff10fd68..7ac9da42 100644 --- a/internal/webhook/tenant/validation/warnings.go +++ b/internal/webhook/tenant/validation/warnings.go @@ -5,6 +5,7 @@ package validation import ( "context" + "strings" admissionv1 "k8s.io/api/admission/v1" "k8s.io/client-go/tools/events" @@ -12,6 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/runtime/handlers" ) @@ -28,6 +30,7 @@ func WarningHandler(cfg configuration.Configuration) handlers.TypedHandler[*caps func (h *warningHandler) OnCreate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, _ admission.Decoder, _ events.EventRecorder, @@ -37,7 +40,13 @@ func (h *warningHandler) OnCreate( } } -func (h *warningHandler) OnDelete(client.Client, *capsulev1beta2.Tenant, admission.Decoder, events.EventRecorder) handlers.Func { +func (h *warningHandler) OnDelete( + client.Client, + client.Reader, + *capsulev1beta2.Tenant, + admission.Decoder, + events.EventRecorder, +) handlers.Func { return func(context.Context, admission.Request) *admission.Response { return nil } @@ -45,6 +54,7 @@ func (h *warningHandler) OnDelete(client.Client, *capsulev1beta2.Tenant, admissi func (h *warningHandler) OnUpdate( _ client.Client, + _ client.Reader, tnt *capsulev1beta2.Tenant, old *capsulev1beta2.Tenant, _ admission.Decoder, @@ -121,5 +131,15 @@ func (h *warningHandler) handle(tnt *capsulev1beta2.Tenant, req admission.Reques ) } + if tnt.GetAnnotations() != nil { + for k := range tnt.GetAnnotations() { + if strings.HasPrefix(k, meta.ResourceQuotaAnnotationPrefix) { + response.Warnings = append(response.Warnings, + "custom quotas via tenant annotations are deprecated and will be removed in a future release. Please migrate to GlobalCustomQuotas. See: https://projectcapsule.dev/docs/resource-management/customquotas/#globalcustomquota.", + ) + } + } + } + return response } diff --git a/internal/webhook/tenantresource/objects.go b/internal/webhook/tenantresource/objects.go deleted file mode 100644 index 55f2d15b..00000000 --- a/internal/webhook/tenantresource/objects.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package tenant - -import ( - "context" - "fmt" - "strings" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/client-go/tools/events" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/internal/webhook/utils" - evt "github.com/projectcapsule/capsule/pkg/runtime/events" - "github.com/projectcapsule/capsule/pkg/runtime/handlers" - "github.com/projectcapsule/capsule/pkg/runtime/indexers/tenantresource" - "github.com/projectcapsule/capsule/pkg/tenant" -) - -type cordoningHandler struct{} - -func WriteOpsHandler() handlers.Handler { - return &cordoningHandler{} -} - -func (h *cordoningHandler) OnCreate(client.Client, admission.Decoder, events.EventRecorder) handlers.Func { - return func(context.Context, admission.Request) *admission.Response { - return nil - } -} - -func (h *cordoningHandler) OnDelete(client client.Client, _ admission.Decoder, recorder events.EventRecorder) handlers.Func { - return func(ctx context.Context, req admission.Request) *admission.Response { - return h.handler(ctx, client, req, recorder) - } -} - -func (h *cordoningHandler) OnUpdate(client client.Client, _ admission.Decoder, recorder events.EventRecorder) handlers.Func { - return func(ctx context.Context, req admission.Request) *admission.Response { - return h.handler(ctx, client, req, recorder) - } -} - -func (h *cordoningHandler) handler(ctx context.Context, clt client.Client, req admission.Request, recorder events.EventRecorder) *admission.Response { - tnt, err := tenant.TenantByStatusNamespace(ctx, clt, req.Namespace) - if err != nil { - return utils.ErroredResponse(err) - } - - if tnt == nil { - return nil - } - - // Checking if the object is managed by a TenantResource, local or global - ors := capsulev1beta2.ObjectReferenceStatus{ - ObjectReferenceAbstract: capsulev1beta2.ObjectReferenceAbstract{ - Kind: req.Kind.Kind, - Namespace: req.Namespace, - APIVersion: req.Kind.Version, - }, - Name: req.Name, - } - - global, local := &capsulev1beta2.GlobalTenantResourceList{}, &capsulev1beta2.TenantResourceList{} - - if err := clt.List(ctx, global, client.MatchingFieldsSelector{Selector: fields.OneTermEqualSelector(tenantresource.IndexerFieldName, ors.String())}); err != nil { - return utils.ErroredResponse(err) - } - - if err := clt.List(ctx, local, client.MatchingFieldsSelector{Selector: fields.OneTermEqualSelector(tenantresource.IndexerFieldName, ors.String())}); err != nil { - return utils.ErroredResponse(err) - } - - if len(local.Items) > 0 || len(global.Items) > 0 { - recorder.Eventf(tnt, nil, corev1.EventTypeWarning, evt.ReasonTenantResourceWriteOp, evt.ActionValidationDenied, "%s %s/%s cannot be %sd, resource is managed by the Tenant", req.Kind.String(), req.Namespace, req.Name, strings.ToLower(string(req.Operation))) - - response := admission.Denied(fmt.Sprintf("resource %s is managed at the Tenant level", req.Name)) - - return &response - } - - return nil -} diff --git a/internal/webhook/utils/resources.go b/internal/webhook/utils/resources.go index f22d02e1..853f6b2c 100644 --- a/internal/webhook/utils/resources.go +++ b/internal/webhook/utils/resources.go @@ -21,7 +21,7 @@ import ( const TRUE string = "true" // Get PriorityClass by name (Does not return error if not found). -func GetPriorityClassByName(ctx context.Context, c client.Client, name string) (*schedulev1.PriorityClass, error) { +func GetPriorityClassByName(ctx context.Context, c client.Reader, name string) (*schedulev1.PriorityClass, error) { class := &schedulev1.PriorityClass{} if err := c.Get(ctx, types.NamespacedName{Name: name}, class); err != nil { return nil, err @@ -31,7 +31,7 @@ func GetPriorityClassByName(ctx context.Context, c client.Client, name string) ( } // Get StorageClass by name (Does not return error if not found). -func GetStorageClassByName(ctx context.Context, c client.Client, name string) (*storagev1.StorageClass, error) { +func GetStorageClassByName(ctx context.Context, c client.Reader, name string) (*storagev1.StorageClass, error) { class := &storagev1.StorageClass{} if err := c.Get(ctx, types.NamespacedName{Name: name}, class); err != nil { return nil, err @@ -41,7 +41,7 @@ func GetStorageClassByName(ctx context.Context, c client.Client, name string) (* } // Get IngressClass by name (Does not return error if not found). -func GetIngressClassByName(ctx context.Context, version *version.Version, c client.Client, ingressClassName *string) (client.Object, error) { +func GetIngressClassByName(ctx context.Context, version *version.Version, c client.Reader, ingressClassName *string) (client.Object, error) { if ingressClassName == nil { return nil, nil } @@ -67,7 +67,7 @@ func GetIngressClassByName(ctx context.Context, version *version.Version, c clie } // Get GatewayClassClass by name (Does not return error if not found). -func GetGatewayClassClassByObjectName(ctx context.Context, c client.Client, gatewayClassName gatewayv1.ObjectName) (*gatewayv1.GatewayClass, error) { +func GetGatewayClassClassByObjectName(ctx context.Context, c client.Reader, gatewayClassName gatewayv1.ObjectName) (*gatewayv1.GatewayClass, error) { objName := reflect.ValueOf(gatewayClassName).String() gatewayClass := &gatewayv1.GatewayClass{} @@ -79,7 +79,7 @@ func GetGatewayClassClassByObjectName(ctx context.Context, c client.Client, gate } // Get DeviceClass by name (Does not return error if not found). -func GetDeviceClassByName(ctx context.Context, c client.Client, name string) (*resources.DeviceClass, error) { +func GetDeviceClassByName(ctx context.Context, c client.Reader, name string) (*resources.DeviceClass, error) { class := &resources.DeviceClass{} if err := c.Get(ctx, types.NamespacedName{Name: name}, class); err != nil { return nil, err diff --git a/internal/webhook/utils/tenant_get.go b/internal/webhook/utils/tenant_get.go index e2e875e3..3169b59e 100644 --- a/internal/webhook/utils/tenant_get.go +++ b/internal/webhook/utils/tenant_get.go @@ -15,20 +15,22 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + ad "github.com/projectcapsule/capsule/pkg/runtime/admission" "github.com/projectcapsule/capsule/pkg/runtime/configuration" "github.com/projectcapsule/capsule/pkg/tenant" + "github.com/projectcapsule/capsule/pkg/users" ) -// getNamespaceTenant returns namespace owner tenant. func GetNamespaceTenant( ctx context.Context, - client client.Client, + reader client.Reader, + cache client.Client, ns *corev1.Namespace, - req admission.Request, + user users.AdmissionUser, cfg configuration.Configuration, recorder events.EventRecorder, ) (*capsulev1beta2.Tenant, *admission.Response) { - tnt, err := tenant.GetTenantByLabelsAndUser(ctx, client, cfg, ns, req.UserInfo) + tnt, err := tenant.GetTenantByLabelsAndUser(ctx, reader, cfg, ns, user) if err != nil { response := admission.Errored(http.StatusBadRequest, err) @@ -36,10 +38,17 @@ func GetNamespaceTenant( } if tnt != nil { + if !validateNamespacePrefix(cfg, ns, tnt) { + return nil, ad.Deny(fmt.Sprintf( + "The Namespace name must start with '%s-' when ForceTenantPrefix is enabled in the Tenant.", + tnt.GetName(), + )) + } + return tnt, nil } - tnts, err := tenant.GetTenantByUserInfo(ctx, client, cfg, ns, req.UserInfo.Username, req.UserInfo.Groups) + tnts, err := tenant.GetTenantByUserInfo(ctx, cache, cfg, ns, user) if err != nil { response := admission.Errored(http.StatusBadRequest, err) @@ -47,44 +56,83 @@ func GetNamespaceTenant( } if len(tnts) == 0 { - response := admission.Denied("You do not have any Tenant assigned: please, reach out to the system administrators") - - return nil, &response + return nil, ad.Deny("You do not have any Tenant assigned: please, reach out to the system administrators") } if len(tnts) == 1 { - // Check if namespace needs Tenant name prefix - if !validateNamespacePrefix(ns, &tnts[0]) { - response := admission.Denied(fmt.Sprintf("The Namespace name must start with '%s-' when ForceTenantPrefix is enabled in the Tenant.", tnts[0].GetName())) - - return nil, &response + if !validateNamespacePrefix(cfg, ns, &tnts[0]) { + return nil, ad.Deny(fmt.Sprintf( + "The Namespace name must start with '%s-' when ForceTenantPrefix is enabled in the Tenant.", + tnts[0].GetName(), + )) } return &tnts[0], nil } - if cfg.ForceTenantPrefix() { - for _, t := range tnts { - if strings.HasPrefix(ns.GetName(), fmt.Sprintf("%s-", t.GetName())) { - return &t, nil - } + tnt, ambiguous := resolveTenantByClosestNamespacePrefix(ns.GetName(), tnts) + if ambiguous { + return nil, ad.Deny("The Namespace prefix matches more than one available Tenant") + } + + if tnt != nil { + if !validateNamespacePrefix(cfg, ns, tnt) { + return nil, ad.Deny(fmt.Sprintf( + "The Namespace name must start with '%s-' when ForceTenantPrefix is enabled in the Tenant.", + tnt.GetName(), + )) } - response := admission.Denied("The Namespace prefix used doesn't match any available Tenant") + return tnt, nil + } - return nil, &response + if cfg.ForceTenantPrefix() { + return nil, ad.Deny("The Namespace prefix used doesn't match any available Tenant") } return nil, nil } -func validateNamespacePrefix(ns *corev1.Namespace, tenant *capsulev1beta2.Tenant) bool { - // Check if ForceTenantPrefix is true - if tenant.Spec.ForceTenantPrefix != nil && *tenant.Spec.ForceTenantPrefix { - if !strings.HasPrefix(ns.GetName(), fmt.Sprintf("%s-", tenant.GetName())) { - return false +func resolveTenantByClosestNamespacePrefix( + namespaceName string, + tnts []capsulev1beta2.Tenant, +) (*capsulev1beta2.Tenant, bool) { + var matched *capsulev1beta2.Tenant + + matchedPrefixLen := -1 + + ambiguous := false + + for i := range tnts { + prefix := fmt.Sprintf("%s-", tnts[i].GetName()) + if !strings.HasPrefix(namespaceName, prefix) { + continue + } + + switch { + case len(prefix) > matchedPrefixLen: + matched = &tnts[i] + matchedPrefixLen = len(prefix) + ambiguous = false + + case len(prefix) == matchedPrefixLen: + ambiguous = true } } - return true + return matched, ambiguous +} + +func validateNamespacePrefix(cfg configuration.Configuration, ns *corev1.Namespace, tenant *capsulev1beta2.Tenant) bool { + enforce := cfg.ForceTenantPrefix() + + if tenant.Spec.ForceTenantPrefix != nil { + enforce = *tenant.Spec.ForceTenantPrefix + } + + if !enforce { + return true + } + + return strings.HasPrefix(ns.GetName(), tenant.GetName()+"-") } diff --git a/pkg/api/allowed_list_test.go b/pkg/api/allowed_list_test.go index 6a2dda6e..af8b7784 100644 --- a/pkg/api/allowed_list_test.go +++ b/pkg/api/allowed_list_test.go @@ -7,8 +7,9 @@ package api_test import ( "testing" - "github.com/projectcapsule/capsule/pkg/api" "github.com/stretchr/testify/assert" + + "github.com/projectcapsule/capsule/pkg/api" ) func TestAllowedListSpec_ExactMatch(t *testing.T) { diff --git a/internal/webhook/utils/error.go b/pkg/api/errors/allowed.go similarity index 84% rename from internal/webhook/utils/error.go rename to pkg/api/errors/allowed.go index 9992f7f9..428096b5 100644 --- a/internal/webhook/utils/error.go +++ b/pkg/api/errors/allowed.go @@ -1,24 +1,15 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package utils +package errors import ( "fmt" - "net/http" "strings" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "github.com/projectcapsule/capsule/pkg/api" ) -func ErroredResponse(err error) *admission.Response { - response := admission.Errored(http.StatusInternalServerError, err) - - return &response -} - func DefaultAllowedValuesErrorMessage(allowed api.DefaultAllowedListSpec, err string) string { return AllowedValuesErrorMessage(allowed.SelectorAllowedListSpec, err) } diff --git a/pkg/api/errors/devices.go b/pkg/api/errors/devices.go index 56a4b8fc..6eb4b3d2 100644 --- a/pkg/api/errors/devices.go +++ b/pkg/api/errors/devices.go @@ -6,7 +6,6 @@ package errors import ( "fmt" - "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/api" ) @@ -18,7 +17,7 @@ type DeviceClassForbiddenError struct { func (i DeviceClassForbiddenError) Error() string { err := fmt.Sprintf("Device Class %s is forbidden for the current Tenant: ", i.deviceClassName) - return utils.AllowedValuesErrorMessage(i.spec, err) + return AllowedValuesErrorMessage(i.spec, err) } func NewDeviceClassForbidden(class string, spec api.SelectorAllowedListSpec) error { @@ -39,5 +38,5 @@ func NewDeviceClassUndefined(spec api.SelectorAllowedListSpec) error { } func (i DeviceClassUndefinedError) Error() string { - return utils.AllowedValuesErrorMessage(i.spec, "Selected DeviceClass is forbidden for the current Tenant or does not exist. Specify a device Class which is allowed by ") + return AllowedValuesErrorMessage(i.spec, "Selected DeviceClass is forbidden for the current Tenant or does not exist. Specify a device Class which is allowed by ") } diff --git a/pkg/api/errors/gateway.go b/pkg/api/errors/gateway.go index df292283..6c761026 100644 --- a/pkg/api/errors/gateway.go +++ b/pkg/api/errors/gateway.go @@ -9,7 +9,6 @@ import ( gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" - "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/api" ) @@ -60,7 +59,7 @@ func NewGatewayClassForbidden(class string, spec api.DefaultAllowedListSpec) err func (i GatewayClassForbiddenError) Error() string { err := fmt.Sprintf("Gateway Class %s is forbidden for the current Tenant: ", i.gatewayClassName) - return utils.DefaultAllowedValuesErrorMessage(i.spec, err) + return DefaultAllowedValuesErrorMessage(i.spec, err) } type GatewayClassUndefinedError struct { @@ -74,5 +73,5 @@ func NewGatewayClassUndefined(spec api.DefaultAllowedListSpec) error { } func (i GatewayClassUndefinedError) Error() string { - return utils.DefaultAllowedValuesErrorMessage(i.spec, "No gateway Class is forbidden for the current Tenant. Specify a gateway Class which is allowed within the Tenant: ") + return DefaultAllowedValuesErrorMessage(i.spec, "No gateway Class is forbidden for the current Tenant. Specify a gateway Class which is allowed within the Tenant: ") } diff --git a/pkg/api/errors/ingress.go b/pkg/api/errors/ingress.go index 352fa8ec..272cb221 100644 --- a/pkg/api/errors/ingress.go +++ b/pkg/api/errors/ingress.go @@ -7,7 +7,6 @@ import ( "fmt" "strings" - "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/api" ) @@ -42,7 +41,7 @@ func NewIngressClassForbidden(class string, spec api.DefaultAllowedListSpec) err func (i IngressClassForbiddenError) Error() string { err := fmt.Sprintf("Ingress Class %s is forbidden for the current Tenant: ", i.ingressClassName) - return utils.DefaultAllowedValuesErrorMessage(i.spec, err) + return DefaultAllowedValuesErrorMessage(i.spec, err) } type IngressHostnameNotValidError struct { @@ -97,7 +96,7 @@ func NewIngressClassUndefined(spec api.DefaultAllowedListSpec) error { } func (i IngressClassUndefinedError) Error() string { - return utils.DefaultAllowedValuesErrorMessage(i.spec, "No Ingress Class is forbidden for the current Tenant. Specify a Ingress Class which is allowed within the Tenant: ") + return DefaultAllowedValuesErrorMessage(i.spec, "No Ingress Class is forbidden for the current Tenant. Specify a Ingress Class which is allowed within the Tenant: ") } type IngressClassNotValidError struct { @@ -115,7 +114,7 @@ func NewIngressClassNotValid(class string, spec api.DefaultAllowedListSpec) erro func (i IngressClassNotValidError) Error() string { err := fmt.Sprintf("Ingress Class %s is forbidden for the current Tenant: ", i.ingressClassName) - return utils.DefaultAllowedValuesErrorMessage(i.spec, err) + return DefaultAllowedValuesErrorMessage(i.spec, err) } //nolint:predeclared,revive diff --git a/pkg/api/errors/pods.go b/pkg/api/errors/pods.go index 2f0a7c89..3c131d3f 100644 --- a/pkg/api/errors/pods.go +++ b/pkg/api/errors/pods.go @@ -7,7 +7,6 @@ import ( "fmt" "strings" - "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/api" ) @@ -115,7 +114,7 @@ func NewPodPriorityClassForbidden(priorityClassName string, spec api.DefaultAllo func (f PodPriorityClassForbiddenError) Error() (err string) { msg := fmt.Sprintf("Pod Priority Class %s is forbidden for the current Tenant: ", f.priorityClassName) - return utils.DefaultAllowedValuesErrorMessage(f.spec, msg) + return DefaultAllowedValuesErrorMessage(f.spec, msg) } type PodRuntimeClassForbiddenError struct { @@ -133,5 +132,5 @@ func NewPodRuntimeClassForbidden(runtimeClassName string, spec api.DefaultAllowe func (f PodRuntimeClassForbiddenError) Error() (err string) { err = fmt.Sprintf("Pod Runtime Class %s is forbidden for the current Tenant: ", f.runtimeClassName) - return utils.DefaultAllowedValuesErrorMessage(f.spec, err) + return DefaultAllowedValuesErrorMessage(f.spec, err) } diff --git a/pkg/api/errors/storage.go b/pkg/api/errors/storage.go index e7cc7f9c..53c7714e 100644 --- a/pkg/api/errors/storage.go +++ b/pkg/api/errors/storage.go @@ -6,7 +6,6 @@ package errors import ( "fmt" - "github.com/projectcapsule/capsule/internal/webhook/utils" "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" evt "github.com/projectcapsule/capsule/pkg/runtime/events" @@ -41,7 +40,7 @@ func NewStorageClassNotValid(storageClasses api.DefaultAllowedListSpec) error { func (s StorageClassNotValidError) Error() (err string) { msg := "A valid Storage Class must be used: " - return utils.DefaultAllowedValuesErrorMessage(s.spec, msg) + return DefaultAllowedValuesErrorMessage(s.spec, msg) } type StorageClassForbiddenError struct { @@ -59,7 +58,7 @@ func NewStorageClassForbidden(className string, storageClasses api.DefaultAllowe func (f StorageClassForbiddenError) Error() string { msg := fmt.Sprintf("Storage Class %s is forbidden for the current Tenant ", f.className) - return utils.DefaultAllowedValuesErrorMessage(f.spec, msg) + return DefaultAllowedValuesErrorMessage(f.spec, msg) } type MissingPVTenantLabelsError struct { diff --git a/pkg/api/errors/utils.go b/pkg/api/errors/utils.go new file mode 100644 index 00000000..669a3256 --- /dev/null +++ b/pkg/api/errors/utils.go @@ -0,0 +1,18 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package errors + +import ( + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +func IgnoreGone(err error) bool { + return err == nil || + apierrors.IsNotFound(err) || + apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) || + strings.Contains(err.Error(), " not found") +} diff --git a/pkg/api/meta/annotations.go b/pkg/api/meta/annotations.go index 4c28cc42..416a7927 100644 --- a/pkg/api/meta/annotations.go +++ b/pkg/api/meta/annotations.go @@ -4,8 +4,14 @@ package meta import ( + "context" "strings" + "time" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -40,6 +46,35 @@ func ReleaseAnnotationRemove(obj client.Object) { annotationRemove(obj, ReleaseAnnotation) } +func TriggerRequestReconcileAnnotation( + ctx context.Context, + c client.Client, + gvk schema.GroupVersionKind, + key types.NamespacedName, +) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + + if err := c.Get(ctx, key, obj); err != nil { + return err + } + + base := obj.DeepCopy() + + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + + annotations[ReconcileAnnotation] = time.Now().UTC().Format(time.RFC3339Nano) + + obj.SetAnnotations(annotations) + + return c.Patch(ctx, obj, client.MergeFrom(base)) + }) +} + func annotationRemove(obj client.Object, anno string) { annotations := obj.GetAnnotations() diff --git a/pkg/api/meta/conditions.go b/pkg/api/meta/conditions.go index f7740254..de15f7a7 100644 --- a/pkg/api/meta/conditions.go +++ b/pkg/api/meta/conditions.go @@ -10,27 +10,40 @@ import ( const ( // ReadyCondition indicates the resource is ready and fully reconciled. - ReadyCondition string = "Ready" - CordonedCondition string = "Cordoned" - NotReadyCondition string = "NotReady" + ReadyCondition string = "Ready" + CordonedCondition string = "Cordoned" + TerminatingCondition string = "Terminating" + NotReadyCondition string = "NotReady" AssignedCondition string = "Assigned" BoundCondition string = "Bound" ExhaustedCondition string = "Exhausted" // FailedReason indicates a condition or event observed a failure (Claim Rejected). - SucceededReason string = "Succeeded" - FailedReason string = "Failed" - ActiveReason string = "Active" - CordonedReason string = "Cordoned" - PoolExhaustedReason string = "PoolExhausted" - QueueExhaustedReason string = "QueueExhausted" - NamespaceExhaustedReason string = "NamespaceExhausted" - NoExhaustionsReason string = "NoExhaustions" - InUseReason string = "InUse" - UnusedReason string = "Unused" + SucceededReason string = "Succeeded" + FailedReason string = "Failed" + ActiveReason string = "Active" + CordonedReason string = "Cordoned" + TerminatingReason string = "Terminating" + ReconcilingReason string = "Reconciling" + PoolExhaustedReason string = "PoolExhausted" + QueueExhaustedReason string = "QueueExhausted" + NamespaceExhaustedReason string = "NamespaceExhausted" + NoExhaustionsReason string = "NoExhaustions" + InUseReason string = "InUse" + UnusedReason string = "Unused" + PendingUnmanagedContentReason string = "PendingUnmanagedContent" ) +func IsStatusConditionTrue(conditions ConditionList, conditionType string) bool { + cond := conditions.GetConditionByType(conditionType) + if cond == nil { + return false + } + + return cond.Status == metav1.ConditionTrue +} + // +kubebuilder:object:generate=true type ConditionList []Condition @@ -140,6 +153,26 @@ func NewAssignedCondition(obj client.Object) Condition { } } +func NewReadyConditionReconcilingReason(obj client.Object) Condition { + return Condition{ + Type: ReadyCondition, + Status: metav1.ConditionUnknown, + Reason: ReconcilingReason, + Message: "processing", + LastTransitionTime: metav1.Now(), + } +} + +func NewTerminatingConditionReason(obj client.Object) Condition { + return Condition{ + Type: TerminatingCondition, + Status: metav1.ConditionTrue, + Reason: SucceededReason, + Message: "cleaning up", + LastTransitionTime: metav1.Now(), + } +} + // Disregards fields like LastTransitionTime and Version, which are not relevant for the API. func (c *Condition) UpdateCondition(condition Condition) (updated bool) { if condition.Type == c.Type && diff --git a/pkg/api/meta/const.go b/pkg/api/meta/const.go index 804abeea..0c8a915a 100644 --- a/pkg/api/meta/const.go +++ b/pkg/api/meta/const.go @@ -4,7 +4,9 @@ package meta const ( - ValueTrue string = "true" - ValueFalse string = "false" - ValueController string = "controller" + ValueTrue string = "true" + ValueFalse string = "false" + ValueController string = "controller" + ValueControllerResources string = "resources" + ValueControllerReplications string = "replications" ) diff --git a/pkg/api/meta/finalizers.go b/pkg/api/meta/finalizers.go index 92122459..6dc95e82 100644 --- a/pkg/api/meta/finalizers.go +++ b/pkg/api/meta/finalizers.go @@ -3,6 +3,63 @@ package meta +import "encoding/json" + const ( - ControllerFinalizer = "controller.projectcapsule.dev/finalize" + ControllerFinalizer = "controller.projectcapsule.dev/finalize" + LegacyResourceFinalizer = "capsule.clastix.io/resources" ) + +var EmptyFinalizersMergePatch = []byte(`{"metadata":{"finalizers":[]}}`) + +func FilterFinalizers(finalizers []string, ignored map[string]struct{}) (remaining []string, removed bool) { + if len(finalizers) == 0 { + return nil, false + } + + if len(ignored) == 0 { + return nil, true + } + + remaining = make([]string, 0, len(finalizers)) + + removed = false + + for _, f := range finalizers { + if _, ok := ignored[f]; ok { + remaining = append(remaining, f) + + continue + } + + removed = true + } + + if len(remaining) == 0 { + return nil, removed + } + + return remaining, removed +} + +func BuildFinalizersMergePatch(finalizers []string) []byte { + if len(finalizers) == 0 { + return EmptyFinalizersMergePatch + } + + type metadata struct { + Finalizers []string `json:"finalizers"` + } + + type patch struct { + Metadata metadata `json:"metadata"` + } + + raw, _ := json.Marshal(patch{ //nolint:errchkjson + Metadata: metadata{ + Finalizers: finalizers, + }, + }) + + return raw +} diff --git a/pkg/api/meta/finalizers_test.go b/pkg/api/meta/finalizers_test.go new file mode 100644 index 00000000..4c929902 --- /dev/null +++ b/pkg/api/meta/finalizers_test.go @@ -0,0 +1,167 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package meta_test + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +func TestFilterFinalizers(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + finalizers []string + ignored map[string]struct{} + want []string + wantRemoved bool + }{ + { + name: "nil finalizers", + finalizers: nil, + ignored: map[string]struct{}{"keep.me": {}}, + want: nil, + wantRemoved: false, + }, + { + name: "empty finalizers", + finalizers: []string{}, + ignored: map[string]struct{}{"keep.me": {}}, + want: nil, + wantRemoved: false, + }, + { + name: "empty ignored removes all", + finalizers: []string{"a", "b"}, + ignored: map[string]struct{}{}, + want: nil, + wantRemoved: true, + }, + { + name: "all finalizers ignored", + finalizers: []string{"a", "b"}, + ignored: map[string]struct{}{"a": {}, "b": {}}, + want: []string{"a", "b"}, + wantRemoved: false, + }, + { + name: "some ignored some removed", + finalizers: []string{"a", "b", "c"}, + ignored: map[string]struct{}{"b": {}}, + want: []string{"b"}, + wantRemoved: true, + }, + { + name: "none ignored all removed", + finalizers: []string{"a", "b", "c"}, + ignored: map[string]struct{}{"x": {}}, + want: nil, + wantRemoved: true, + }, + { + name: "duplicates preserved when ignored", + finalizers: []string{"a", "b", "a"}, + ignored: map[string]struct{}{"a": {}}, + want: []string{"a", "a"}, + wantRemoved: true, + }, + { + name: "order preserved for ignored finalizers", + finalizers: []string{"c", "a", "b"}, + ignored: map[string]struct{}{"b": {}, "c": {}}, + want: []string{"c", "b"}, + wantRemoved: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, gotRemoved := meta.FilterFinalizers(tt.finalizers, tt.ignored) + + if gotRemoved != tt.wantRemoved { + t.Fatalf("FilterFinalizers() removed = %v, want %v", gotRemoved, tt.wantRemoved) + } + + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("FilterFinalizers() finalizers = %#v, want %#v", got, tt.want) + } + }) + } +} + +func TestBuildFinalizersMergePatch(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + finalizers []string + wantJSON string + }{ + { + name: "nil finalizers", + finalizers: nil, + wantJSON: `{"metadata":{"finalizers":[]}}`, + }, + { + name: "empty finalizers", + finalizers: []string{}, + wantJSON: `{"metadata":{"finalizers":[]}}`, + }, + { + name: "single finalizer", + finalizers: []string{"example.com/test"}, + wantJSON: `{"metadata":{"finalizers":["example.com/test"]}}`, + }, + { + name: "multiple finalizers", + finalizers: []string{"a", "b", "c"}, + wantJSON: `{"metadata":{"finalizers":["a","b","c"]}}`, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := meta.BuildFinalizersMergePatch(tt.finalizers) + + if string(got) != tt.wantJSON { + t.Fatalf("BuildFinalizersMergePatch() = %s, want %s", string(got), tt.wantJSON) + } + }) + } +} + +func TestBuildFinalizersMergePatch_ProducesValidJSON(t *testing.T) { + t.Parallel() + + got := meta.BuildFinalizersMergePatch([]string{"a", "b"}) + + var decoded map[string]any + if err := json.Unmarshal(got, &decoded); err != nil { + t.Fatalf("BuildFinalizersMergePatch() produced invalid JSON: %v", err) + } + + metadata, ok := decoded["metadata"].(map[string]any) + if !ok { + t.Fatalf("metadata field missing or wrong type: %#v", decoded["metadata"]) + } + + finalizers, ok := metadata["finalizers"].([]any) + if !ok { + t.Fatalf("finalizers field missing or wrong type: %#v", metadata["finalizers"]) + } + + if len(finalizers) != 2 || finalizers[0] != "a" || finalizers[1] != "b" { + t.Fatalf("unexpected finalizers: %#v", finalizers) + } +} diff --git a/pkg/api/meta/labels.go b/pkg/api/meta/labels.go index 65f4d3e8..8860834f 100644 --- a/pkg/api/meta/labels.go +++ b/pkg/api/meta/labels.go @@ -6,11 +6,15 @@ package meta import ( "strings" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" ) const ( + ResourcesLabel = "capsule.clastix.io/resources" + TenantNameLabel = "kubernetes.io/metadata.name" TenantLabel = "capsule.clastix.io/tenant" @@ -20,15 +24,15 @@ const ( FreezeLabel = "projectcapsule.dev/freeze" - OwnerPromotionLabel = "owner.projectcapsule.dev/promote" + OwnerPromotionLabel = "owner.projectcapsule.dev/promote" + ServiceAccountPromotionLabel = "projectcapsule.dev/promote" CordonedLabel = "projectcapsule.dev/cordoned" CapsuleNameLabel = "projectcapsule.dev/name" CreatedByCapsuleLabel = "projectcapsule.dev/created-by" - - CustomResourcesLabel = "projectcapsule.dev/custom-resources" + CustomResourcesLabel = "projectcapsule.dev/custom-resources" NewManagedByCapsuleLabel = "projectcapsule.dev/managed-by" ManagedByCapsuleLabel = "capsule.clastix.io/managed-by" @@ -110,3 +114,27 @@ func LabelsChanged(keys []string, oldLabels, newLabels map[string]string) bool { return false } + +func LabelsChangedUnstructured(oldObj, newObj unstructured.Unstructured) bool { + return !labels.Equals(labels.Set(oldObj.GetLabels()), labels.Set(newObj.GetLabels())) +} + +// Collect all mentioned keys from a LabelSelector. +func LabelSelectorKeys(sel *metav1.LabelSelector) map[string]struct{} { + out := map[string]struct{}{} + if sel == nil { + return out + } + + for k := range sel.MatchLabels { + out[k] = struct{}{} + } + + for _, expr := range sel.MatchExpressions { + if expr.Key != "" { + out[expr.Key] = struct{}{} + } + } + + return out +} diff --git a/pkg/api/meta/managers.go b/pkg/api/meta/managers.go index f6a2b257..30af3155 100644 --- a/pkg/api/meta/managers.go +++ b/pkg/api/meta/managers.go @@ -3,6 +3,12 @@ package meta +import ( + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + const ( FieldManagerCapsulePrefix = "projectcapsule.dev" FieldManagerCapsuleController = "projectcapsule.dev/controller" @@ -11,3 +17,48 @@ const ( func ControllerFieldOwnerPrefix(fieldowner string) string { return FieldManagerCapsulePrefix + "/" + fieldowner } + +func ControllerFieldOwner() string { + return ControllerFieldOwnerPrefix("controller") +} + +func ResourceControllerFieldOwnerPrefix() string { + return FieldManagerCapsulePrefix + "/resource/controller" +} + +// CapsuleFieldOwners returns the set of managers that start with the Capsule prefix. +func CapsuleFieldOwners(obj *unstructured.Unstructured, prefix string) map[string]struct{} { + out := map[string]struct{}{} + if obj == nil { + return out + } + + for _, mf := range obj.GetManagedFields() { + mgr := mf.Manager + if mgr == "" { + continue + } + + if strings.HasPrefix(mgr, prefix) { + out[mgr] = struct{}{} + } + } + + return out +} + +func HasExactlyCapsuleOwners(obj *unstructured.Unstructured, prefix string, allowed []string) bool { + owners := CapsuleFieldOwners(obj, prefix) + + if len(owners) != len(allowed) { + return false + } + + for _, a := range allowed { + if _, ok := owners[a]; !ok { + return false + } + } + + return true +} diff --git a/pkg/api/meta/names.go b/pkg/api/meta/names.go index 4da09447..e6b94792 100644 --- a/pkg/api/meta/names.go +++ b/pkg/api/meta/names.go @@ -12,3 +12,7 @@ func NameForManagedRuleStatus() string { func NameForManagedRoleBindings(hash string) string { return fmt.Sprintf("capsule:managed:%s", hash) } + +func NameForManagedPoolResourceQuota(name string) string { + return fmt.Sprintf("capsule-pool-%s", name) +} diff --git a/pkg/api/meta/processed.go b/pkg/api/meta/processed.go new file mode 100644 index 00000000..47081c1a --- /dev/null +++ b/pkg/api/meta/processed.go @@ -0,0 +1,74 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package meta + +import ( + "sort" + + "github.com/projectcapsule/capsule/pkg/runtime/gvk" +) + +type ProcessedItems []ObjectReferenceStatus + +// Adds a condition by type. +func (p *ProcessedItems) UpdateItem(item ObjectReferenceStatus) { + for i, stat := range *p { + if p.isEqual(stat, item) { + (*p)[i].ObjectReferenceStatusCondition = item.ObjectReferenceStatusCondition + + return + } + } + + *p = append(*p, item) +} + +// Removes a condition by type. +func (p *ProcessedItems) RemoveItem(item ObjectReferenceStatus) { + filtered := make(ProcessedItems, 0, len(*p)) + + for _, stat := range *p { + if !p.isEqual(stat, item) { + filtered = append(filtered, stat) + } + } + + *p = filtered +} + +// Removes a condition by type. +// Returns actual item pointer, not a copy. +func (p *ProcessedItems) GetItem(ref gvk.ResourceID) *ObjectReferenceStatus { + for i := range *p { + if (*p)[i].ResourceID == ref { + return &(*p)[i] + } + } + + return nil +} + +func (p ProcessedItems) SortDeterministic() { + sort.Slice(p, func(i, j int) bool { + a, b := p[i], p[j] + + if a.Tenant != b.Tenant { + return a.Tenant < b.Tenant + } + + if a.Namespace != b.Namespace { + return a.Namespace < b.Namespace + } + + if a.Name != b.Name { + return a.Name < b.Name + } + + return a.Kind < b.Kind + }) +} + +func (p *ProcessedItems) isEqual(a, b ObjectReferenceStatus) bool { + return a.ResourceID == b.ResourceID +} diff --git a/pkg/api/meta/processed_test.go b/pkg/api/meta/processed_test.go new file mode 100644 index 00000000..6dc49efc --- /dev/null +++ b/pkg/api/meta/processed_test.go @@ -0,0 +1,188 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package meta_test + +import ( + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/gvk" +) + +func mkItem(tenant, namespace, name, kind string, status metav1.ConditionStatus, condType, msg string, created bool, lastApply metav1.Time) meta.ObjectReferenceStatus { + return meta.ObjectReferenceStatus{ + ResourceID: gvk.ResourceID{ + TenantResourceIDWithOrigin: gvk.TenantResourceIDWithOrigin{ + TenantResourceID: gvk.TenantResourceID{Tenant: tenant}, + Origin: "", + }, + Group: "", + Version: "", + Kind: kind, + Name: name, + Namespace: namespace, + }, + ObjectReferenceStatusCondition: meta.ObjectReferenceStatusCondition{ + Status: status, + Type: condType, + Message: msg, + LastApply: lastApply, + Created: created, + }, + } +} + +func TestProcessedItems_UpdateItem_AppendsNew(t *testing.T) { + var p meta.ProcessedItems + + now := metav1.NewTime(time.Now()) + + item := mkItem("t1", "ns1", "name1", "Secret", metav1.ConditionTrue, "Ready", "ok", true, now) + p.UpdateItem(item) + + if len(p) != 1 { + t.Fatalf("expected 1 item, got %d", len(p)) + } + if p[0].ResourceID != item.ResourceID { + t.Fatalf("expected ResourceID %+v, got %+v", item.ResourceID, p[0].ResourceID) + } + if p[0].ObjectReferenceStatusCondition != item.ObjectReferenceStatusCondition { + t.Fatalf("expected condition %+v, got %+v", item.ObjectReferenceStatusCondition, p[0].ObjectReferenceStatusCondition) + } +} + +func TestProcessedItems_UpdateItem_UpdatesExistingWithoutDuplicate(t *testing.T) { + now1 := metav1.NewTime(time.Now().Add(-1 * time.Hour)) + now2 := metav1.NewTime(time.Now()) + + original := mkItem("t1", "ns1", "name1", "Secret", metav1.ConditionFalse, "Ready", "initial", false, now1) + updated := mkItem("t1", "ns1", "name1", "Secret", metav1.ConditionTrue, "Ready", "updated", true, now2) + + p := meta.ProcessedItems{original} + p.UpdateItem(updated) + + if len(p) != 1 { + t.Fatalf("expected 1 item (updated in place), got %d", len(p)) + } + + // Resource identity must remain same + if p[0].ResourceID != original.ResourceID { + t.Fatalf("expected ResourceID unchanged %+v, got %+v", original.ResourceID, p[0].ResourceID) + } + + // Condition should be replaced + if p[0].ObjectReferenceStatusCondition != updated.ObjectReferenceStatusCondition { + t.Fatalf("expected condition replaced with %+v, got %+v", updated.ObjectReferenceStatusCondition, p[0].ObjectReferenceStatusCondition) + } +} + +func TestProcessedItems_RemoveItem_RemovesMatchingByResourceID(t *testing.T) { + now := metav1.NewTime(time.Now()) + + a := mkItem("t1", "ns1", "name1", "Secret", metav1.ConditionTrue, "Ready", "a", true, now) + b := mkItem("t1", "ns1", "name2", "ConfigMap", metav1.ConditionTrue, "Ready", "b", true, now) + c := mkItem("t2", "ns9", "name9", "Secret", metav1.ConditionFalse, "Ready", "c", false, now) + + p := meta.ProcessedItems{a, b, c} + + // Remove b + p.RemoveItem(b) + + if len(p) != 2 { + t.Fatalf("expected 2 items after removal, got %d", len(p)) + } + + // Ensure b is gone, others remain + for _, it := range p { + if it.ResourceID == b.ResourceID { + t.Fatalf("did not expect removed item %+v to remain", b.ResourceID) + } + } + // Ensure a and c still exist + foundA, foundC := false, false + for _, it := range p { + if it.ResourceID == a.ResourceID { + foundA = true + } + if it.ResourceID == c.ResourceID { + foundC = true + } + } + if !foundA || !foundC { + t.Fatalf("expected remaining items to include a=%v c=%v", foundA, foundC) + } +} + +func TestProcessedItems_GetItem_ReturnsPointerToSliceElement(t *testing.T) { + now := metav1.NewTime(time.Now()) + + a := mkItem("t1", "ns1", "name1", "Secret", metav1.ConditionFalse, "Ready", "a", false, now) + b := mkItem("t1", "ns1", "name2", "Secret", metav1.ConditionFalse, "Ready", "b", false, now) + + p := meta.ProcessedItems{a, b} + + ptr := p.GetItem(b.ResourceID) + if ptr == nil { + t.Fatalf("expected to find item %v", b.ResourceID) + } + + // Mutate through pointer and ensure slice reflects it (proves it's not a copy) + ptr.Message = "changed" + ptr.Status = metav1.ConditionTrue + ptr.Created = true + + if p[1].Message != "changed" { + t.Fatalf("expected slice element Message to be changed, got %q", p[1].Message) + } + if p[1].Status != metav1.ConditionTrue { + t.Fatalf("expected slice element Status to be changed, got %q", p[1].Status) + } + if p[1].Created != true { + t.Fatalf("expected slice element Created to be changed, got %v", p[1].Created) + } + + // Not found case + none := p.GetItem(gvk.ResourceID{Name: "does-not-exist"}) + if none != nil { + t.Fatalf("expected nil for non-existent item, got %+v", none) + } +} + +func TestProcessedItems_SortDeterministic(t *testing.T) { + now := metav1.NewTime(time.Now()) + + // Intentionally shuffled order + i1 := mkItem("tenant-b", "ns-a", "name-a", "Secret", metav1.ConditionTrue, "Ready", "", true, now) + i2 := mkItem("tenant-a", "ns-b", "name-a", "Secret", metav1.ConditionTrue, "Ready", "", true, now) + i3 := mkItem("tenant-a", "ns-a", "name-b", "Secret", metav1.ConditionTrue, "Ready", "", true, now) + i4 := mkItem("tenant-a", "ns-a", "name-a", "ZKind", metav1.ConditionTrue, "Ready", "", true, now) + i5 := mkItem("tenant-a", "ns-a", "name-a", "AKind", metav1.ConditionTrue, "Ready", "", true, now) + i6 := mkItem("tenant-b", "ns-a", "name-a", "ConfigMap", metav1.ConditionTrue, "Ready", "", true, now) + + p := meta.ProcessedItems{i1, i2, i3, i4, i5, i6} + p.SortDeterministic() + + // Expected order by: Tenant, Namespace, Name, Kind + want := []gvk.ResourceID{ + i5.ResourceID, // tenant-a, ns-a, name-a, AKind + i4.ResourceID, // tenant-a, ns-a, name-a, ZKind + i3.ResourceID, // tenant-a, ns-a, name-b, Secret + i2.ResourceID, // tenant-a, ns-b, name-a, Secret + i6.ResourceID, // tenant-b, ns-a, name-a, ConfigMap + i1.ResourceID, // tenant-b, ns-a, name-a, Secret + } + + if len(p) != len(want) { + t.Fatalf("expected %d items, got %d", len(want), len(p)) + } + + for idx := range want { + if p[idx].ResourceID != want[idx] { + t.Fatalf("at index %d: expected %v, got %v", idx, want[idx], p[idx].ResourceID) + } + } +} diff --git a/pkg/api/meta/reference.go b/pkg/api/meta/reference.go index c9329b5a..c975eeab 100644 --- a/pkg/api/meta/reference.go +++ b/pkg/api/meta/reference.go @@ -3,7 +3,12 @@ package meta -import k8stypes "k8s.io/apimachinery/pkg/types" +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + + "github.com/projectcapsule/capsule/pkg/runtime/gvk" +) // NamespaceName must be a lowercase RFC1123 label. // +kubebuilder:validation:Pattern=^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ @@ -66,6 +71,23 @@ type NamespacedObjectReference struct { Namespace RFC1123SubdomainName `json:"namespace,omitempty"` } +// NamespacedObjectReference contains enough information to locate the referenced Kubernetes resource object in any +// namespace. +// +kubebuilder:object:generate=true +type NamespacedObjectWithUIDReference struct { + // UID of the tracked Tenant to pin point tracking + // +required + k8stypes.UID `json:"uid,omitempty" protobuf:"bytes,5,opt,name=uid"` + + // Name of the referent. + // +required + Name string `json:"name"` + + // Namespace of the referent, when not specified it acts as LocalObjectReference. + // +optional + Namespace RFC1123SubdomainName `json:"namespace,omitempty"` +} + // NamespacedObjectReference contains enough information to locate the referenced Kubernetes resource object in any // namespace. But the namespace is required. // +kubebuilder:object:generate=true @@ -118,3 +140,50 @@ type NamespacedRFC1123ObjectReferenceWithNamespaceWithUID struct { // +required Namespace RFC1123SubdomainName `json:"namespace,omitempty"` } + +// Advanced Status Item for pin pointing items in tenants/namespaces. +// +kubebuilder:object:generate=true +type ObjectReferenceStatus struct { + gvk.ResourceID `json:",inline"` + + ObjectReferenceStatusCondition `json:"status,omitempty"` +} + +// +kubebuilder:object:generate=true +type ObjectReferenceStatusCondition struct { + // status of the condition, one of True, False, Unknown. + // +required + // +kubebuilder:validation:Required + // +kubebuilder:validation:Enum=True;False;Unknown + Status metav1.ConditionStatus `json:"status" protobuf:"bytes,2,opt,name=status"` + // message is a human readable message indicating details about the transition. + // This may be an empty string. + // +kubebuilder:validation:MaxLength=32768 + Message string `json:"message,omitempty" protobuf:"bytes,6,opt,name=message"` + // type of condition in CamelCase or in foo.example.com/CamelCase. + // --- + // Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + // useful (see .node.status.conditions), the ability to deconflict is important. + // The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + // +required + // +kubebuilder:validation:Required + // +kubebuilder:validation:Pattern=`^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$` + // +kubebuilder:validation:MaxLength=316 + Type string `json:"type" protobuf:"bytes,1,opt,name=type"` + + // An opaque value that represents the internal version of this object that can + // be used by clients to determine when objects have changed. May be used for optimistic + // concurrency, change detection, and the watch operation on a resource or set of resources. + // Clients must treat these values as opaque and passed unmodified back to the server. + // They may only be valid for a particular resource or set of resources. + // + // Populated by the system. + // Read-only. + // Value must be treated as opaque by clients and . + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + // +optional + LastApply metav1.Time `json:"lastApply,omitempty,omitzero" protobuf:"bytes,8,opt,name=lastApply"` + + // Indicates wether the resource was created or adopted + Created bool `json:"created,omitempty"` +} diff --git a/pkg/api/meta/selectors.go b/pkg/api/meta/selectors.go new file mode 100644 index 00000000..0db94e17 --- /dev/null +++ b/pkg/api/meta/selectors.go @@ -0,0 +1,35 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package meta + +import ( + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" +) + +var WithoutCapsuleManagedResourcesLabelSelector = func() string { + req, _ := labels.NewRequirement( + NewManagedByCapsuleLabel, + selection.NotIn, + []string{ + ValueController, + ValueControllerResources, + }, + ) + + return labels.NewSelector().Add(*req).String() +}() + +var WithCapsuleManagedResourcesLabelSelector = func() string { + req, _ := labels.NewRequirement( + NewManagedByCapsuleLabel, + selection.In, + []string{ + ValueController, + ValueControllerResources, + }, + ) + + return labels.NewSelector().Add(*req).String() +}() diff --git a/pkg/api/meta/zz_generated.deepcopy.go b/pkg/api/meta/zz_generated.deepcopy.go index 6a8926a8..452acc75 100644 --- a/pkg/api/meta/zz_generated.deepcopy.go +++ b/pkg/api/meta/zz_generated.deepcopy.go @@ -106,6 +106,21 @@ func (in *NamespacedObjectReferenceWithNamespace) DeepCopy() *NamespacedObjectRe return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NamespacedObjectWithUIDReference) DeepCopyInto(out *NamespacedObjectWithUIDReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespacedObjectWithUIDReference. +func (in *NamespacedObjectWithUIDReference) DeepCopy() *NamespacedObjectWithUIDReference { + if in == nil { + return nil + } + out := new(NamespacedObjectWithUIDReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NamespacedRFC1123ObjectReference) DeepCopyInto(out *NamespacedRFC1123ObjectReference) { *out = *in @@ -150,3 +165,36 @@ func (in *NamespacedRFC1123ObjectReferenceWithNamespaceWithUID) DeepCopy() *Name in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectReferenceStatus) DeepCopyInto(out *ObjectReferenceStatus) { + *out = *in + out.ResourceID = in.ResourceID + in.ObjectReferenceStatusCondition.DeepCopyInto(&out.ObjectReferenceStatusCondition) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectReferenceStatus. +func (in *ObjectReferenceStatus) DeepCopy() *ObjectReferenceStatus { + if in == nil { + return nil + } + out := new(ObjectReferenceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectReferenceStatusCondition) DeepCopyInto(out *ObjectReferenceStatusCondition) { + *out = *in + in.LastApply.DeepCopyInto(&out.LastApply) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectReferenceStatusCondition. +func (in *ObjectReferenceStatusCondition) DeepCopy() *ObjectReferenceStatusCondition { + if in == nil { + return nil + } + out := new(ObjectReferenceStatusCondition) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/api/namespace_rule_type.go b/pkg/api/namespace_rule_type.go new file mode 100644 index 00000000..00ba9c8f --- /dev/null +++ b/pkg/api/namespace_rule_type.go @@ -0,0 +1,54 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// For future inmplementatiosn where users might manage RuleStatus CRs tehmselves +// +kubebuilder:object:generate=true +type NamespaceRuleBodyNamespace struct { + // Enforcement for given rule + //+optional + Enforce NamespaceRuleEnforceBody `json:"enforce,omitzero"` +} + +// Rules Distributed via Tenants +// +kubebuilder:object:generate=true +type NamespaceRuleBodyTenant struct { + NamespaceRuleBodyNamespace `json:",inline"` + + // Select namespaces which are going to be targeted with this rule + NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"` + + // Permissions for given rule + //+optional + Permissions NamespaceRulePermissionBody `json:"permissions,omitzero"` +} + +// +kubebuilder:object:generate=true +type NamespaceRuleEnforceBody struct { + // Define registries which are allowed to be used within this tenant + // The rules are aggregated, since you can use Regular Expressions the match registry endpoints + Registries []OCIRegistry `json:"registries,omitempty"` +} + +// +kubebuilder:object:generate=true +type NamespaceRulePermissionBody struct { + // Define Promotion Rules which distributed additional ClusterRoles across the Tenant + // for promoted ServiceAccounts. + Promotions []*NamespaceRulePromotionRule `json:"rules,omitempty"` +} + +// +kubebuilder:object:generate=true +type NamespaceRulePromotionRule struct { + // ClusterRoles granted to the promoted ServiceAccounts across the Tenant + // kubebuilder:validation:Minimum=1 + ClusterRoles []string `json:"clusterRoles,omitempty"` + + // Match ServiceAccounts which are promoted which are granted these additional ClusterRoles + // across the Tenant + Selector *metav1.LabelSelector `json:"selector,omitempty"` +} diff --git a/pkg/api/owner_list_test.go b/pkg/api/owner_list_test.go deleted file mode 100644 index f4b82247..00000000 --- a/pkg/api/owner_list_test.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package api_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/projectcapsule/capsule/pkg/api" -) - -func TestOwnerListSpec_FindOwner(t *testing.T) { - bla := api.OwnerSpec{ - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.UserOwner, - Name: "bla", - }, - }, - ProxyOperations: []api.ProxySettings{ - { - Kind: api.IngressClassesProxy, - Operations: []api.ProxyOperation{"Delete"}, - }, - }, - } - bar := api.OwnerSpec{ - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, - Name: "bar", - }, - }, - ProxyOperations: []api.ProxySettings{ - { - Kind: api.StorageClassesProxy, - Operations: []api.ProxyOperation{"Delete"}, - }, - }, - } - baz := api.OwnerSpec{ - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.UserOwner, - Name: "baz", - }, - }, - ProxyOperations: []api.ProxySettings{ - { - Kind: api.StorageClassesProxy, - Operations: []api.ProxyOperation{"Update"}, - }, - }, - } - fim := api.OwnerSpec{ - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, - Name: "fim", - }, - }, - ProxyOperations: []api.ProxySettings{ - { - Kind: api.NodesProxy, - Operations: []api.ProxyOperation{"List"}, - }, - }, - } - bom := api.OwnerSpec{ - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, - Name: "bom", - }, - }, - ProxyOperations: []api.ProxySettings{ - { - Kind: api.StorageClassesProxy, - Operations: []api.ProxyOperation{"Delete"}, - }, - { - Kind: api.NodesProxy, - Operations: []api.ProxyOperation{"Delete"}, - }, - }, - } - qip := api.OwnerSpec{ - CoreOwnerSpec: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, - Name: "qip", - }, - }, - ProxyOperations: []api.ProxySettings{ - { - Kind: api.StorageClassesProxy, - Operations: []api.ProxyOperation{"List", "Delete"}, - }, - }, - } - owners := api.OwnerListSpec{bom, qip, bla, bar, baz, fim} - - assert.Equal(t, owners.FindOwner("bom", api.GroupOwner), bom) - assert.Equal(t, owners.FindOwner("qip", api.ServiceAccountOwner), qip) - assert.Equal(t, owners.FindOwner("bla", api.UserOwner), bla) - assert.Equal(t, owners.FindOwner("bar", api.GroupOwner), bar) - assert.Equal(t, owners.FindOwner("baz", api.UserOwner), baz) - assert.Equal(t, owners.FindOwner("fim", api.ServiceAccountOwner), fim) - assert.Equal(t, owners.FindOwner("notfound", api.ServiceAccountOwner), api.OwnerSpec{}) -} diff --git a/pkg/api/processor/accumulator.go b/pkg/api/processor/accumulator.go new file mode 100644 index 00000000..70783eba --- /dev/null +++ b/pkg/api/processor/accumulator.go @@ -0,0 +1,55 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package processor + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/projectcapsule/capsule/pkg/runtime/gvk" +) + +// Keeps track of generated items. +type Accumulator = map[string]*AccumulatorItem + +// Keeps track of generated items. +type AccumulatorItem struct { + Resource gvk.ResourceID + Objects *[]AccumulatorObject +} + +// Keeps track of generated items. +type AccumulatorObject struct { + Origin gvk.TenantResourceIDWithOrigin + Object *unstructured.Unstructured +} + +func AccumulatorAdd( + acc Accumulator, + resource gvk.ResourceID, + obj AccumulatorObject, +) { + if acc == nil { + return + } + + key := resource.GetKey("") + + if entry, ok := acc[key]; ok && entry != nil { + if entry.Objects == nil { + list := make([]AccumulatorObject, 0, 1) + entry.Objects = &list + } + + *entry.Objects = append(*entry.Objects, obj) + + return + } + + list := []AccumulatorObject{obj} + + acc[key] = &AccumulatorItem{ + Resource: resource, + Objects: &list, + } +} diff --git a/pkg/api/processor/processor.go b/pkg/api/processor/processor.go new file mode 100644 index 00000000..e2eebaaf --- /dev/null +++ b/pkg/api/processor/processor.go @@ -0,0 +1,27 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package processor + +import ( + k8smeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/projectcapsule/capsule/pkg/runtime/configuration" +) + +type Processor struct { + Configuration configuration.Configuration + AllowCrossNamespaceSelection bool + GatherClient client.Reader + Mapper k8smeta.RESTMapper +} + +type ProcessorOptions struct { + FieldOwnerPrefix string + Prune bool + Adopt bool + Force bool + Owner *metav1.OwnerReference +} diff --git a/pkg/api/processor/processor_func.go b/pkg/api/processor/processor_func.go new file mode 100644 index 00000000..f53ba103 --- /dev/null +++ b/pkg/api/processor/processor_func.go @@ -0,0 +1,460 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package processor + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + k8smeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/projectcapsule/capsule/pkg/api/meta" + clt "github.com/projectcapsule/capsule/pkg/runtime/client" +) + +//nolint:gocognit +func (p *Processor) Reconcile( + ctx context.Context, + log logr.Logger, + c client.Client, + processed *meta.ProcessedItems, + acc Accumulator, + opts ProcessorOptions, +) (err error) { + itemErrors := 0 + + log.V(5).Info("starting pruning items", "present", len(*processed)) + + failAndContinue := func(i meta.ObjectReferenceStatus, msg string, err error) bool { // replace ItemType + if err == nil { + return false + } + + itemErrors++ + i.Status = metav1.ConditionFalse + i.Message = msg + err.Error() + processed.UpdateItem(i) + + return true + } + + for _, i := range *processed { + if _, exists := acc[i.GetKey("")]; exists { + continue + } + + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(i.GetGVK()) + obj.SetName(i.GetName()) + + ns := i.GetNamespace() + if ns != "" { + obj.SetNamespace(ns) + } + + if i.LastApply.IsZero() { + processed.RemoveItem(i) + + continue + } + + if opts.Prune { + log.V(4).Info("pruning resources", "Kind", i.Kind, "Name", i.Name, "Namespace", i.Namespace) + + fieldOwner := opts.FieldOwnerPrefix + "/" + i.FieldOwner("") + + deleted, reconErr := p.Prune(ctx, c, obj, fieldOwner, &i) + if failAndContinue(i, "pruning failed for item: ", reconErr) { + continue + } + + if deleted { + processed.RemoveItem(i) + + continue + } + } + + // Disown item (only when GET succeeded) + patches, err := p.handleRemoveManagedMetadata(ctx, c, obj, opts.Owner) + if err != nil { + if apierrors.IsNotFound(err) { + processed.RemoveItem(i) + + continue + } + + if failAndContinue(i, "disowning failed for item: ", err) { + continue + } + } + + //nolint:nestif + if len(patches) > 0 { + err = clt.ApplyPatches(ctx, c, obj, patches, meta.ResourceControllerFieldOwnerPrefix()) + if err != nil { + if apierrors.IsNotFound(err) { + processed.RemoveItem(i) + + continue + } + + if failAndContinue(i, "removing metdata failed for item: ", err) { + continue + } + } + } + + processed.RemoveItem(i) + } + + if itemErrors > 0 { + return fmt.Errorf("pruning of %d resources failed", itemErrors) + } + + log.V(5).Info("accumulation after pruning", "items", len(acc)) + + for _, item := range acc { + or := meta.ObjectReferenceStatus{ + ResourceID: item.Resource, + ObjectReferenceStatusCondition: meta.ObjectReferenceStatusCondition{ + Type: meta.ReadyCondition, + }, + } + + hadError := false + + for _, obj := range *item.Objects { + fieldOwner := opts.FieldOwnerPrefix + "/" + item.Resource.FieldOwner("") + + ver, created, err := p.Apply( + ctx, + c, + obj.Object, + fieldOwner, + opts.Force, + opts.Adopt, + opts.Owner, + processed.GetItem(item.Resource), + ) + + or.Created = created + + if err != nil { + hadError = true + or.Status = metav1.ConditionFalse + or.Message = "apply failed for item " + obj.Origin.Origin + ": " + err.Error() + + log.V(4).Info("failed to apply item", "item", obj.Origin.Origin) + } else { + if ver != nil { + or.LastApply = *ver + } + + or.Status = metav1.ConditionTrue + + log.V(4).Info("successfully applied item", "item", obj.Origin.Origin, "version", ver) + } + + processed.UpdateItem(or) + } + + if hadError { + itemErrors++ + } + } + + if itemErrors > 0 { + return fmt.Errorf("applying of %d resources failed", itemErrors) + } + + // Running Healthchecks + + log.V(4).Info("processing completed") + + return nil +} + +// Prune by reverting the patch by the given fieldOwner +// If the item was created by the controller and has no more field-managers we are going to delete. +func (r *Processor) Prune( + ctx context.Context, + c client.Client, + obj *unstructured.Unstructured, + fieldOwner string, + current *meta.ObjectReferenceStatus, +) (deleted bool, err error) { + actual := &unstructured.Unstructured{} + actual.SetGroupVersionKind(obj.GroupVersionKind()) + actual.SetName(obj.GetName()) + + mapping, err := r.Mapper.RESTMapping(obj.GroupVersionKind().GroupKind(), obj.GroupVersionKind().Version) + if err != nil { + return false, err + } + + // Handles the case where the namespace was already deleted + if mapping.Scope.Name() == k8smeta.RESTScopeNameNamespace { + namespace := obj.GetNamespace() + actual.SetNamespace(namespace) + + ns := &corev1.Namespace{} + if err := r.GatherClient.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + + return false, err + } + } + + err = c.Get(ctx, client.ObjectKeyFromObject(actual), actual) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + + return false, err + } + + deletable, err := r.handlePruneDeletion( + ctx, + c, + actual, + fieldOwner, + current, + ) + if err != nil { + return deletable, err + } + + if deletable { + err = c.Delete(ctx, actual) + if apierrors.IsNotFound(err) { + return deletable, nil + } + + return deletable, err + } + + err = clt.PatchApply(ctx, c, obj, fieldOwner, false) + if apierrors.IsNotFound(err) { + return true, nil + } + + return false, err +} + +// Completely prune the resource when there's no more managers and the resource was created by the controller. +func (r *Processor) handlePruneDeletion( + ctx context.Context, + c client.Client, + actual *unstructured.Unstructured, + fieldOwner string, + current *meta.ObjectReferenceStatus, +) (bool, error) { + if current != nil && current.Created { + return true, nil + } + + labels := actual.GetLabels() + if _, ok := labels[meta.CreatedByCapsuleLabel]; !ok { + return false, nil + } + + return meta.HasExactlyCapsuleOwners(actual, meta.FieldManagerCapsulePrefix+"/resource/", []string{ + fieldOwner, + meta.ResourceControllerFieldOwnerPrefix(), + }), nil +} + +// Remove metadata from the controller when an object +// is not pruned. +func (r *Processor) handleRemoveManagedMetadata( + ctx context.Context, + c client.Client, + obj *unstructured.Unstructured, + ownerreference *metav1.OwnerReference, +) (patches []clt.JSONPatch, err error) { + existingObject := obj.DeepCopy() + + err = c.Get(ctx, client.ObjectKeyFromObject(existingObject), existingObject) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + + return nil, err + } + + // Remove Ownerreference if given + if ownerreference != nil { + patches = append(patches, clt.RemoveOwnerReferencePatch(existingObject.GetOwnerReferences(), ownerreference)...) + } + + // Remove Managed Labels + if v, ok := existingObject.GetLabels()[meta.NewManagedByCapsuleLabel]; !ok || v != meta.ValueControllerReplications { + return patches, nil + } + + patches = append(patches, clt.PatchRemoveLabels(existingObject.GetLabels(), []string{ + meta.NewManagedByCapsuleLabel, + })...) + + return patches, nil +} + +func (r *Processor) Apply( + ctx context.Context, + c client.Client, + obj *unstructured.Unstructured, + fieldOwner string, + force bool, + adopt bool, + ownerreference *metav1.OwnerReference, + current *meta.ObjectReferenceStatus, +) (lastApply *metav1.Time, created bool, err error) { + log := log.FromContext(ctx) + + actual := &unstructured.Unstructured{} + actual.SetGroupVersionKind(obj.GroupVersionKind()) + actual.SetName(obj.GetName()) + + ns := obj.GetNamespace() + if ns != "" { + actual.SetNamespace(ns) + } + + key := client.ObjectKeyFromObject(actual) + + // We need to mark an item if we create it with our patch to make proper Garbage Collection + // If it does not yet exist mark it + patches, created, err := r.handleCreatedMetadata(ctx, c, obj, ownerreference, adopt, current) + if err != nil { + return nil, created, fmt.Errorf("evaluating managed metadata: %w", err) + } + + err = retry.OnError( + retry.DefaultBackoff, + apierrors.IsConflict, + func() error { + return clt.PatchApply(ctx, c, obj, fieldOwner, force) + }, + ) + if err != nil { + return nil, created, fmt.Errorf("applying object failed: %w", err) + } + + err = retry.OnError( + retry.DefaultBackoff, + apierrors.IsNotFound, + func() error { + return c.Get(ctx, key, actual) + }, + ) + if err != nil { + return nil, created, fmt.Errorf("failed to get object after apply: %w", err) + } + + // Apply metadata patches if needed + log.V(4).Info("applying patches", "items", len(patches)) + + if len(patches) > 0 { + if err := clt.ApplyPatches(ctx, c, actual, patches, meta.ResourceControllerFieldOwnerPrefix()); err != nil { + return nil, created, err + } + } + + return clt.LastApplyTimeForManager(actual, fieldOwner), created, nil +} + +func (r *Processor) handleCreatedMetadata( + ctx context.Context, + c client.Client, + obj *unstructured.Unstructured, + ownerreference *metav1.OwnerReference, + allowAdoption bool, + current *meta.ObjectReferenceStatus, +) (patches []clt.JSONPatch, created bool, err error) { + created = false + + existingObject := obj.DeepCopy() + + err = c.Get(ctx, client.ObjectKeyFromObject(existingObject), existingObject) + + switch { + case apierrors.IsNotFound(err): + created = true + err = nil + case err != nil: + return nil, created, err + default: + if current != nil { + if current.Created { + created = true + } + } + + labels := existingObject.GetLabels() + + if v, ok := labels[meta.CreatedByCapsuleLabel]; ok && v == meta.ValueControllerReplications { + created = true + } + + if _, ok := labels[meta.ResourcesLabel]; ok { + created = true + + patches = append(patches, clt.PatchRemoveLabels(existingObject.GetLabels(), []string{ + meta.ResourcesLabel, + })..., + ) + } + } + + if created { + if ownerreference != nil { + patches = append(patches, clt.AddOwnerReferencePatch(existingObject.GetOwnerReferences(), ownerreference)...) + } + + if v, ok := existingObject.GetLabels()[meta.CreatedByCapsuleLabel]; !ok || v != meta.ValueControllerReplications { + patches = append(patches, clt.AddLabelsPatch(existingObject.GetLabels(), map[string]string{ + meta.CreatedByCapsuleLabel: meta.ValueControllerReplications, + })...) + + // Ensure There are labels otherwise the next patch overwrites labels struct + if existingObject.GetLabels() == nil { + existingObject.SetLabels(map[string]string{ + meta.CreatedByCapsuleLabel: meta.ValueControllerReplications, + }) + } + } + } + + if created || allowAdoption { + if v, ok := existingObject.GetLabels()[meta.NewManagedByCapsuleLabel]; !ok || v != meta.ValueControllerReplications { + patches = append(patches, clt.AddLabelsPatch(existingObject.GetLabels(), map[string]string{ + meta.NewManagedByCapsuleLabel: meta.ValueControllerReplications, + })...) + } + + return patches, created, err + } + + return nil, created, fmt.Errorf( + "object %s/%s %s/%s exists and cannot be adopted", + existingObject.GetAPIVersion(), + existingObject.GetKind(), + existingObject.GetNamespace(), + existingObject.GetName(), + ) +} diff --git a/pkg/api/additional_role_bindings.go b/pkg/api/rbac/additional_role_bindings.go similarity index 65% rename from pkg/api/additional_role_bindings.go rename to pkg/api/rbac/additional_role_bindings.go index ed2bdd65..77dc9fef 100644 --- a/pkg/api/additional_role_bindings.go +++ b/pkg/api/rbac/additional_role_bindings.go @@ -1,9 +1,13 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package api +package rbac -import rbacv1 "k8s.io/api/rbac/v1" +import ( + rbacv1 "k8s.io/api/rbac/v1" + + "github.com/projectcapsule/capsule/pkg/api/meta" +) // +kubebuilder:object:generate=true @@ -16,3 +20,10 @@ type AdditionalRoleBindingsSpec struct { // Additional Annotations for the synchronized rolebindings Annotations map[string]string `json:"annotations,omitempty"` } + +type AdditionalRoleBindingsWithNamespaceSpec struct { + AdditionalRoleBindingsSpec `json:",inline"` + + // Target Namespace + Namespace meta.RFC1123SubdomainName `json:"namespace"` +} diff --git a/pkg/api/owner.go b/pkg/api/rbac/owner.go similarity index 99% rename from pkg/api/owner.go rename to pkg/api/rbac/owner.go index 1d7438a6..aadcd14d 100644 --- a/pkg/api/owner.go +++ b/pkg/api/rbac/owner.go @@ -1,7 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package api +package rbac import ( rbacv1 "k8s.io/api/rbac/v1" diff --git a/pkg/api/owner_list.go b/pkg/api/rbac/owner_list.go similarity index 95% rename from pkg/api/owner_list.go rename to pkg/api/rbac/owner_list.go index 4a3bc8e5..14d698c1 100644 --- a/pkg/api/owner_list.go +++ b/pkg/api/rbac/owner_list.go @@ -1,7 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package api +package rbac import ( "slices" @@ -30,7 +30,7 @@ func (o OwnerListSpec) IsOwner(name string, groups []string) bool { } func (o OwnerListSpec) ToStatusOwners() OwnerStatusListSpec { - list := OwnerStatusListSpec{} + list := make(OwnerStatusListSpec, 0, len(o)) for _, owner := range o { list = append(list, owner.CoreOwnerSpec) } diff --git a/pkg/api/rbac/owner_list_test.go b/pkg/api/rbac/owner_list_test.go new file mode 100644 index 00000000..fc31de83 --- /dev/null +++ b/pkg/api/rbac/owner_list_test.go @@ -0,0 +1,112 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package rbac_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/projectcapsule/capsule/pkg/api/rbac" +) + +func TestOwnerListSpec_FindOwner(t *testing.T) { + bla := rbac.OwnerSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, + Name: "bla", + }, + }, + ProxyOperations: []rbac.ProxySettings{ + { + Kind: rbac.IngressClassesProxy, + Operations: []rbac.ProxyOperation{"Delete"}, + }, + }, + } + bar := rbac.OwnerSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, + Name: "bar", + }, + }, + ProxyOperations: []rbac.ProxySettings{ + { + Kind: rbac.StorageClassesProxy, + Operations: []rbac.ProxyOperation{"Delete"}, + }, + }, + } + baz := rbac.OwnerSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, + Name: "baz", + }, + }, + ProxyOperations: []rbac.ProxySettings{ + { + Kind: rbac.StorageClassesProxy, + Operations: []rbac.ProxyOperation{"Update"}, + }, + }, + } + fim := rbac.OwnerSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, + Name: "fim", + }, + }, + ProxyOperations: []rbac.ProxySettings{ + { + Kind: rbac.NodesProxy, + Operations: []rbac.ProxyOperation{"List"}, + }, + }, + } + bom := rbac.OwnerSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, + Name: "bom", + }, + }, + ProxyOperations: []rbac.ProxySettings{ + { + Kind: rbac.StorageClassesProxy, + Operations: []rbac.ProxyOperation{"Delete"}, + }, + { + Kind: rbac.NodesProxy, + Operations: []rbac.ProxyOperation{"Delete"}, + }, + }, + } + qip := rbac.OwnerSpec{ + CoreOwnerSpec: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, + Name: "qip", + }, + }, + ProxyOperations: []rbac.ProxySettings{ + { + Kind: rbac.StorageClassesProxy, + Operations: []rbac.ProxyOperation{"List", "Delete"}, + }, + }, + } + owners := rbac.OwnerListSpec{bom, qip, bla, bar, baz, fim} + + assert.Equal(t, owners.FindOwner("bom", rbac.GroupOwner), bom) + assert.Equal(t, owners.FindOwner("qip", rbac.ServiceAccountOwner), qip) + assert.Equal(t, owners.FindOwner("bla", rbac.UserOwner), bla) + assert.Equal(t, owners.FindOwner("bar", rbac.GroupOwner), bar) + assert.Equal(t, owners.FindOwner("baz", rbac.UserOwner), baz) + assert.Equal(t, owners.FindOwner("fim", rbac.ServiceAccountOwner), fim) + assert.Equal(t, owners.FindOwner("notfound", rbac.ServiceAccountOwner), rbac.OwnerSpec{}) +} diff --git a/pkg/api/owner_status_list.go b/pkg/api/rbac/owner_status_list.go similarity index 99% rename from pkg/api/owner_status_list.go rename to pkg/api/rbac/owner_status_list.go index d4fc61c9..ee279e5c 100644 --- a/pkg/api/owner_status_list.go +++ b/pkg/api/rbac/owner_status_list.go @@ -1,7 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package api +package rbac import ( "sort" diff --git a/pkg/api/owner_status_list_test.go b/pkg/api/rbac/owner_status_list_test.go similarity index 74% rename from pkg/api/owner_status_list_test.go rename to pkg/api/rbac/owner_status_list_test.go index bbb418e7..7843ce18 100644 --- a/pkg/api/owner_status_list_test.go +++ b/pkg/api/rbac/owner_status_list_test.go @@ -1,7 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package api_test +package rbac_test import ( "math/rand" @@ -10,17 +10,17 @@ import ( "testing" "time" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -func slowIsOwner(o api.OwnerStatusListSpec, name string, groups []string) bool { +func slowIsOwner(o rbac.OwnerStatusListSpec, name string, groups []string) bool { for _, owner := range o { switch owner.Kind { - case api.UserOwner, api.ServiceAccountOwner: + case rbac.UserOwner, rbac.ServiceAccountOwner: if name == owner.Name { return true } - case api.GroupOwner: + case rbac.GroupOwner: for _, group := range groups { if group == owner.Name { return true @@ -32,13 +32,13 @@ func slowIsOwner(o api.OwnerStatusListSpec, name string, groups []string) bool { } // linearFind is the obvious, slow, but correct reference implementation. -func linearFind(o api.OwnerStatusListSpec, name string, kind api.OwnerKind) (api.CoreOwnerSpec, bool) { +func linearFind(o rbac.OwnerStatusListSpec, name string, kind rbac.OwnerKind) (rbac.CoreOwnerSpec, bool) { for _, x := range o { if x.Kind == kind && x.Name == name { return x, true } } - return api.CoreOwnerSpec{}, false + return rbac.CoreOwnerSpec{}, false } // randomName generates a simple lowercase name of length n. @@ -52,11 +52,11 @@ func randomName(rnd *rand.Rand, n int) string { } func TestUpsert_AddsNewOwnerToEmptyList(t *testing.T) { - var list api.OwnerStatusListSpec + var list rbac.OwnerStatusListSpec - list.Upsert(api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + list.Upsert(rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "alice", }, ClusterRoles: []string{"admin"}, @@ -66,7 +66,7 @@ func TestUpsert_AddsNewOwnerToEmptyList(t *testing.T) { t.Fatalf("expected 1 owner, got %d", len(list)) } got := list[0] - if got.Kind != api.UserOwner || got.Name != "alice" { + if got.Kind != rbac.UserOwner || got.Name != "alice" { t.Fatalf("unexpected owner: %+v", got) } if !reflect.DeepEqual(got.ClusterRoles, []string{"admin"}) { @@ -75,19 +75,19 @@ func TestUpsert_AddsNewOwnerToEmptyList(t *testing.T) { } func TestUpsert_MergesClusterRolesForExistingOwner(t *testing.T) { - list := api.OwnerStatusListSpec{ + list := rbac.OwnerStatusListSpec{ { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "alice", }, ClusterRoles: []string{"admin", "capsule-namespace-deleter"}, }, } - list.Upsert(api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + list.Upsert(rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "alice", }, ClusterRoles: []string{"extra-sad"}, @@ -97,7 +97,7 @@ func TestUpsert_MergesClusterRolesForExistingOwner(t *testing.T) { t.Fatalf("expected 1 owner, got %d", len(list)) } got := list[0] - if got.Kind != api.UserOwner || got.Name != "alice" { + if got.Kind != rbac.UserOwner || got.Name != "alice" { t.Fatalf("unexpected owner: %+v", got) } @@ -109,19 +109,19 @@ func TestUpsert_MergesClusterRolesForExistingOwner(t *testing.T) { } func TestUpsert_DeduplicatesClusterRoles(t *testing.T) { - list := api.OwnerStatusListSpec{ + list := rbac.OwnerStatusListSpec{ { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "alice", }, ClusterRoles: []string{"admin", "viewer"}, }, } - list.Upsert(api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + list.Upsert(rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "alice", }, ClusterRoles: []string{"viewer", "editor"}, @@ -139,18 +139,18 @@ func TestUpsert_DeduplicatesClusterRoles(t *testing.T) { } func TestUpsert_KeepsListSortedAndMergesIntoExistingInUnsortedInitialSlice(t *testing.T) { - // Start with an unsorted slice, as could come from API/server - list := api.OwnerStatusListSpec{ + // Start with an unsorted slice, as could come from rbac/server + list := rbac.OwnerStatusListSpec{ { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "bob", }, ClusterRoles: []string{"bob-role"}, }, { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "alice", }, ClusterRoles: []string{"admin"}, @@ -158,9 +158,9 @@ func TestUpsert_KeepsListSortedAndMergesIntoExistingInUnsortedInitialSlice(t *te } // Upsert another alice - list.Upsert(api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + list.Upsert(rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "alice", }, ClusterRoles: []string{"extra"}, @@ -172,16 +172,16 @@ func TestUpsert_KeepsListSortedAndMergesIntoExistingInUnsortedInitialSlice(t *te // Ensure sorted by Kind.Name: alice before bob // (relies on ByKindAndName order) - sorted := make(api.OwnerStatusListSpec, len(list)) + sorted := make(rbac.OwnerStatusListSpec, len(list)) copy(sorted, list) - sort.Sort(api.GetByKindAndName(sorted)) + sort.Sort(rbac.GetByKindAndName(sorted)) if !reflect.DeepEqual(list, sorted) { t.Fatalf("expected list to be sorted by kind+name, got %#v", list) } // Find alice and check roles - var alice *api.CoreOwnerSpec + var alice *rbac.CoreOwnerSpec for i := range list { if list[i].Name == "alice" { alice = &list[i] @@ -199,19 +199,19 @@ func TestUpsert_KeepsListSortedAndMergesIntoExistingInUnsortedInitialSlice(t *te } func TestGetByKindAndNameOrdering(t *testing.T) { - o := api.OwnerStatusListSpec{ - api.CoreOwnerSpec{UserSpec: api.UserSpec{Name: "b", Kind: api.ServiceAccountOwner}}, - api.CoreOwnerSpec{UserSpec: api.UserSpec{Name: "z", Kind: api.UserOwner}}, - api.CoreOwnerSpec{UserSpec: api.UserSpec{Name: "a", Kind: api.GroupOwner}}, - api.CoreOwnerSpec{UserSpec: api.UserSpec{Name: "a", Kind: api.UserOwner}}, + o := rbac.OwnerStatusListSpec{ + rbac.CoreOwnerSpec{UserSpec: rbac.UserSpec{Name: "b", Kind: rbac.ServiceAccountOwner}}, + rbac.CoreOwnerSpec{UserSpec: rbac.UserSpec{Name: "z", Kind: rbac.UserOwner}}, + rbac.CoreOwnerSpec{UserSpec: rbac.UserSpec{Name: "a", Kind: rbac.GroupOwner}}, + rbac.CoreOwnerSpec{UserSpec: rbac.UserSpec{Name: "a", Kind: rbac.UserOwner}}, } // Sort using production ordering - got := append(api.OwnerStatusListSpec(nil), o...) - sort.Sort(api.GetByKindAndName(got)) + got := append(rbac.OwnerStatusListSpec(nil), o...) + sort.Sort(rbac.GetByKindAndName(got)) // Manually sorted expectation using the same logic. - want := append(api.OwnerStatusListSpec(nil), o...) + want := append(rbac.OwnerStatusListSpec(nil), o...) sort.Slice(want, func(i, j int) bool { if want[i].Kind.String() != want[j].Kind.String() { return want[i].Kind.String() < want[j].Kind.String() @@ -232,10 +232,10 @@ func TestGetByKindAndNameOrdering(t *testing.T) { func TestFindOwner_Randomized(t *testing.T) { rnd := rand.New(rand.NewSource(42)) // fixed seed for deterministic test runs - ownerKinds := []api.OwnerKind{ - api.GroupOwner, - api.UserOwner, - api.ServiceAccountOwner, + ownerKinds := []rbac.OwnerKind{ + rbac.GroupOwner, + rbac.UserOwner, + rbac.ServiceAccountOwner, } const ( @@ -245,12 +245,12 @@ func TestFindOwner_Randomized(t *testing.T) { ) for listIdx := 0; listIdx < numLists; listIdx++ { - var list api.OwnerStatusListSpec + var list rbac.OwnerStatusListSpec n := rnd.Intn(maxLength) for i := 0; i < n; i++ { k := ownerKinds[rnd.Intn(len(ownerKinds))] - list = append(list, api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + list = append(list, rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: randomName(rnd, 3+rnd.Intn(4)), // length 3–6 Kind: k, }, @@ -259,7 +259,7 @@ func TestFindOwner_Randomized(t *testing.T) { for lookupIdx := 0; lookupIdx < numLookupsPerList; lookupIdx++ { var qName string - var qKind api.OwnerKind + var qKind rbac.OwnerKind if len(list) > 0 && rnd.Float64() < 0.6 { // 60% of lookups: pick a real element, must be found @@ -272,7 +272,7 @@ func TestFindOwner_Randomized(t *testing.T) { qKind = ownerKinds[rnd.Intn(len(ownerKinds))] } - listCopy := append(api.OwnerStatusListSpec(nil), list...) + listCopy := append(rbac.OwnerStatusListSpec(nil), list...) gotOwner, gotFound := listCopy.FindOwner(qName, qKind) wantOwner, wantFound := linearFind(list, qName, qKind) @@ -291,10 +291,10 @@ func TestFindOwner_Randomized(t *testing.T) { func TestIsOwner_RandomizedMatchesSlowImplementation(t *testing.T) { rnd := rand.New(rand.NewSource(time.Now().UnixNano())) - ownerKinds := []api.OwnerKind{ - api.UserOwner, - api.GroupOwner, - api.ServiceAccountOwner, + ownerKinds := []rbac.OwnerKind{ + rbac.UserOwner, + rbac.GroupOwner, + rbac.ServiceAccountOwner, } const ( @@ -306,12 +306,12 @@ func TestIsOwner_RandomizedMatchesSlowImplementation(t *testing.T) { for listIdx := 0; listIdx < numLists; listIdx++ { // Generate a random owner list (possibly with duplicates). - var owners api.OwnerStatusListSpec + var owners rbac.OwnerStatusListSpec nOwners := rnd.Intn(maxOwnersPerList) for i := 0; i < nOwners; i++ { kind := ownerKinds[rnd.Intn(len(ownerKinds))] - owners = append(owners, api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ + owners = append(owners, rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ Name: randomName(rnd, 3+rnd.Intn(4)), // length 3–6 Kind: kind, }, diff --git a/pkg/api/owner_test.go b/pkg/api/rbac/owner_test.go similarity index 78% rename from pkg/api/owner_test.go rename to pkg/api/rbac/owner_test.go index 8156cdad..b68aad26 100644 --- a/pkg/api/owner_test.go +++ b/pkg/api/rbac/owner_test.go @@ -1,40 +1,40 @@ -package api_test +package rbac_test import ( "testing" rbacv1 "k8s.io/api/rbac/v1" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) func TestCoreOwnerSpec_ToAdditionalRolebindings(t *testing.T) { tests := []struct { name string - in api.CoreOwnerSpec - want []api.AdditionalRoleBindingsSpec + in rbac.CoreOwnerSpec + want []rbac.AdditionalRoleBindingsSpec }{ { name: "no cluster roles yields empty slice", - in: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + in: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "alice", }, ClusterRoles: nil, }, - want: []api.AdditionalRoleBindingsSpec{}, + want: []rbac.AdditionalRoleBindingsSpec{}, }, { name: "one role creates one binding with subject", - in: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + in: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "alice", }, ClusterRoles: []string{"admin"}, }, - want: []api.AdditionalRoleBindingsSpec{ + want: []rbac.AdditionalRoleBindingsSpec{ { ClusterRoleName: "admin", Subjects: []rbacv1.Subject{ @@ -45,14 +45,14 @@ func TestCoreOwnerSpec_ToAdditionalRolebindings(t *testing.T) { }, { name: "multiple roles create one binding per role (preserves order)", - in: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + in: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "devops", }, ClusterRoles: []string{"view", "edit"}, }, - want: []api.AdditionalRoleBindingsSpec{ + want: []rbac.AdditionalRoleBindingsSpec{ { ClusterRoleName: "view", Subjects: []rbacv1.Subject{ @@ -69,14 +69,14 @@ func TestCoreOwnerSpec_ToAdditionalRolebindings(t *testing.T) { }, { name: "serviceaccount subject is split correctly in bindings", - in: api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, + in: rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:capsule-system:capsule", }, ClusterRoles: []string{"admin", "service-admin"}, }, - want: []api.AdditionalRoleBindingsSpec{ + want: []rbac.AdditionalRoleBindingsSpec{ { ClusterRoleName: "admin", Subjects: []rbacv1.Subject{ diff --git a/pkg/api/rbac/promotion.go b/pkg/api/rbac/promotion.go new file mode 100644 index 00000000..4a4e6940 --- /dev/null +++ b/pkg/api/rbac/promotion.go @@ -0,0 +1,42 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package rbac + +import ( + rbacv1 "k8s.io/api/rbac/v1" + + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +// +kubebuilder:object:generate=true +type PromotionSpec struct { + UserSpec `json:",inline"` + + // Defines additional cluster-roles for the specific Owner. + // +kubebuilder:default={admin,capsule-namespace-deleter} + ClusterRoles []string `json:"clusterRoles,omitempty"` + + // Defines additional cluster-roles for the specific Owner. + Targets []string `json:"targets,omitempty"` +} + +func (o PromotionSpec) ToAdditionalRolebindings() []AdditionalRoleBindingsWithNamespaceSpec { + bindings := make([]AdditionalRoleBindingsWithNamespaceSpec, 0, len(o.ClusterRoles)) + + for _, ns := range o.Targets { + for _, clusterRoleName := range o.ClusterRoles { + bindings = append(bindings, AdditionalRoleBindingsWithNamespaceSpec{ + Namespace: meta.RFC1123SubdomainName(ns), + AdditionalRoleBindingsSpec: AdditionalRoleBindingsSpec{ + ClusterRoleName: clusterRoleName, + Subjects: []rbacv1.Subject{ + o.Subject(), + }, + }, + }) + } + } + + return bindings +} diff --git a/pkg/api/rbac/promotion_status_list.go b/pkg/api/rbac/promotion_status_list.go new file mode 100644 index 00000000..350a7965 --- /dev/null +++ b/pkg/api/rbac/promotion_status_list.go @@ -0,0 +1,139 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package rbac + +import ( + "sort" + "strings" +) + +// +kubebuilder:object:generate=true + +type PromotionStatusListSpec []PromotionSpec + +func (o *PromotionStatusListSpec) Upsert(newPromotion PromotionSpec) { + newPromotion.ClusterRoles = mergeSortedStrings(nil, newPromotion.ClusterRoles) + newPromotion.Targets = mergeSortedStrings(nil, newPromotion.Targets) + + promotions := *o + + for i := range promotions { + if !samePromotionIdentity(promotions[i], newPromotion) { + continue + } + + promotions[i].ClusterRoles = mergeSortedStrings(promotions[i].ClusterRoles, newPromotion.ClusterRoles) + + sort.Sort(GetPromotionByKindNameAndTargets(promotions)) + + *o = promotions + + return + } + + promotions = append(promotions, newPromotion) + sort.Sort(GetPromotionByKindNameAndTargets(promotions)) + + *o = promotions +} + +func (o PromotionStatusListSpec) FindUser(name string, kind OwnerKind) (PromotionSpec, bool) { + result := PromotionSpec{ + UserSpec: UserSpec{ + Name: name, + Kind: kind, + }, + } + + found := false + + for _, promotion := range o { + if promotion.Name != name || promotion.Kind != kind { + continue + } + + found = true + result.ClusterRoles = mergeSortedStrings(result.ClusterRoles, promotion.ClusterRoles) + result.Targets = mergeSortedStrings(result.Targets, promotion.Targets) + } + + if !found { + return PromotionSpec{}, false + } + + return result, true +} + +type GetPromotionByKindNameAndTargets PromotionStatusListSpec + +func (b GetPromotionByKindNameAndTargets) Len() int { + return len(b) +} + +func (b GetPromotionByKindNameAndTargets) Less(i, j int) bool { + return promotionLess(b[i], b[j]) +} + +func (b GetPromotionByKindNameAndTargets) Swap(i, j int) { + b[i], b[j] = b[j], b[i] +} + +func promotionLess(a, b PromotionSpec) bool { + if a.Kind.String() != b.Kind.String() { + return a.Kind.String() < b.Kind.String() + } + + if a.Name != b.Name { + return a.Name < b.Name + } + + return stringSliceKey(a.Targets) < stringSliceKey(b.Targets) +} + +func samePromotionIdentity(a, b PromotionSpec) bool { + return a.Kind == b.Kind && + a.Name == b.Name && + stringSliceKey(a.Targets) == stringSliceKey(b.Targets) +} + +func stringSliceKey(values []string) string { + sorted := mergeSortedStrings(nil, values) + + var key strings.Builder + + for i, value := range sorted { + if i > 0 { + key.WriteString("\x00") + } + + key.WriteString(value) + } + + return key.String() +} + +func mergeSortedStrings(existing []string, incoming []string) []string { + if len(existing) == 0 && len(incoming) == 0 { + return nil + } + + values := make(map[string]struct{}, len(existing)+len(incoming)) + + for _, value := range existing { + values[value] = struct{}{} + } + + for _, value := range incoming { + values[value] = struct{}{} + } + + merged := make([]string, 0, len(values)) + for value := range values { + merged = append(merged, value) + } + + sort.Strings(merged) + + return merged +} diff --git a/pkg/api/rbac/promotion_status_list_test.go b/pkg/api/rbac/promotion_status_list_test.go new file mode 100644 index 00000000..9c96be99 --- /dev/null +++ b/pkg/api/rbac/promotion_status_list_test.go @@ -0,0 +1,378 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package rbac + +import ( + "testing" +) + +func TestPromotionStatusListSpec_Upsert(t *testing.T) { + t.Run("adds new promotion", func(t *testing.T) { + promotions := PromotionStatusListSpec{} + + promotions.Upsert(PromotionSpec{ + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source:gitops", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-a"}, + }) + + expected := PromotionStatusListSpec{ + { + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source:gitops", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-a"}, + }, + } + + if !equalPromotions(promotions, expected) { + t.Fatalf("unexpected promotions\nexpected: %#v\ngot: %#v", expected, promotions) + } + }) + + t.Run("merges clusterroles for same kind name and targets", func(t *testing.T) { + promotions := PromotionStatusListSpec{} + + promotions.Upsert(PromotionSpec{ + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source:gitops", + }, + ClusterRoles: []string{"secret-replicator"}, + Targets: []string{"target-b", "target-a"}, + }) + + promotions.Upsert(PromotionSpec{ + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source:gitops", + }, + ClusterRoles: []string{"configmap-replicator"}, + Targets: []string{"target-a", "target-b"}, + }) + + expected := PromotionStatusListSpec{ + { + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source:gitops", + }, + ClusterRoles: []string{"configmap-replicator", "secret-replicator"}, + Targets: []string{"target-a", "target-b"}, + }, + } + + if !equalPromotions(promotions, expected) { + t.Fatalf("unexpected promotions\nexpected: %#v\ngot: %#v", expected, promotions) + } + }) + + t.Run("keeps dedicated entries for same owner with different targets", func(t *testing.T) { + promotions := PromotionStatusListSpec{} + + promotions.Upsert(PromotionSpec{ + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source:gitops", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-a", "target-b"}, + }) + + promotions.Upsert(PromotionSpec{ + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source:gitops", + }, + ClusterRoles: []string{"secret-replicator"}, + Targets: []string{"target-b"}, + }) + + expected := PromotionStatusListSpec{ + { + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source:gitops", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-a", "target-b"}, + }, + { + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source:gitops", + }, + ClusterRoles: []string{"secret-replicator"}, + Targets: []string{"target-b"}, + }, + } + + if !equalPromotions(promotions, expected) { + t.Fatalf("unexpected promotions\nexpected: %#v\ngot: %#v", expected, promotions) + } + }) + + t.Run("deduplicates clusterroles and targets", func(t *testing.T) { + promotions := PromotionStatusListSpec{} + + promotions.Upsert(PromotionSpec{ + UserSpec: UserSpec{ + Kind: UserOwner, + Name: "alice", + }, + ClusterRoles: []string{"view", "view", "edit"}, + Targets: []string{"target-b", "target-a", "target-a"}, + }) + + expected := PromotionStatusListSpec{ + { + UserSpec: UserSpec{ + Kind: UserOwner, + Name: "alice", + }, + ClusterRoles: []string{"edit", "view"}, + Targets: []string{"target-a", "target-b"}, + }, + } + + if !equalPromotions(promotions, expected) { + t.Fatalf("unexpected promotions\nexpected: %#v\ngot: %#v", expected, promotions) + } + }) + + t.Run("sorts promotions by kind name and targets", func(t *testing.T) { + promotions := PromotionStatusListSpec{} + + promotions.Upsert(PromotionSpec{ + UserSpec: UserSpec{ + Kind: UserOwner, + Name: "bob", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-b"}, + }) + + promotions.Upsert(PromotionSpec{ + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source:gitops", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-a"}, + }) + + promotions.Upsert(PromotionSpec{ + UserSpec: UserSpec{ + Kind: UserOwner, + Name: "alice", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-a"}, + }) + + expected := PromotionStatusListSpec{ + { + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source:gitops", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-a"}, + }, + { + UserSpec: UserSpec{ + Kind: UserOwner, + Name: "alice", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-a"}, + }, + { + UserSpec: UserSpec{ + Kind: UserOwner, + Name: "bob", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-b"}, + }, + } + + if !equalPromotions(promotions, expected) { + t.Fatalf("unexpected promotions\nexpected: %#v\ngot: %#v", expected, promotions) + } + }) +} + +func TestPromotionStatusListSpec_FindUser(t *testing.T) { + t.Run("finds and aggregates promotions for user", func(t *testing.T) { + promotions := PromotionStatusListSpec{ + { + UserSpec: UserSpec{ + Kind: UserOwner, + Name: "alice", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-a", "target-b"}, + }, + { + UserSpec: UserSpec{ + Kind: UserOwner, + Name: "alice", + }, + ClusterRoles: []string{"edit"}, + Targets: []string{"target-b", "target-c"}, + }, + { + UserSpec: UserSpec{ + Kind: UserOwner, + Name: "bob", + }, + ClusterRoles: []string{"admin"}, + Targets: []string{"target-d"}, + }, + } + + got, found := promotions.FindUser("alice", UserOwner) + if !found { + t.Fatal("expected user to be found") + } + + expected := PromotionSpec{ + UserSpec: UserSpec{ + Kind: UserOwner, + Name: "alice", + }, + ClusterRoles: []string{"edit", "view"}, + Targets: []string{"target-a", "target-b", "target-c"}, + } + + if !equalPromotion(got, expected) { + t.Fatalf("unexpected promotion\nexpected: %#v\ngot: %#v", expected, got) + } + }) + + t.Run("does not find user with different kind", func(t *testing.T) { + promotions := PromotionStatusListSpec{ + { + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "alice", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-a"}, + }, + } + + _, found := promotions.FindUser("alice", UserOwner) + if found { + t.Fatal("expected user not to be found for different kind") + } + }) + + t.Run("returns false when user does not exist", func(t *testing.T) { + promotions := PromotionStatusListSpec{ + { + UserSpec: UserSpec{ + Kind: UserOwner, + Name: "alice", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-a"}, + }, + } + + got, found := promotions.FindUser("missing", UserOwner) + if found { + t.Fatal("expected user not to be found") + } + + if !equalPromotion(got, PromotionSpec{}) { + t.Fatalf("expected zero promotion, got %#v", got) + } + }) +} + +func TestMergeSortedStrings(t *testing.T) { + tests := []struct { + name string + existing []string + incoming []string + expected []string + }{ + { + name: "returns nil when both slices are empty", + existing: nil, + incoming: nil, + expected: nil, + }, + { + name: "merges sorts and deduplicates values", + existing: []string{"b", "a"}, + incoming: []string{"c", "a"}, + expected: []string{"a", "b", "c"}, + }, + { + name: "handles empty existing values", + existing: nil, + incoming: []string{"b", "a", "b"}, + expected: []string{"a", "b"}, + }, + { + name: "handles empty incoming values", + existing: []string{"b", "a", "b"}, + incoming: nil, + expected: []string{"a", "b"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := mergeSortedStrings(tt.existing, tt.incoming) + + if !equalStrings(got, tt.expected) { + t.Fatalf("unexpected strings\nexpected: %#v\ngot: %#v", tt.expected, got) + } + }) + } +} + +func equalPromotions(a, b PromotionStatusListSpec) bool { + if len(a) != len(b) { + return false + } + + for i := range a { + if !equalPromotion(a[i], b[i]) { + return false + } + } + + return true +} + +func equalPromotion(a, b PromotionSpec) bool { + return a.Kind == b.Kind && + a.Name == b.Name && + equalStrings(a.ClusterRoles, b.ClusterRoles) && + equalStrings(a.Targets, b.Targets) +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} diff --git a/pkg/api/rbac/promotion_test.go b/pkg/api/rbac/promotion_test.go new file mode 100644 index 00000000..8ba6e572 --- /dev/null +++ b/pkg/api/rbac/promotion_test.go @@ -0,0 +1,218 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package rbac + +import ( + "testing" + + rbacv1 "k8s.io/api/rbac/v1" + + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +func TestPromotionSpec_ToAdditionalRolebindings(t *testing.T) { + tests := []struct { + name string + promotion PromotionSpec + expected []AdditionalRoleBindingsWithNamespaceSpec + }{ + { + name: "creates rolebindings for every target and clusterrole", + promotion: PromotionSpec{ + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source-ns:gitops", + }, + ClusterRoles: []string{"configmap-replicator", "secret-replicator"}, + Targets: []string{"target-a", "target-b"}, + }, + expected: []AdditionalRoleBindingsWithNamespaceSpec{ + { + Namespace: meta.RFC1123SubdomainName("target-a"), + AdditionalRoleBindingsSpec: AdditionalRoleBindingsSpec{ + ClusterRoleName: "configmap-replicator", + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.ServiceAccountKind, + Name: "gitops", + Namespace: "source-ns", + }, + }, + }, + }, + { + Namespace: meta.RFC1123SubdomainName("target-a"), + AdditionalRoleBindingsSpec: AdditionalRoleBindingsSpec{ + ClusterRoleName: "secret-replicator", + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.ServiceAccountKind, + Name: "gitops", + Namespace: "source-ns", + }, + }, + }, + }, + { + Namespace: meta.RFC1123SubdomainName("target-b"), + AdditionalRoleBindingsSpec: AdditionalRoleBindingsSpec{ + ClusterRoleName: "configmap-replicator", + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.ServiceAccountKind, + Name: "gitops", + Namespace: "source-ns", + }, + }, + }, + }, + { + Namespace: meta.RFC1123SubdomainName("target-b"), + AdditionalRoleBindingsSpec: AdditionalRoleBindingsSpec{ + ClusterRoleName: "secret-replicator", + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.ServiceAccountKind, + Name: "gitops", + Namespace: "source-ns", + }, + }, + }, + }, + }, + }, + { + name: "returns empty list without targets", + promotion: PromotionSpec{ + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source-ns:gitops", + }, + ClusterRoles: []string{"configmap-replicator"}, + }, + expected: []AdditionalRoleBindingsWithNamespaceSpec{}, + }, + { + name: "returns empty list without clusterroles", + promotion: PromotionSpec{ + UserSpec: UserSpec{ + Kind: ServiceAccountOwner, + Name: "system:serviceaccount:source-ns:gitops", + }, + Targets: []string{"target-a"}, + }, + expected: []AdditionalRoleBindingsWithNamespaceSpec{}, + }, + { + name: "creates user subject", + promotion: PromotionSpec{ + UserSpec: UserSpec{ + Kind: UserOwner, + Name: "alice", + }, + ClusterRoles: []string{"view"}, + Targets: []string{"target-a"}, + }, + expected: []AdditionalRoleBindingsWithNamespaceSpec{ + { + Namespace: meta.RFC1123SubdomainName("target-a"), + AdditionalRoleBindingsSpec: AdditionalRoleBindingsSpec{ + ClusterRoleName: "view", + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.UserKind, + Name: "alice", + APIGroup: rbacv1.GroupName, + }, + }, + }, + }, + }, + }, + { + name: "creates group subject", + promotion: PromotionSpec{ + UserSpec: UserSpec{ + Kind: GroupOwner, + Name: "developers", + }, + ClusterRoles: []string{"edit"}, + Targets: []string{"target-a"}, + }, + expected: []AdditionalRoleBindingsWithNamespaceSpec{ + { + Namespace: meta.RFC1123SubdomainName("target-a"), + AdditionalRoleBindingsSpec: AdditionalRoleBindingsSpec{ + ClusterRoleName: "edit", + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.GroupKind, + Name: "developers", + APIGroup: rbacv1.GroupName, + }, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.promotion.ToAdditionalRolebindings() + + if !equalAdditionalRoleBindingsWithNamespace(got, tt.expected) { + t.Fatalf("unexpected rolebindings\nexpected: %#v\ngot: %#v", tt.expected, got) + } + }) + } +} + +func equalAdditionalRoleBindingsWithNamespace(a, b []AdditionalRoleBindingsWithNamespaceSpec) bool { + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i].Namespace != b[i].Namespace { + return false + } + + if a[i].ClusterRoleName != b[i].ClusterRoleName { + return false + } + + if !equalSubjects(a[i].Subjects, b[i].Subjects) { + return false + } + } + + return true +} + +func equalSubjects(a, b []rbacv1.Subject) bool { + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i].Kind != b[i].Kind { + return false + } + + if a[i].Name != b[i].Name { + return false + } + + if a[i].Namespace != b[i].Namespace { + return false + } + + if a[i].APIGroup != b[i].APIGroup { + return false + } + } + + return true +} diff --git a/pkg/api/rbac/subject.go b/pkg/api/rbac/subject.go new file mode 100644 index 00000000..1863db91 --- /dev/null +++ b/pkg/api/rbac/subject.go @@ -0,0 +1,10 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package rbac + +type SubjectRoles struct { + Kind string + Name string + Roles []string +} diff --git a/pkg/api/users.go b/pkg/api/rbac/users.go similarity index 98% rename from pkg/api/users.go rename to pkg/api/rbac/users.go index 6ef76e05..88c6c5cc 100644 --- a/pkg/api/users.go +++ b/pkg/api/rbac/users.go @@ -1,7 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package api +package rbac import ( "strings" diff --git a/pkg/api/users_list.go b/pkg/api/rbac/users_list.go similarity index 77% rename from pkg/api/users_list.go rename to pkg/api/rbac/users_list.go index c9b5b9eb..8489b3b5 100644 --- a/pkg/api/users_list.go +++ b/pkg/api/rbac/users_list.go @@ -1,7 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package api +package rbac import ( "sort" @@ -37,7 +37,9 @@ func (o *UserListSpec) Upsert(newUser UserSpec) { } users = append(users, newUser) + sort.Sort(ByKindName(users)) + *o = users } @@ -92,6 +94,48 @@ func (o UserListSpec) FindUser(name string, kind OwnerKind) (UserSpec, bool) { return UserSpec{}, false } +func (o UserListSpec) SplitUsersAndGroups() (users []string, groups []string) { + seenU := make(map[string]struct{}, len(o)) + seenG := make(map[string]struct{}, len(o)) + + for _, s := range o { + if s.Name == "" { + continue + } + + switch s.Kind { + case UserOwner, ServiceAccountOwner: + seenU[s.Name] = struct{}{} + case GroupOwner: + seenG[s.Name] = struct{}{} + default: + continue + } + } + + if len(seenU) > 0 { + users = make([]string, 0, len(seenU)) + + for u := range seenU { + users = append(users, u) + } + + sort.Strings(users) + } + + if len(seenG) > 0 { + groups = make([]string, 0, len(seenG)) + + for g := range seenG { + groups = append(groups, g) + } + + sort.Strings(groups) + } + + return users, groups +} + func (o UserListSpec) GetByKinds(kinds []OwnerKind) []string { if len(o) == 0 || len(kinds) == 0 { return nil diff --git a/pkg/api/users_list_test.go b/pkg/api/rbac/users_list_test.go similarity index 54% rename from pkg/api/users_list_test.go rename to pkg/api/rbac/users_list_test.go index aaf0ac11..87eb540a 100644 --- a/pkg/api/users_list_test.go +++ b/pkg/api/rbac/users_list_test.go @@ -1,7 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package api_test +package rbac_test import ( "math/rand" @@ -10,26 +10,26 @@ import ( "testing" "time" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -func linearFindUser(list api.UserListSpec, name string, kind api.OwnerKind) (api.UserSpec, bool) { +func linearFindUser(list rbac.UserListSpec, name string, kind rbac.OwnerKind) (rbac.UserSpec, bool) { for _, u := range list { if u.Kind == kind && u.Name == name { return u, true } } - return api.UserSpec{}, false + return rbac.UserSpec{}, false } -func slowIsPresent(u api.UserListSpec, name string, groups []string) bool { +func slowIsPresent(u rbac.UserListSpec, name string, groups []string) bool { for _, user := range u { switch user.Kind { - case api.UserOwner, api.ServiceAccountOwner: + case rbac.UserOwner, rbac.ServiceAccountOwner: if name == user.Name { return true } - case api.GroupOwner: + case rbac.GroupOwner: for _, group := range groups { if group == user.Name { return true @@ -41,19 +41,19 @@ func slowIsPresent(u api.UserListSpec, name string, groups []string) bool { } func TestByKindNameOrdering_UserListSpec(t *testing.T) { - u := api.UserListSpec{ - api.UserSpec{Name: "b", Kind: api.ServiceAccountOwner}, - api.UserSpec{Name: "z", Kind: api.UserOwner}, - api.UserSpec{Name: "a", Kind: api.GroupOwner}, - api.UserSpec{Name: "a", Kind: api.UserOwner}, + u := rbac.UserListSpec{ + rbac.UserSpec{Name: "b", Kind: rbac.ServiceAccountOwner}, + rbac.UserSpec{Name: "z", Kind: rbac.UserOwner}, + rbac.UserSpec{Name: "a", Kind: rbac.GroupOwner}, + rbac.UserSpec{Name: "a", Kind: rbac.UserOwner}, } // Sort using production ordering - got := append(api.UserListSpec(nil), u...) - sort.Sort(api.ByKindName(got)) + got := append(rbac.UserListSpec(nil), u...) + sort.Sort(rbac.ByKindName(got)) // Manually sorted expectation using the same logic. - want := append(api.UserListSpec(nil), u...) + want := append(rbac.UserListSpec(nil), u...) sort.Slice(want, func(i, j int) bool { if want[i].Kind.String() != want[j].Kind.String() { return want[i].Kind.String() < want[j].Kind.String() @@ -74,10 +74,10 @@ func TestByKindNameOrdering_UserListSpec(t *testing.T) { func TestFindUser_Randomized(t *testing.T) { rnd := rand.New(rand.NewSource(42)) - ownerKinds := []api.OwnerKind{ - api.GroupOwner, - api.UserOwner, - api.ServiceAccountOwner, + ownerKinds := []rbac.OwnerKind{ + rbac.GroupOwner, + rbac.UserOwner, + rbac.ServiceAccountOwner, } const ( @@ -87,11 +87,11 @@ func TestFindUser_Randomized(t *testing.T) { ) for listIdx := 0; listIdx < numLists; listIdx++ { - var list api.UserListSpec + var list rbac.UserListSpec n := rnd.Intn(maxLength) for i := 0; i < n; i++ { k := ownerKinds[rnd.Intn(len(ownerKinds))] - list = append(list, api.UserSpec{ + list = append(list, rbac.UserSpec{ Name: randomName(rnd, 3+rnd.Intn(4)), // length 3–6 Kind: k, }) @@ -99,7 +99,7 @@ func TestFindUser_Randomized(t *testing.T) { for lookupIdx := 0; lookupIdx < numLookupsPerList; lookupIdx++ { var qName string - var qKind api.OwnerKind + var qKind rbac.OwnerKind if len(list) > 0 && rnd.Float64() < 0.6 { // 60% of lookups: pick a real element, must be found @@ -112,7 +112,7 @@ func TestFindUser_Randomized(t *testing.T) { qKind = ownerKinds[rnd.Intn(len(ownerKinds))] } - listCopy := append(api.UserListSpec(nil), list...) // FindUser sorts in-place + listCopy := append(rbac.UserListSpec(nil), list...) // FindUser sorts in-place gotUser, gotFound := listCopy.FindUser(qName, qKind) wantUser, wantFound := linearFindUser(list, qName, qKind) @@ -131,10 +131,10 @@ func TestFindUser_Randomized(t *testing.T) { func TestIsPresent_RandomizedMatchesSlowImplementation(t *testing.T) { rnd := rand.New(rand.NewSource(time.Now().UnixNano())) - ownerKinds := []api.OwnerKind{ - api.UserOwner, - api.GroupOwner, - api.ServiceAccountOwner, + ownerKinds := []rbac.OwnerKind{ + rbac.UserOwner, + rbac.GroupOwner, + rbac.ServiceAccountOwner, } const ( @@ -146,11 +146,11 @@ func TestIsPresent_RandomizedMatchesSlowImplementation(t *testing.T) { for listIdx := 0; listIdx < numLists; listIdx++ { // Generate a random user list (possibly with duplicates). - var users api.UserListSpec + var users rbac.UserListSpec nOwners := rnd.Intn(maxOwnersPerList) for i := 0; i < nOwners; i++ { kind := ownerKinds[rnd.Intn(len(ownerKinds))] - users = append(users, api.UserSpec{ + users = append(users, rbac.UserSpec{ Name: randomName(rnd, 3+rnd.Intn(4)), // length 3–6 Kind: kind, }) @@ -193,12 +193,12 @@ func TestIsPresent_RandomizedMatchesSlowImplementation(t *testing.T) { } func TestGetByKinds_Basic(t *testing.T) { - users := api.UserListSpec{ - api.UserSpec{Name: "alice", Kind: api.UserOwner}, - api.UserSpec{Name: "svc-1", Kind: api.ServiceAccountOwner}, - api.UserSpec{Name: "team-a", Kind: api.GroupOwner}, - api.UserSpec{Name: "bob", Kind: api.UserOwner}, - api.UserSpec{Name: "team-b", Kind: api.GroupOwner}, + users := rbac.UserListSpec{ + rbac.UserSpec{Name: "alice", Kind: rbac.UserOwner}, + rbac.UserSpec{Name: "svc-1", Kind: rbac.ServiceAccountOwner}, + rbac.UserSpec{Name: "team-a", Kind: rbac.GroupOwner}, + rbac.UserSpec{Name: "bob", Kind: rbac.UserOwner}, + rbac.UserSpec{Name: "team-b", Kind: rbac.GroupOwner}, } eqStrings := func(got, want []string) bool { @@ -219,21 +219,21 @@ func TestGetByKinds_Basic(t *testing.T) { } // Single kind: UserOwner - gotUsers := users.GetByKinds([]api.OwnerKind{api.UserOwner}) + gotUsers := users.GetByKinds([]rbac.OwnerKind{rbac.UserOwner}) wantUsers := []string{"alice", "bob"} if !eqStrings(gotUsers, wantUsers) { t.Fatalf("GetByKinds([UserOwner]) = %v, want %v", gotUsers, wantUsers) } // Single kind: GroupOwner - gotGroups := users.GetByKinds([]api.OwnerKind{api.GroupOwner}) + gotGroups := users.GetByKinds([]rbac.OwnerKind{rbac.GroupOwner}) wantGroups := []string{"team-a", "team-b"} if !eqStrings(gotGroups, wantGroups) { t.Fatalf("GetByKinds([GroupOwner]) = %v, want %v", gotGroups, wantGroups) } // Multiple kinds: UserOwner + ServiceAccountOwner - gotUsersAndSAs := users.GetByKinds([]api.OwnerKind{api.UserOwner, api.ServiceAccountOwner}) + gotUsersAndSAs := users.GetByKinds([]rbac.OwnerKind{rbac.UserOwner, rbac.ServiceAccountOwner}) wantUsersAndSAs := []string{"alice", "bob", "svc-1"} if !eqStrings(gotUsersAndSAs, wantUsersAndSAs) { t.Fatalf("GetByKinds([UserOwner,ServiceAccountOwner]) = %v, want %v", @@ -247,7 +247,7 @@ func TestGetByKinds_Basic(t *testing.T) { } // Kind not present at all - gotUnknown := users.GetByKinds([]api.OwnerKind{api.OwnerKind("does-not-exist")}) + gotUnknown := users.GetByKinds([]rbac.OwnerKind{rbac.OwnerKind("does-not-exist")}) if gotUnknown != nil { t.Fatalf("GetByKinds([unknown]) = %v, want nil", gotUnknown) } @@ -256,10 +256,10 @@ func TestGetByKinds_Basic(t *testing.T) { func TestGetByKinds_Randomized(t *testing.T) { rnd := rand.New(rand.NewSource(123)) - ownerKinds := []api.OwnerKind{ - api.UserOwner, - api.GroupOwner, - api.ServiceAccountOwner, + ownerKinds := []rbac.OwnerKind{ + rbac.UserOwner, + rbac.GroupOwner, + rbac.ServiceAccountOwner, } const ( @@ -268,11 +268,11 @@ func TestGetByKinds_Randomized(t *testing.T) { ) for listIdx := 0; listIdx < numLists; listIdx++ { - var users api.UserListSpec + var users rbac.UserListSpec n := rnd.Intn(maxOwnersPerList) for i := 0; i < n; i++ { k := ownerKinds[rnd.Intn(len(ownerKinds))] - users = append(users, api.UserSpec{ + users = append(users, rbac.UserSpec{ Name: randomName(rnd, 3+rnd.Intn(4)), // reuse your helper Kind: k, }) @@ -281,7 +281,7 @@ func TestGetByKinds_Randomized(t *testing.T) { // Try several random kind-subsets per list for subsetIdx := 0; subsetIdx < 10; subsetIdx++ { // Build a random subset of kinds - var kinds []api.OwnerKind + var kinds []rbac.OwnerKind for _, k := range ownerKinds { if rnd.Float64() < 0.5 { kinds = append(kinds, k) @@ -291,7 +291,7 @@ func TestGetByKinds_Randomized(t *testing.T) { got := users.GetByKinds(kinds) // Reference implementation: filter + sort - kindSet := make(map[api.OwnerKind]struct{}, len(kinds)) + kindSet := make(map[rbac.OwnerKind]struct{}, len(kinds)) for _, k := range kinds { kindSet[k] = struct{}{} } @@ -323,3 +323,139 @@ func TestGetByKinds_Randomized(t *testing.T) { } } } + +func TestUserListSpec_SplitUsersAndGroups(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in rbac.UserListSpec + wantUsers []string + wantGroups []string + }{ + { + name: "nil", + in: nil, + wantUsers: nil, + wantGroups: nil, + }, + { + name: "empty", + in: rbac.UserListSpec{}, + wantUsers: nil, + wantGroups: nil, + }, + { + name: "users_only_sorted_and_deduped", + in: rbac.UserListSpec{ + {Kind: rbac.UserOwner, Name: "zara"}, + {Kind: rbac.UserOwner, Name: "alice"}, + {Kind: rbac.UserOwner, Name: "alice"}, + {Kind: rbac.UserOwner, Name: "bob"}, + }, + wantUsers: []string{"alice", "bob", "zara"}, + wantGroups: nil, + }, + { + name: "groups_only_sorted_and_deduped", + in: rbac.UserListSpec{ + {Kind: rbac.GroupOwner, Name: "team-b"}, + {Kind: rbac.GroupOwner, Name: "team-a"}, + {Kind: rbac.GroupOwner, Name: "team-a"}, + }, + wantUsers: nil, + wantGroups: []string{"team-a", "team-b"}, + }, + { + name: "mix_users_groups_and_serviceaccounts", + in: rbac.UserListSpec{ + {Kind: rbac.GroupOwner, Name: "ops"}, + {Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:ns:sa"}, + {Kind: rbac.UserOwner, Name: "alice"}, + {Kind: rbac.GroupOwner, Name: "dev"}, + }, + wantUsers: []string{"alice", "system:serviceaccount:ns:sa"}, + wantGroups: []string{"dev", "ops"}, + }, + { + name: "ignore_empty_names", + in: rbac.UserListSpec{ + {Kind: rbac.UserOwner, Name: ""}, + {Kind: rbac.GroupOwner, Name: ""}, + {Kind: rbac.ServiceAccountOwner, Name: ""}, + {Kind: rbac.UserOwner, Name: "alice"}, + }, + wantUsers: []string{"alice"}, + wantGroups: nil, + }, + { + name: "all_kinds", + in: rbac.UserListSpec{ + {Kind: rbac.ServiceAccountOwner, Name: "x"}, + {Kind: rbac.UserOwner, Name: "alice"}, + {Kind: rbac.GroupOwner, Name: "dev"}, + }, + wantUsers: []string{"alice", "x"}, + wantGroups: []string{"dev"}, + }, + { + name: "same_name_in_user_and_group_goes_to_respective_buckets", + in: rbac.UserListSpec{ + {Kind: rbac.UserOwner, Name: "same"}, + {Kind: rbac.GroupOwner, Name: "same"}, + }, + wantUsers: []string{"same"}, + wantGroups: []string{"same"}, + }, + { + name: "deterministic_ordering_with_many_values", + in: rbac.UserListSpec{ + {Kind: rbac.UserOwner, Name: "c"}, + {Kind: rbac.UserOwner, Name: "a"}, + {Kind: rbac.UserOwner, Name: "b"}, + {Kind: rbac.GroupOwner, Name: "g2"}, + {Kind: rbac.GroupOwner, Name: "g1"}, + }, + wantUsers: []string{"a", "b", "c"}, + wantGroups: []string{"g1", "g2"}, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + gotUsers, gotGroups := tt.in.SplitUsersAndGroups() + + if !reflect.DeepEqual(gotUsers, tt.wantUsers) { + t.Fatalf("users = %#v, want %#v", gotUsers, tt.wantUsers) + } + if !reflect.DeepEqual(gotGroups, tt.wantGroups) { + t.Fatalf("groups = %#v, want %#v", gotGroups, tt.wantGroups) + } + }) + } +} + +func TestUserListSpec_SplitUsersAndGroups_Idempotent(t *testing.T) { + t.Parallel() + + in := rbac.UserListSpec{ + {Kind: rbac.GroupOwner, Name: "team-b"}, + {Kind: rbac.UserOwner, Name: "zara"}, + {Kind: rbac.UserOwner, Name: "alice"}, + {Kind: rbac.GroupOwner, Name: "team-a"}, + {Kind: rbac.UserOwner, Name: "alice"}, + } + + u1, g1 := in.SplitUsersAndGroups() + u2, g2 := in.SplitUsersAndGroups() + + if !reflect.DeepEqual(u1, u2) { + t.Fatalf("users not deterministic: first=%#v second=%#v", u1, u2) + } + if !reflect.DeepEqual(g1, g2) { + t.Fatalf("groups not deterministic: first=%#v second=%#v", g1, g2) + } +} diff --git a/pkg/api/users_test.go b/pkg/api/rbac/users_test.go similarity index 81% rename from pkg/api/users_test.go rename to pkg/api/rbac/users_test.go index a93a1d9c..8e5c0008 100644 --- a/pkg/api/users_test.go +++ b/pkg/api/rbac/users_test.go @@ -1,26 +1,26 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package api_test +package rbac_test import ( "testing" rbacv1 "k8s.io/api/rbac/v1" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) func TestUserSpec_Subject_ServiceAccount(t *testing.T) { tests := []struct { name string - in api.UserSpec + in rbac.UserSpec want rbacv1.Subject }{ { name: "system serviceaccount format", - in: api.UserSpec{ - Kind: api.ServiceAccountOwner, + in: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:capsule-system:capsule", }, want: rbacv1.Subject{ @@ -31,8 +31,8 @@ func TestUserSpec_Subject_ServiceAccount(t *testing.T) { }, { name: "minimal ns:name style (still splits from end)", - in: api.UserSpec{ - Kind: api.ServiceAccountOwner, + in: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "ns:sa", }, want: rbacv1.Subject{ @@ -43,8 +43,8 @@ func TestUserSpec_Subject_ServiceAccount(t *testing.T) { }, { name: "extra segments (uses last two)", - in: api.UserSpec{ - Kind: api.ServiceAccountOwner, + in: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: "a:b:c:d", }, want: rbacv1.Subject{ @@ -68,13 +68,13 @@ func TestUserSpec_Subject_ServiceAccount(t *testing.T) { func TestUserSpec_Subject_UserAndGroup(t *testing.T) { tests := []struct { name string - in api.UserSpec + in rbac.UserSpec want rbacv1.Subject }{ { name: "user subject", - in: api.UserSpec{ - Kind: api.UserOwner, + in: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "alice", }, want: rbacv1.Subject{ @@ -85,8 +85,8 @@ func TestUserSpec_Subject_UserAndGroup(t *testing.T) { }, { name: "group subject", - in: api.UserSpec{ - Kind: api.GroupOwner, + in: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "devops", }, want: rbacv1.Subject{ diff --git a/pkg/api/rbac/zz_generated.deepcopy.go b/pkg/api/rbac/zz_generated.deepcopy.go new file mode 100644 index 00000000..08b1e709 --- /dev/null +++ b/pkg/api/rbac/zz_generated.deepcopy.go @@ -0,0 +1,247 @@ +//go:build !ignore_autogenerated + +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by controller-gen. DO NOT EDIT. + +package rbac + +import ( + "k8s.io/api/rbac/v1" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AdditionalRoleBindingsSpec) DeepCopyInto(out *AdditionalRoleBindingsSpec) { + *out = *in + if in.Subjects != nil { + in, out := &in.Subjects, &out.Subjects + *out = make([]v1.Subject, len(*in)) + copy(*out, *in) + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalRoleBindingsSpec. +func (in *AdditionalRoleBindingsSpec) DeepCopy() *AdditionalRoleBindingsSpec { + if in == nil { + return nil + } + out := new(AdditionalRoleBindingsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoreOwnerSpec) DeepCopyInto(out *CoreOwnerSpec) { + *out = *in + out.UserSpec = in.UserSpec + if in.ClusterRoles != nil { + in, out := &in.ClusterRoles, &out.ClusterRoles + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoreOwnerSpec. +func (in *CoreOwnerSpec) DeepCopy() *CoreOwnerSpec { + if in == nil { + return nil + } + out := new(CoreOwnerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in OwnerListSpec) DeepCopyInto(out *OwnerListSpec) { + { + in := &in + *out = make(OwnerListSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OwnerListSpec. +func (in OwnerListSpec) DeepCopy() OwnerListSpec { + if in == nil { + return nil + } + out := new(OwnerListSpec) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OwnerSpec) DeepCopyInto(out *OwnerSpec) { + *out = *in + in.CoreOwnerSpec.DeepCopyInto(&out.CoreOwnerSpec) + if in.ProxyOperations != nil { + in, out := &in.ProxyOperations, &out.ProxyOperations + *out = make([]ProxySettings, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OwnerSpec. +func (in *OwnerSpec) DeepCopy() *OwnerSpec { + if in == nil { + return nil + } + out := new(OwnerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in OwnerStatusListSpec) DeepCopyInto(out *OwnerStatusListSpec) { + { + in := &in + *out = make(OwnerStatusListSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OwnerStatusListSpec. +func (in OwnerStatusListSpec) DeepCopy() OwnerStatusListSpec { + if in == nil { + return nil + } + out := new(OwnerStatusListSpec) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PromotionSpec) DeepCopyInto(out *PromotionSpec) { + *out = *in + out.UserSpec = in.UserSpec + if in.ClusterRoles != nil { + in, out := &in.ClusterRoles, &out.ClusterRoles + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Targets != nil { + in, out := &in.Targets, &out.Targets + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PromotionSpec. +func (in *PromotionSpec) DeepCopy() *PromotionSpec { + if in == nil { + return nil + } + out := new(PromotionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in PromotionStatusListSpec) DeepCopyInto(out *PromotionStatusListSpec) { + { + in := &in + *out = make(PromotionStatusListSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PromotionStatusListSpec. +func (in PromotionStatusListSpec) DeepCopy() PromotionStatusListSpec { + if in == nil { + return nil + } + out := new(PromotionStatusListSpec) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxySettings) DeepCopyInto(out *ProxySettings) { + *out = *in + if in.Operations != nil { + in, out := &in.Operations, &out.Operations + *out = make([]ProxyOperation, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxySettings. +func (in *ProxySettings) DeepCopy() *ProxySettings { + if in == nil { + return nil + } + out := new(ProxySettings) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in UserListSpec) DeepCopyInto(out *UserListSpec) { + { + in := &in + *out = make(UserListSpec, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserListSpec. +func (in UserListSpec) DeepCopy() UserListSpec { + if in == nil { + return nil + } + out := new(UserListSpec) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UserSpec) DeepCopyInto(out *UserSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserSpec. +func (in *UserSpec) DeepCopy() *UserSpec { + if in == nil { + return nil + } + out := new(UserSpec) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/api/scope.go b/pkg/api/scope.go new file mode 100644 index 00000000..432f895d --- /dev/null +++ b/pkg/api/scope.go @@ -0,0 +1,17 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package api + +const ( + ResourceScopeNamespace ResourceScope = "Namespace" + ResourceScopeTenant ResourceScope = "Tenant" + ResourceScopeNone ResourceScope = "None" +) + +// +kubebuilder:validation:Enum=Namespace;Tenant;None +type ResourceScope string + +func (p ResourceScope) String() string { + return string(p) +} diff --git a/pkg/api/tenant_roles.go b/pkg/api/tenant_roles.go deleted file mode 100644 index affae338..00000000 --- a/pkg/api/tenant_roles.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package api - -// Type to extract all clusterroles for a subject on a tenant -// from the owner and additionalRoleBindings spec. -type TenantSubjectRoles struct { - Kind string - Name string - ClusterRoles []string -} diff --git a/pkg/api/zz_generated.deepcopy.go b/pkg/api/zz_generated.deepcopy.go index 7b0fe74d..69764557 100644 --- a/pkg/api/zz_generated.deepcopy.go +++ b/pkg/api/zz_generated.deepcopy.go @@ -10,7 +10,6 @@ package api import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" - rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -77,40 +76,6 @@ func (in *AdditionalMetadataSpec) DeepCopy() *AdditionalMetadataSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AdditionalRoleBindingsSpec) DeepCopyInto(out *AdditionalRoleBindingsSpec) { - *out = *in - if in.Subjects != nil { - in, out := &in.Subjects, &out.Subjects - *out = make([]rbacv1.Subject, len(*in)) - copy(*out, *in) - } - if in.Labels != nil { - in, out := &in.Labels, &out.Labels - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.Annotations != nil { - in, out := &in.Annotations, &out.Annotations - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalRoleBindingsSpec. -func (in *AdditionalRoleBindingsSpec) DeepCopy() *AdditionalRoleBindingsSpec { - if in == nil { - return nil - } - out := new(AdditionalRoleBindingsSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AllowedListSpec) DeepCopyInto(out *AllowedListSpec) { *out = *in @@ -161,27 +126,6 @@ func (in *AllowedServices) DeepCopy() *AllowedServices { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CoreOwnerSpec) DeepCopyInto(out *CoreOwnerSpec) { - *out = *in - out.UserSpec = in.UserSpec - if in.ClusterRoles != nil { - in, out := &in.ClusterRoles, &out.ClusterRoles - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoreOwnerSpec. -func (in *CoreOwnerSpec) DeepCopy() *CoreOwnerSpec { - if in == nil { - return nil - } - out := new(CoreOwnerSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DefaultAllowedListSpec) DeepCopyInto(out *DefaultAllowedListSpec) { *out = *in @@ -260,6 +204,117 @@ func (in *LimitRangesSpec) DeepCopy() *LimitRangesSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NamespaceRuleBodyNamespace) DeepCopyInto(out *NamespaceRuleBodyNamespace) { + *out = *in + in.Enforce.DeepCopyInto(&out.Enforce) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceRuleBodyNamespace. +func (in *NamespaceRuleBodyNamespace) DeepCopy() *NamespaceRuleBodyNamespace { + if in == nil { + return nil + } + out := new(NamespaceRuleBodyNamespace) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NamespaceRuleBodyTenant) DeepCopyInto(out *NamespaceRuleBodyTenant) { + *out = *in + in.NamespaceRuleBodyNamespace.DeepCopyInto(&out.NamespaceRuleBodyNamespace) + if in.NamespaceSelector != nil { + in, out := &in.NamespaceSelector, &out.NamespaceSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + in.Permissions.DeepCopyInto(&out.Permissions) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceRuleBodyTenant. +func (in *NamespaceRuleBodyTenant) DeepCopy() *NamespaceRuleBodyTenant { + if in == nil { + return nil + } + out := new(NamespaceRuleBodyTenant) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NamespaceRuleEnforceBody) DeepCopyInto(out *NamespaceRuleEnforceBody) { + *out = *in + if in.Registries != nil { + in, out := &in.Registries, &out.Registries + *out = make([]OCIRegistry, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceRuleEnforceBody. +func (in *NamespaceRuleEnforceBody) DeepCopy() *NamespaceRuleEnforceBody { + if in == nil { + return nil + } + out := new(NamespaceRuleEnforceBody) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NamespaceRulePermissionBody) DeepCopyInto(out *NamespaceRulePermissionBody) { + *out = *in + if in.Promotions != nil { + in, out := &in.Promotions, &out.Promotions + *out = make([]*NamespaceRulePromotionRule, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(NamespaceRulePromotionRule) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceRulePermissionBody. +func (in *NamespaceRulePermissionBody) DeepCopy() *NamespaceRulePermissionBody { + if in == nil { + return nil + } + out := new(NamespaceRulePermissionBody) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NamespaceRulePromotionRule) DeepCopyInto(out *NamespaceRulePromotionRule) { + *out = *in + if in.ClusterRoles != nil { + in, out := &in.ClusterRoles, &out.ClusterRoles + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceRulePromotionRule. +func (in *NamespaceRulePromotionRule) DeepCopy() *NamespaceRulePromotionRule { + if in == nil { + return nil + } + out := new(NamespaceRulePromotionRule) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkPolicySpec) DeepCopyInto(out *NetworkPolicySpec) { *out = *in @@ -307,85 +362,6 @@ func (in *OCIRegistry) DeepCopy() *OCIRegistry { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in OwnerListSpec) DeepCopyInto(out *OwnerListSpec) { - { - in := &in - *out = make(OwnerListSpec, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OwnerListSpec. -func (in OwnerListSpec) DeepCopy() OwnerListSpec { - if in == nil { - return nil - } - out := new(OwnerListSpec) - in.DeepCopyInto(out) - return *out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OwnerSpec) DeepCopyInto(out *OwnerSpec) { - *out = *in - in.CoreOwnerSpec.DeepCopyInto(&out.CoreOwnerSpec) - if in.ProxyOperations != nil { - in, out := &in.ProxyOperations, &out.ProxyOperations - *out = make([]ProxySettings, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Labels != nil { - in, out := &in.Labels, &out.Labels - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.Annotations != nil { - in, out := &in.Annotations, &out.Annotations - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OwnerSpec. -func (in *OwnerSpec) DeepCopy() *OwnerSpec { - if in == nil { - return nil - } - out := new(OwnerSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in OwnerStatusListSpec) DeepCopyInto(out *OwnerStatusListSpec) { - { - in := &in - *out = make(OwnerStatusListSpec, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OwnerStatusListSpec. -func (in OwnerStatusListSpec) DeepCopy() OwnerStatusListSpec { - if in == nil { - return nil - } - out := new(OwnerStatusListSpec) - in.DeepCopyInto(out) - return *out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodOptions) DeepCopyInto(out *PodOptions) { *out = *in @@ -423,26 +399,6 @@ func (in *PoolExhaustionResource) DeepCopy() *PoolExhaustionResource { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ProxySettings) DeepCopyInto(out *ProxySettings) { - *out = *in - if in.Operations != nil { - in, out := &in.Operations, &out.Operations - *out = make([]ProxyOperation, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxySettings. -func (in *ProxySettings) DeepCopy() *ProxySettings { - if in == nil { - return nil - } - out := new(ProxySettings) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceQuotaSpec) DeepCopyInto(out *ResourceQuotaSpec) { *out = *in @@ -545,37 +501,3 @@ func (in *ServiceOptions) DeepCopy() *ServiceOptions { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in UserListSpec) DeepCopyInto(out *UserListSpec) { - { - in := &in - *out = make(UserListSpec, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserListSpec. -func (in UserListSpec) DeepCopy() UserListSpec { - if in == nil { - return nil - } - out := new(UserListSpec) - in.DeepCopyInto(out) - return *out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *UserSpec) DeepCopyInto(out *UserSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserSpec. -func (in *UserSpec) DeepCopy() *UserSpec { - if in == nil { - return nil - } - out := new(UserSpec) - in.DeepCopyInto(out) - return out -} diff --git a/pkg/runtime/admission/accumulation.go b/pkg/runtime/admission/accumulation.go new file mode 100644 index 00000000..c8bc6dca --- /dev/null +++ b/pkg/runtime/admission/accumulation.go @@ -0,0 +1,29 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package admission + +import ( + "gomodules.xyz/jsonpatch/v2" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func AccumulateAdmissionResponse( + accumulated []jsonpatch.JsonPatchOperation, + response *admission.Response, +) ([]jsonpatch.JsonPatchOperation, *admission.Response) { + if response == nil { + return accumulated, nil + } + + // Denied or errored responses must stop immediately. + if !response.Allowed { + return accumulated, response + } + + if len(response.Patches) > 0 { + accumulated = append(accumulated, response.Patches...) + } + + return accumulated, nil +} diff --git a/pkg/runtime/admission/conditions.go b/pkg/runtime/admission/conditions.go new file mode 100644 index 00000000..9d2910be --- /dev/null +++ b/pkg/runtime/admission/conditions.go @@ -0,0 +1,85 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package admission + +import ( + "fmt" + "strings" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + "k8s.io/apiserver/pkg/authentication/serviceaccount" + + "github.com/projectcapsule/capsule/pkg/api/rbac" +) + +const ( + falseValue string = "false" +) + +func BuildGatingUserCondition(opts WebhookOptions, users rbac.UserListSpec, admins rbac.UserListSpec) []admissionregistrationv1.MatchCondition { + var parts []string + + if opts.CapsuleUsers { + parts = append(parts, ServiceAccountGroupGuardExpr()) + parts = append(parts, CelUserOrGroupExpr(users)) + } + + if opts.Administrators { + parts = append(parts, CelUserOrGroupExpr(admins)) + } + + if len(parts) == 0 { + return nil + } + + expr := parts[0] + for i := 1; i < len(parts); i++ { + expr = fmt.Sprintf("(%s) || (%s)", expr, parts[i]) + } + + return []admissionregistrationv1.MatchCondition{ + { + Name: "capsule-user-gate", + Expression: expr, + }, + } +} + +func ServiceAccountGroupGuardExpr() string { + return fmt.Sprintf("request.userInfo.groups.exists(g, g == %s)", CelQuote(serviceaccount.AllServiceAccountsGroup)) +} + +func CelUserOrGroupExpr(l rbac.UserListSpec) string { + users, groups := l.SplitUsersAndGroups() + + userExpr := falseValue + if len(users) > 0 { + userExpr = fmt.Sprintf("request.userInfo.username in %s", CelStringList(users)) + } + + groupExpr := falseValue + if len(groups) > 0 { + groupExpr = fmt.Sprintf("request.userInfo.groups.exists(g, g in %s)", CelStringList(groups)) + } + + return fmt.Sprintf("(%s) || (%s)", userExpr, groupExpr) +} + +func CelStringList(items []string) string { + q := make([]string, 0, len(items)) + for _, it := range items { + q = append(q, CelQuote(it)) + } + + return "[" + strings.Join(q, ",") + "]" +} + +func CelQuote(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `'`, `\'`) + s = strings.ReplaceAll(s, "\n", `\n`) + s = strings.ReplaceAll(s, "\t", `\t`) + + return "'" + s + "'" +} diff --git a/pkg/runtime/admission/conditions_test.go b/pkg/runtime/admission/conditions_test.go new file mode 100644 index 00000000..a4559e54 --- /dev/null +++ b/pkg/runtime/admission/conditions_test.go @@ -0,0 +1,225 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package admission_test + +import ( + "testing" + + "github.com/projectcapsule/capsule/pkg/api/rbac" + "github.com/projectcapsule/capsule/pkg/runtime/admission" +) + +const serviceAccountGuard = "request.userInfo.groups.exists(g, g == 'system:serviceaccounts')" + +func TestCelQuote(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + want string + }{ + {"empty", "", "''"}, + {"plain", "abc", "'abc'"}, + {"single_quote", "a'b", "'a\\'b'"}, + {"backslash", `a\b`, "'a\\\\b'"}, + {"newline", "a\nb", "'a\\nb'"}, + {"tab", "a\tb", "'a\\tb'"}, + {"combo", "a\\b'c\nd\te", "'a\\\\b\\'c\\nd\\te'"}, + // Ensure already-escaped sequences are not double-escaped in an unexpected way: + {"literal_backslash_n", `a\nb`, "'a\\\\nb'"}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := admission.CelQuote(tt.in); got != tt.want { + t.Fatalf("CelQuote(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +func TestCelStringList(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in []string + want string + }{ + {"nil", nil, "[]"}, + {"empty", []string{}, "[]"}, + {"single", []string{"alice"}, "['alice']"}, + {"two", []string{"alice", "bob"}, "['alice','bob']"}, + {"needs_escape", []string{`a\b`, "x'y", "n\n", "t\t"}, "['a\\\\b','x\\'y','n\\n','t\\t']"}, + {"preserve_order", []string{"b", "a"}, "['b','a']"}, // caller controls sorting + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := admission.CelStringList(tt.in); got != tt.want { + t.Fatalf("CelStringList(%v) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +func TestCelUserOrGroupExpr(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in rbac.UserListSpec + want string + }{ + { + name: "empty_list", + in: nil, + want: "(false) || (false)", + }, + { + name: "users_only", + in: rbac.UserListSpec{ + {Kind: rbac.UserOwner, Name: "alice"}, + {Kind: rbac.UserOwner, Name: "bob"}, + }, + // NOTE: SplitUsersAndGroups sorts => alice,bob + want: "(request.userInfo.username in ['alice','bob']) || (false)", + }, + { + name: "groups_only", + in: rbac.UserListSpec{ + {Kind: rbac.GroupOwner, Name: "dev"}, + {Kind: rbac.GroupOwner, Name: "ops"}, + }, + // NOTE: sorted dev,ops + want: "(false) || (request.userInfo.groups.exists(g, g in ['dev','ops']))", + }, + { + name: "users_and_groups", + in: rbac.UserListSpec{ + {Kind: rbac.GroupOwner, Name: "team-b"}, + {Kind: rbac.UserOwner, Name: "zara"}, + {Kind: rbac.UserOwner, Name: "alice"}, + {Kind: rbac.GroupOwner, Name: "team-a"}, + }, + want: "(request.userInfo.username in ['alice','zara']) || (request.userInfo.groups.exists(g, g in ['team-a','team-b']))", + }, + { + name: "serviceaccounts_are_users_bucket", + in: rbac.UserListSpec{ + {Kind: rbac.ServiceAccountOwner, Name: "system:serviceaccount:ns:sa"}, + }, + want: "(request.userInfo.username in ['system:serviceaccount:ns:sa']) || (false)", + }, + { + name: "escaping_in_names", + in: rbac.UserListSpec{ + {Kind: rbac.UserOwner, Name: "a\\b"}, + {Kind: rbac.GroupOwner, Name: "x'y"}, + }, + // users sorted: a\b ; groups: x'y + want: "(request.userInfo.username in ['a\\\\b']) || (request.userInfo.groups.exists(g, g in ['x\\'y']))", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := admission.CelUserOrGroupExpr(tt.in); got != tt.want { + t.Fatalf("CelUserOrGroupExpr() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestBuildGatingUserCondition(t *testing.T) { + t.Parallel() + + users := rbac.UserListSpec{ + {Kind: rbac.UserOwner, Name: "alice"}, + {Kind: rbac.GroupOwner, Name: "projectcapsule.dev"}, + } + admins := rbac.UserListSpec{ + {Kind: rbac.UserOwner, Name: "root"}, + {Kind: rbac.GroupOwner, Name: "system:masters"}, + } + + tests := []struct { + name string + opts admission.WebhookOptions + wantNil bool + wantExpr string + }{ + { + name: "no_options_enabled_returns_nil", + opts: admission.WebhookOptions{}, + wantNil: true, + }, + { + name: "capsule_users_only", + opts: admission.WebhookOptions{CapsuleUsers: true}, + wantExpr: "(" + serviceAccountGuard + ") || " + + "((request.userInfo.username in ['alice']) || (request.userInfo.groups.exists(g, g in ['projectcapsule.dev'])))", + }, + { + name: "administrators_only", + opts: admission.WebhookOptions{Administrators: true}, + wantExpr: "(request.userInfo.username in ['root']) || (request.userInfo.groups.exists(g, g in ['system:masters']))", + }, + { + name: "both_enabled_or_combined", + opts: admission.WebhookOptions{CapsuleUsers: true, Administrators: true}, + wantExpr: "((" + serviceAccountGuard + ") || " + + "((request.userInfo.username in ['alice']) || (request.userInfo.groups.exists(g, g in ['projectcapsule.dev'])))) || " + + "((request.userInfo.username in ['root']) || (request.userInfo.groups.exists(g, g in ['system:masters'])))", + }, + { + name: "capsule_users_enabled_but_empty_list", + opts: admission.WebhookOptions{CapsuleUsers: true}, + wantExpr: "(" + serviceAccountGuard + ") || ((false) || (false))", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + u := users + a := admins + if tt.name == "capsule_users_enabled_but_empty_list" { + u = nil + a = nil + } + + got := admission.BuildGatingUserCondition(tt.opts, u, a) + + if tt.wantNil { + if got != nil { + t.Fatalf("expected nil, got %#v", got) + } + return + } + + if got == nil || len(got) != 1 { + t.Fatalf("expected exactly 1 MatchCondition, got %#v", got) + } + if got[0].Name != "capsule-user-gate" { + t.Fatalf("MatchCondition.Name = %q, want %q", got[0].Name, "capsule-user-gate") + } + if got[0].Expression != tt.wantExpr { + t.Fatalf("MatchCondition.Expression = %q, want %q", got[0].Expression, tt.wantExpr) + } + }) + } +} diff --git a/pkg/runtime/admission/dynamic.go b/pkg/runtime/admission/dynamic.go new file mode 100644 index 00000000..9ada4836 --- /dev/null +++ b/pkg/runtime/admission/dynamic.go @@ -0,0 +1,89 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package admission + +import ( + "strings" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +// +kubebuilder:object:generate=true +type DynamicAdmissionConfig struct { + // Name the Admission Webhook + Name meta.RFC1123Name `json:"name,omitempty"` + // Labels added to the Admission Webhook + // +optional + Labels map[string]string `json:"labels,omitempty"` + // Annotations added to the Admission Webhook + // +optional + Annotations map[string]string `json:"annotations,omitempty"` + // whats the problem + Client *admissionregistrationv1.WebhookClientConfig `json:"client"` +} + +func DynamicWebhookURL(baseURL *string, webhookPath string) *string { + cleanPath := normalizePath(webhookPath) + if cleanPath == "" { + if baseURL == nil || *baseURL == "" { + return nil + } + + u := *baseURL + + return &u + } + + if baseURL == nil || *baseURL == "" { + u := cleanPath + + return &u + } + + base := strings.TrimRight(*baseURL, "/") + + if base == strings.TrimRight(cleanPath, "/") { + u := cleanPath + + return &u + } + + if strings.HasSuffix(base, cleanPath) { + u := base + + return &u + } + + u := base + cleanPath + + return &u +} + +func DynamicClientWithPath( + in admissionregistrationv1.WebhookClientConfig, + webhookPath string, +) admissionregistrationv1.WebhookClientConfig { + out := in + + if out.URL != nil { + out.URL = DynamicWebhookURL(out.URL, webhookPath) + + return out + } + + cleanPath := normalizePath(webhookPath) + if cleanPath == "" { + return out + } + + if out.Service != nil { + svc := *out.Service + svc.Path = &cleanPath + out.Service = &svc + } + + return out +} diff --git a/pkg/runtime/admission/dynamic_test.go b/pkg/runtime/admission/dynamic_test.go new file mode 100644 index 00000000..3426107a --- /dev/null +++ b/pkg/runtime/admission/dynamic_test.go @@ -0,0 +1,237 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package admission_test + +import ( + "bytes" + "testing" + + "github.com/projectcapsule/capsule/pkg/runtime/admission" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" +) + +func TestDynamicClientWithPath_EmptyPath_NoChange(t *testing.T) { + t.Parallel() + + origURL := "https://example.com" + in := admissionregistrationv1.WebhookClientConfig{ + URL: &origURL, + CABundle: []byte("ca"), + } + + out := admission.DynamicClientWithPath(in, "") + + if out.URL == nil || *out.URL != origURL { + t.Fatalf("URL changed unexpectedly: got=%v want=%q", ptrStr(out.URL), origURL) + } + if out.Service != nil { + t.Fatalf("Service changed unexpectedly: got=%#v want=nil", out.Service) + } + if !bytes.Equal(out.CABundle, in.CABundle) { + t.Fatalf("CABundle changed unexpectedly: got=%q want=%q", out.CABundle, in.CABundle) + } +} + +func TestDynamicClientWithPath_URLMode_AppendsNormalizedPath(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + base string + path string + want string + }{ + {"no_trailing_slash_path_with_leading", "https://example.com", "/validate", "https://example.com/validate"}, + {"no_trailing_slash_path_without_leading", "https://example.com", "validate", "https://example.com/validate"}, + {"trailing_slash_base", "https://example.com/", "/validate", "https://example.com/validate"}, + {"multiple_slashes_path", "https://example.com/", "///a//b/", "https://example.com/a/b"}, + {"root_path", "https://example.com/", "/", "https://example.com/"}, + {"only_slashes_path", "https://example.com", "////", "https://example.com/"}, + {"dot_segments", "https://example.com", "/a/./b/../c", "https://example.com/a/c"}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + inURL := tt.base + in := admissionregistrationv1.WebhookClientConfig{ + URL: &inURL, + } + + out := admission.DynamicClientWithPath(in, tt.path) + + if out.URL == nil { + t.Fatalf("URL is nil, want %q", tt.want) + } + if *out.URL != tt.want { + t.Fatalf("URL = %q, want %q", *out.URL, tt.want) + } + }) + } +} + +func TestDynamicClientWithPath_URLMode_DoesNotMutateInput(t *testing.T) { + t.Parallel() + + orig := "https://example.com/" + in := admissionregistrationv1.WebhookClientConfig{URL: &orig} + + out := admission.DynamicClientWithPath(in, "/validate") + + // output should differ + if out.URL == nil || *out.URL != "https://example.com/validate" { + t.Fatalf("unexpected output URL: got=%v", ptrStr(out.URL)) + } + + // input must remain unchanged + if in.URL == nil || *in.URL != "https://example.com/" { + t.Fatalf("input was mutated: in.URL=%v", ptrStr(in.URL)) + } +} + +func TestDynamicClientWithPath_ServiceMode_SetsServicePath(t *testing.T) { + t.Parallel() + + in := admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Namespace: "ns", + Name: "svc", + }, + } + + out := admission.DynamicClientWithPath(in, "validate") + + if out.Service == nil || out.Service.Path == nil { + t.Fatalf("Service/Path not set: %#v", out.Service) + } + if *out.Service.Path != "/validate" { + t.Fatalf("Service.Path = %q, want %q", *out.Service.Path, "/validate") + } + + // other fields preserved + if out.Service.Namespace != "ns" || out.Service.Name != "svc" { + t.Fatalf("Service fields changed unexpectedly: %#v", out.Service) + } +} + +func TestDynamicClientWithPath_ServiceMode_DoesNotMutateInputService(t *testing.T) { + t.Parallel() + + in := admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Namespace: "ns", + Name: "svc", + }, + } + + // capture pointer identity + origSvcPtr := in.Service + + out := admission.DynamicClientWithPath(in, "/validate") + + if out.Service == nil || out.Service.Path == nil || *out.Service.Path != "/validate" { + t.Fatalf("unexpected out.Service: %#v", out.Service) + } + + // Must not mutate input's service struct + if in.Service == nil { + t.Fatalf("input service became nil") + } + if in.Service.Path != nil { + t.Fatalf("input service was mutated; expected Path=nil, got=%q", *in.Service.Path) + } + + // Additionally, output must not reuse the same *ServiceReference pointer + // (since we copy it to avoid aliasing). + if out.Service == origSvcPtr { + t.Fatalf("output service pointer aliases input; expected copy") + } +} + +func TestDynamicClientWithPath_URLTakesPrecedenceOverService(t *testing.T) { + t.Parallel() + + origURL := "https://example.com/" + in := admissionregistrationv1.WebhookClientConfig{ + URL: &origURL, + Service: &admissionregistrationv1.ServiceReference{ + Namespace: "ns", + Name: "svc", + }, + } + + out := admission.DynamicClientWithPath(in, "/validate") + + // URL mode should win + if out.URL == nil || *out.URL != "https://example.com/validate" { + t.Fatalf("URL = %v, want %q", ptrStr(out.URL), "https://example.com/validate") + } + + // Ensure we did not set Service.Path in output when URL used (function returns early) + // Service itself is shallow-copied, so it will be the same pointer as input here. + // Importantly: it must not have been mutated. + if out.Service == nil { + t.Fatalf("Service unexpectedly nil; function should keep it as-is in URL mode") + } + if out.Service.Path != nil { + t.Fatalf("Service.Path was unexpectedly set in URL mode: %q", *out.Service.Path) + } +} + +func TestDynamicClientWithPath_NoURLNoService_NoPanicNoChange(t *testing.T) { + t.Parallel() + + in := admissionregistrationv1.WebhookClientConfig{} + out := admission.DynamicClientWithPath(in, "/validate") + + // No URL/Service to apply to => should be unchanged other than shallow copy. + if out.URL != nil { + t.Fatalf("expected URL nil, got %v", ptrStr(out.URL)) + } + if out.Service != nil { + t.Fatalf("expected Service nil, got %#v", out.Service) + } +} + +func TestDynamicClientWithPath_PreservesCABundle(t *testing.T) { + t.Parallel() + + origURL := "https://example.com" + in := admissionregistrationv1.WebhookClientConfig{ + URL: &origURL, + CABundle: []byte("my-ca"), + } + + out := admission.DynamicClientWithPath(in, "/validate") + + if !bytes.Equal(out.CABundle, []byte("my-ca")) { + t.Fatalf("CABundle not preserved: got=%q want=%q", out.CABundle, "my-ca") + } +} + +func TestDynamicClientWithPath_Idempotent(t *testing.T) { + t.Parallel() + + origURL := "https://example.com/" + in := admissionregistrationv1.WebhookClientConfig{URL: &origURL} + + first := admission.DynamicClientWithPath(in, "/a//b/") + second := admission.DynamicClientWithPath(first, "/a//b/") + + if first.URL == nil || second.URL == nil { + t.Fatalf("unexpected nil URL: first=%v second=%v", ptrStr(first.URL), ptrStr(second.URL)) + } + if *first.URL != *second.URL { + t.Fatalf("not idempotent: first=%q second=%q", *first.URL, *second.URL) + } +} + +func ptrStr(p *string) string { + if p == nil { + return "" + } + return *p +} diff --git a/pkg/runtime/admission/error.go b/pkg/runtime/admission/error.go new file mode 100644 index 00000000..daf1b7ce --- /dev/null +++ b/pkg/runtime/admission/error.go @@ -0,0 +1,16 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package admission + +import ( + "net/http" + + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func ErroredResponse(err error) *admission.Response { + response := admission.Errored(http.StatusInternalServerError, err) + + return &response +} diff --git a/pkg/runtime/admission/utils.go b/pkg/runtime/admission/utils.go new file mode 100644 index 00000000..7dfe0ca8 --- /dev/null +++ b/pkg/runtime/admission/utils.go @@ -0,0 +1,26 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 +package admission + +import ( + "path" + "strings" + + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func Deny(message string) *admission.Response { + response := admission.Denied(message) + + return &response +} + +func normalizePath(p string) string { + if p == "" { + return "" + } + + p = "/" + strings.TrimLeft(p, "/") + + return path.Clean(p) +} diff --git a/pkg/runtime/admission/utils_test.go b/pkg/runtime/admission/utils_test.go new file mode 100644 index 00000000..a9fd596b --- /dev/null +++ b/pkg/runtime/admission/utils_test.go @@ -0,0 +1,84 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package admission + +import "testing" + +func TestNormalizePath(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + want string + }{ + { + name: "empty", + in: "", + want: "", + }, + { + name: "simple_without_slash", + in: "validate", + want: "/validate", + }, + { + name: "already_has_slash", + in: "/validate", + want: "/validate", + }, + { + name: "multiple_leading_slashes", + in: "///validate", + want: "/validate", + }, + { + name: "trailing_slash", + in: "/validate/", + want: "/validate", + }, + { + name: "double_slashes_inside", + in: "/foo//bar", + want: "/foo/bar", + }, + { + name: "relative_dot_segment", + in: "/foo/./bar", + want: "/foo/bar", + }, + { + name: "parent_segment", + in: "/foo/bar/../baz", + want: "/foo/baz", + }, + { + name: "complex_mix", + in: "///foo//bar/../baz/", + want: "/foo/baz", + }, + { + name: "only_slashes", + in: "////", + want: "/", + }, + { + name: "root", + in: "/", + want: "/", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := normalizePath(tt.in) + if got != tt.want { + t.Fatalf("normalizePath(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} diff --git a/pkg/runtime/admission/webhook.go b/pkg/runtime/admission/webhook.go new file mode 100644 index 00000000..661b8592 --- /dev/null +++ b/pkg/runtime/admission/webhook.go @@ -0,0 +1,419 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package admission + +import ( + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/projectcapsule/capsule/pkg/api/rbac" +) + +// +kubebuilder:object:generate=true +type WebhookOptions struct { + // If enabled, the request is only sent to admission if the user is mentioned + // As Part of the Capsule Users + // +kubebuilder:default=false + CapsuleUsers bool `json:"capsuleUsers"` + + // If enabled, the request is only sent to admission if the user is mentioned + // As Part of the Capsule Administrators + // +kubebuilder:default=false + Administrators bool `json:"administrators"` +} + +func NewValidatingWebhook(in *ValidatingWebhook, c *admissionregistrationv1.WebhookClientConfig, users rbac.UserListSpec, admins rbac.UserListSpec) (admissionregistrationv1.ValidatingWebhook, error) { + out := admissionregistrationv1.ValidatingWebhook{ + Name: in.Name, + Rules: in.Rules, + FailurePolicy: in.FailurePolicy, + MatchPolicy: in.MatchPolicy, + NamespaceSelector: in.NamespaceSelector, + ObjectSelector: in.ObjectSelector, + SideEffects: in.SideEffects, + TimeoutSeconds: in.TimeoutSeconds, + AdmissionReviewVersions: in.AdmissionReviewVersions, + } + + webhookPath := "" + if in.Path != nil { + webhookPath = *in.Path + } + + out.ClientConfig = DynamicClientWithPath(*c, webhookPath) + + if len(in.MatchConditions) > 0 { + out.MatchConditions = append([]admissionregistrationv1.MatchCondition(nil), in.MatchConditions...) + + return out, nil + } + + conds := BuildGatingUserCondition(in.Options, users, admins) + if len(conds) > 0 { + out.MatchConditions = conds + } + + return out, nil +} + +// +kubebuilder:object:generate=true +type ValidatingWebhook struct { + // The name of the admission webhook. + // Name should be fully qualified, e.g., imagepolicy.kubernetes.io, where + // "imagepolicy" is the name of the webhook, and kubernetes.io is the name + // of the organization. + // Required. + Name string `json:"name" protobuf:"bytes,1,opt,name=name"` + + // `path` is the URL path which will be sent in any request to + // this service. + Path *string `json:"path" protobuf:"bytes,3,opt,name=path"` + + // Capsule Custom Admission Options + // +optional + Options WebhookOptions `json:"opts"` + + // Rules describes what operations on what resources/subresources the webhook cares about. + // The webhook cares about an operation if it matches _any_ Rule. + // However, in order to prevent ValidatingAdmissionWebhooks and MutatingAdmissionWebhooks + // from putting the cluster in a state which cannot be recovered from without completely + // disabling the plugin, ValidatingAdmissionWebhooks and MutatingAdmissionWebhooks are never called + // on admission requests for ValidatingWebhookConfiguration and MutatingWebhookConfiguration objects. + // +listType=atomic + Rules []admissionregistrationv1.RuleWithOperations `json:"rules,omitempty" protobuf:"bytes,3,rep,name=rules"` + + // FailurePolicy defines how unrecognized errors from the admission endpoint are handled - + // allowed values are Ignore or Fail. Defaults to Fail. + // +optional + FailurePolicy *admissionregistrationv1.FailurePolicyType `json:"failurePolicy,omitempty" protobuf:"bytes,4,opt,name=failurePolicy,casttype=FailurePolicyType"` + + // matchPolicy defines how the "rules" list is used to match incoming requests. + // Allowed values are "Exact" or "Equivalent". + // + // - Exact: match a request only if it exactly matches a specified rule. + // For example, if deployments can be modified via apps/v1, apps/v1beta1, and extensions/v1beta1, + // but "rules" only included `apiGroups:["apps"], apiVersions:["v1"], resources: ["deployments"]`, + // a request to apps/v1beta1 or extensions/v1beta1 would not be sent to the webhook. + // + // - Equivalent: match a request if modifies a resource listed in rules, even via another API group or version. + // For example, if deployments can be modified via apps/v1, apps/v1beta1, and extensions/v1beta1, + // and "rules" only included `apiGroups:["apps"], apiVersions:["v1"], resources: ["deployments"]`, + // a request to apps/v1beta1 or extensions/v1beta1 would be converted to apps/v1 and sent to the webhook. + // + // Defaults to "Equivalent" + // +optional + MatchPolicy *admissionregistrationv1.MatchPolicyType `json:"matchPolicy,omitempty" protobuf:"bytes,9,opt,name=matchPolicy,casttype=MatchPolicyType"` + + // NamespaceSelector decides whether to run the webhook on an object based + // on whether the namespace for that object matches the selector. If the + // object itself is a namespace, the matching is performed on + // object.metadata.labels. If the object is another cluster scoped resource, + // it never skips the webhook. + // + // For example, to run the webhook on any objects whose namespace is not + // associated with "runlevel" of "0" or "1"; you will set the selector as + // follows: + // "namespaceSelector": { + // "matchExpressions": [ + // { + // "key": "runlevel", + // "operator": "NotIn", + // "values": [ + // "0", + // "1" + // ] + // } + // ] + // } + // + // If instead you want to only run the webhook on any objects whose + // namespace is associated with the "environment" of "prod" or "staging"; + // you will set the selector as follows: + // "namespaceSelector": { + // "matchExpressions": [ + // { + // "key": "environment", + // "operator": "In", + // "values": [ + // "prod", + // "staging" + // ] + // } + // ] + // } + // + // See + // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + // for more examples of label selectors. + // + // Default to the empty LabelSelector, which matches everything. + // +optional + NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty" protobuf:"bytes,5,opt,name=namespaceSelector"` + + // ObjectSelector decides whether to run the webhook based on if the + // object has matching labels. objectSelector is evaluated against both + // the oldObject and newObject that would be sent to the webhook, and + // is considered to match if either object matches the selector. A null + // object (oldObject in the case of create, or newObject in the case of + // delete) or an object that cannot have labels (like a + // DeploymentRollback or a PodProxyOptions object) is not considered to + // match. + // Use the object selector only if the webhook is opt-in, because end + // users may skip the admission webhook by setting the labels. + // Default to the empty LabelSelector, which matches everything. + // +optional + ObjectSelector *metav1.LabelSelector `json:"objectSelector,omitempty" protobuf:"bytes,10,opt,name=objectSelector"` + + // SideEffects states whether this webhook has side effects. + // Acceptable values are: None, NoneOnDryRun (webhooks created via v1beta1 may also specify Some or Unknown). + // Webhooks with side effects MUST implement a reconciliation system, since a request may be + // rejected by a future step in the admission chain and the side effects therefore need to be undone. + // Requests with the dryRun attribute will be auto-rejected if they match a webhook with + // sideEffects == Unknown or Some. + SideEffects *admissionregistrationv1.SideEffectClass `json:"sideEffects" protobuf:"bytes,6,opt,name=sideEffects,casttype=SideEffectClass"` + + // TimeoutSeconds specifies the timeout for this webhook. After the timeout passes, + // the webhook call will be ignored or the API call will fail based on the + // failure policy. + // The timeout value must be between 1 and 30 seconds. + // Default to 10 seconds. + // +optional + TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty" protobuf:"varint,7,opt,name=timeoutSeconds"` + + // AdmissionReviewVersions is an ordered list of preferred `AdmissionReview` + // versions the Webhook expects. API server will try to use first version in + // the list which it supports. If none of the versions specified in this list + // supported by API server, validation will fail for this object. + // If a persisted webhook configuration specifies allowed versions and does not + // include any versions known to the API Server, calls to the webhook will fail + // and be subject to the failure policy. + // +listType=atomic + AdmissionReviewVersions []string `json:"admissionReviewVersions" protobuf:"bytes,8,rep,name=admissionReviewVersions"` + + // MatchConditions is a list of conditions that must be met for a request to be sent to this + // webhook. Match conditions filter requests that have already been matched by the rules, + // namespaceSelector, and objectSelector. An empty list of matchConditions matches all requests. + // There are a maximum of 64 match conditions allowed. + // + // The exact matching logic is (in order): + // 1. If ANY matchCondition evaluates to FALSE, the webhook is skipped. + // 2. If ALL matchConditions evaluate to TRUE, the webhook is called. + // 3. If any matchCondition evaluates to an error (but none are FALSE): + // - If failurePolicy=Fail, reject the request + // - If failurePolicy=Ignore, the error is ignored and the webhook is skipped + // + // +patchMergeKey=name + // +patchStrategy=merge + // +listType=map + // +listMapKey=name + // +optional + MatchConditions []admissionregistrationv1.MatchCondition `json:"matchConditions,omitempty" patchMergeKey:"name" patchStrategy:"merge" protobuf:"bytes,11,opt,name=matchConditions"` +} + +func NewMutatingWebhook(in *MutatingWebhook, c *admissionregistrationv1.WebhookClientConfig, users rbac.UserListSpec, admins rbac.UserListSpec) (admissionregistrationv1.MutatingWebhook, error) { + out := admissionregistrationv1.MutatingWebhook{ + Name: in.Name, + Rules: in.Rules, + FailurePolicy: in.FailurePolicy, + ReinvocationPolicy: in.ReinvocationPolicy, + MatchPolicy: in.MatchPolicy, + NamespaceSelector: in.NamespaceSelector, + ObjectSelector: in.ObjectSelector, + SideEffects: in.SideEffects, + TimeoutSeconds: in.TimeoutSeconds, + AdmissionReviewVersions: in.AdmissionReviewVersions, + } + + webhookPath := "" + if in.Path != nil { + webhookPath = *in.Path + } + + out.ClientConfig = DynamicClientWithPath(*c, webhookPath) + + if len(in.MatchConditions) > 0 { + out.MatchConditions = append([]admissionregistrationv1.MatchCondition(nil), in.MatchConditions...) + + return out, nil + } + + conds := BuildGatingUserCondition(in.Options, users, admins) + if len(conds) > 0 { + out.MatchConditions = conds + } + + return out, nil +} + +// +kubebuilder:object:generate=true +type MutatingWebhook struct { + // The name of the admission webhook. + // Name should be fully qualified, e.g., imagepolicy.kubernetes.io, where + // "imagepolicy" is the name of the webhook, and kubernetes.io is the name + // of the organization. + // Required. + Name string `json:"name" protobuf:"bytes,1,opt,name=name"` + + // `path` is the URL path which will be sent in any request to + // this service. + Path *string `json:"path" protobuf:"bytes,3,opt,name=path"` + + // Capsule Custom Admission Options + // +optional + Options WebhookOptions `json:"opts,omitzero"` + + // Rules describes what operations on what resources/subresources the webhook cares about. + // The webhook cares about an operation if it matches _any_ Rule. + // However, in order to prevent ValidatingAdmissionWebhooks and MutatingAdmissionWebhooks + // from putting the cluster in a state which cannot be recovered from without completely + // disabling the plugin, ValidatingAdmissionWebhooks and MutatingAdmissionWebhooks are never called + // on admission requests for ValidatingWebhookConfiguration and MutatingWebhookConfiguration objects. + // +listType=atomic + Rules []admissionregistrationv1.RuleWithOperations `json:"rules,omitempty" protobuf:"bytes,3,rep,name=rules"` + + // FailurePolicy defines how unrecognized errors from the admission endpoint are handled - + // allowed values are Ignore or Fail. Defaults to Fail. + // +optional + FailurePolicy *admissionregistrationv1.FailurePolicyType `json:"failurePolicy,omitempty" protobuf:"bytes,4,opt,name=failurePolicy,casttype=FailurePolicyType"` + + // matchPolicy defines how the "rules" list is used to match incoming requests. + // Allowed values are "Exact" or "Equivalent". + // + // - Exact: match a request only if it exactly matches a specified rule. + // For example, if deployments can be modified via apps/v1, apps/v1beta1, and extensions/v1beta1, + // but "rules" only included `apiGroups:["apps"], apiVersions:["v1"], resources: ["deployments"]`, + // a request to apps/v1beta1 or extensions/v1beta1 would not be sent to the webhook. + // + // - Equivalent: match a request if modifies a resource listed in rules, even via another API group or version. + // For example, if deployments can be modified via apps/v1, apps/v1beta1, and extensions/v1beta1, + // and "rules" only included `apiGroups:["apps"], apiVersions:["v1"], resources: ["deployments"]`, + // a request to apps/v1beta1 or extensions/v1beta1 would be converted to apps/v1 and sent to the webhook. + // + // Defaults to "Equivalent" + // +optional + MatchPolicy *admissionregistrationv1.MatchPolicyType `json:"matchPolicy,omitempty" protobuf:"bytes,9,opt,name=matchPolicy,casttype=MatchPolicyType"` + + // NamespaceSelector decides whether to run the webhook on an object based + // on whether the namespace for that object matches the selector. If the + // object itself is a namespace, the matching is performed on + // object.metadata.labels. If the object is another cluster scoped resource, + // it never skips the webhook. + // + // For example, to run the webhook on any objects whose namespace is not + // associated with "runlevel" of "0" or "1"; you will set the selector as + // follows: + // "namespaceSelector": { + // "matchExpressions": [ + // { + // "key": "runlevel", + // "operator": "NotIn", + // "values": [ + // "0", + // "1" + // ] + // } + // ] + // } + // + // If instead you want to only run the webhook on any objects whose + // namespace is associated with the "environment" of "prod" or "staging"; + // you will set the selector as follows: + // "namespaceSelector": { + // "matchExpressions": [ + // { + // "key": "environment", + // "operator": "In", + // "values": [ + // "prod", + // "staging" + // ] + // } + // ] + // } + // + // See + // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + // for more examples of label selectors. + // + // Default to the empty LabelSelector, which matches everything. + // +optional + NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty" protobuf:"bytes,5,opt,name=namespaceSelector"` + + // ObjectSelector decides whether to run the webhook based on if the + // object has matching labels. objectSelector is evaluated against both + // the oldObject and newObject that would be sent to the webhook, and + // is considered to match if either object matches the selector. A null + // object (oldObject in the case of create, or newObject in the case of + // delete) or an object that cannot have labels (like a + // DeploymentRollback or a PodProxyOptions object) is not considered to + // match. + // Use the object selector only if the webhook is opt-in, because end + // users may skip the admission webhook by setting the labels. + // Default to the empty LabelSelector, which matches everything. + // +optional + ObjectSelector *metav1.LabelSelector `json:"objectSelector,omitempty" protobuf:"bytes,11,opt,name=objectSelector"` + + // SideEffects states whether this webhook has side effects. + // Acceptable values are: None, NoneOnDryRun (webhooks created via v1beta1 may also specify Some or Unknown). + // Webhooks with side effects MUST implement a reconciliation system, since a request may be + // rejected by a future step in the admission chain and the side effects therefore need to be undone. + // Requests with the dryRun attribute will be auto-rejected if they match a webhook with + // sideEffects == Unknown or Some. + SideEffects *admissionregistrationv1.SideEffectClass `json:"sideEffects" protobuf:"bytes,6,opt,name=sideEffects,casttype=SideEffectClass"` + + // TimeoutSeconds specifies the timeout for this webhook. After the timeout passes, + // the webhook call will be ignored or the API call will fail based on the + // failure policy. + // The timeout value must be between 1 and 30 seconds. + // Default to 10 seconds. + // +optional + TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty" protobuf:"varint,7,opt,name=timeoutSeconds"` + + // AdmissionReviewVersions is an ordered list of preferred `AdmissionReview` + // versions the Webhook expects. API server will try to use first version in + // the list which it supports. If none of the versions specified in this list + // supported by API server, validation will fail for this object. + // If a persisted webhook configuration specifies allowed versions and does not + // include any versions known to the API Server, calls to the webhook will fail + // and be subject to the failure policy. + // +listType=atomic + AdmissionReviewVersions []string `json:"admissionReviewVersions" protobuf:"bytes,8,rep,name=admissionReviewVersions"` + + // reinvocationPolicy indicates whether this webhook should be called multiple times as part of a single admission evaluation. + // Allowed values are "Never" and "IfNeeded". + // + // Never: the webhook will not be called more than once in a single admission evaluation. + // + // IfNeeded: the webhook will be called at least one additional time as part of the admission evaluation + // if the object being admitted is modified by other admission plugins after the initial webhook call. + // Webhooks that specify this option *must* be idempotent, able to process objects they previously admitted. + // Note: + // * the number of additional invocations is not guaranteed to be exactly one. + // * if additional invocations result in further modifications to the object, webhooks are not guaranteed to be invoked again. + // * webhooks that use this option may be reordered to minimize the number of additional invocations. + // * to validate an object after all mutations are guaranteed complete, use a validating admission webhook instead. + // + // Defaults to "Never". + // +optional + ReinvocationPolicy *admissionregistrationv1.ReinvocationPolicyType `json:"reinvocationPolicy,omitempty" protobuf:"bytes,10,opt,name=reinvocationPolicy,casttype=ReinvocationPolicyType"` + + // MatchConditions is a list of conditions that must be met for a request to be sent to this + // webhook. Match conditions filter requests that have already been matched by the rules, + // namespaceSelector, and objectSelector. An empty list of matchConditions matches all requests. + // There are a maximum of 64 match conditions allowed. + // + // The exact matching logic is (in order): + // 1. If ANY matchCondition evaluates to FALSE, the webhook is skipped. + // 2. If ALL matchConditions evaluate to TRUE, the webhook is called. + // 3. If any matchCondition evaluates to an error (but none are FALSE): + // - If failurePolicy=Fail, reject the request + // - If failurePolicy=Ignore, the error is ignored and the webhook is skipped + // + // +patchMergeKey=name + // +patchStrategy=merge + // +listType=map + // +listMapKey=name + // +optional + MatchConditions []admissionregistrationv1.MatchCondition `json:"matchConditions,omitempty" patchMergeKey:"name" patchStrategy:"merge" protobuf:"bytes,12,opt,name=matchConditions"` +} diff --git a/pkg/runtime/admission/zz_generated.deepcopy.go b/pkg/runtime/admission/zz_generated.deepcopy.go new file mode 100644 index 00000000..dd4a64aa --- /dev/null +++ b/pkg/runtime/admission/zz_generated.deepcopy.go @@ -0,0 +1,203 @@ +//go:build !ignore_autogenerated + +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by controller-gen. DO NOT EDIT. + +package admission + +import ( + "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DynamicAdmissionConfig) DeepCopyInto(out *DynamicAdmissionConfig) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Client != nil { + in, out := &in.Client, &out.Client + *out = new(v1.WebhookClientConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamicAdmissionConfig. +func (in *DynamicAdmissionConfig) DeepCopy() *DynamicAdmissionConfig { + if in == nil { + return nil + } + out := new(DynamicAdmissionConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MutatingWebhook) DeepCopyInto(out *MutatingWebhook) { + *out = *in + if in.Path != nil { + in, out := &in.Path, &out.Path + *out = new(string) + **out = **in + } + out.Options = in.Options + if in.Rules != nil { + in, out := &in.Rules, &out.Rules + *out = make([]v1.RuleWithOperations, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.FailurePolicy != nil { + in, out := &in.FailurePolicy, &out.FailurePolicy + *out = new(v1.FailurePolicyType) + **out = **in + } + if in.MatchPolicy != nil { + in, out := &in.MatchPolicy, &out.MatchPolicy + *out = new(v1.MatchPolicyType) + **out = **in + } + if in.NamespaceSelector != nil { + in, out := &in.NamespaceSelector, &out.NamespaceSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.ObjectSelector != nil { + in, out := &in.ObjectSelector, &out.ObjectSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.SideEffects != nil { + in, out := &in.SideEffects, &out.SideEffects + *out = new(v1.SideEffectClass) + **out = **in + } + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + *out = new(int32) + **out = **in + } + if in.AdmissionReviewVersions != nil { + in, out := &in.AdmissionReviewVersions, &out.AdmissionReviewVersions + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ReinvocationPolicy != nil { + in, out := &in.ReinvocationPolicy, &out.ReinvocationPolicy + *out = new(v1.ReinvocationPolicyType) + **out = **in + } + if in.MatchConditions != nil { + in, out := &in.MatchConditions, &out.MatchConditions + *out = make([]v1.MatchCondition, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MutatingWebhook. +func (in *MutatingWebhook) DeepCopy() *MutatingWebhook { + if in == nil { + return nil + } + out := new(MutatingWebhook) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ValidatingWebhook) DeepCopyInto(out *ValidatingWebhook) { + *out = *in + if in.Path != nil { + in, out := &in.Path, &out.Path + *out = new(string) + **out = **in + } + out.Options = in.Options + if in.Rules != nil { + in, out := &in.Rules, &out.Rules + *out = make([]v1.RuleWithOperations, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.FailurePolicy != nil { + in, out := &in.FailurePolicy, &out.FailurePolicy + *out = new(v1.FailurePolicyType) + **out = **in + } + if in.MatchPolicy != nil { + in, out := &in.MatchPolicy, &out.MatchPolicy + *out = new(v1.MatchPolicyType) + **out = **in + } + if in.NamespaceSelector != nil { + in, out := &in.NamespaceSelector, &out.NamespaceSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.ObjectSelector != nil { + in, out := &in.ObjectSelector, &out.ObjectSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.SideEffects != nil { + in, out := &in.SideEffects, &out.SideEffects + *out = new(v1.SideEffectClass) + **out = **in + } + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + *out = new(int32) + **out = **in + } + if in.AdmissionReviewVersions != nil { + in, out := &in.AdmissionReviewVersions, &out.AdmissionReviewVersions + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MatchConditions != nil { + in, out := &in.MatchConditions, &out.MatchConditions + *out = make([]v1.MatchCondition, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ValidatingWebhook. +func (in *ValidatingWebhook) DeepCopy() *ValidatingWebhook { + if in == nil { + return nil + } + out := new(ValidatingWebhook) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WebhookOptions) DeepCopyInto(out *WebhookOptions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookOptions. +func (in *WebhookOptions) DeepCopy() *WebhookOptions { + if in == nil { + return nil + } + out := new(WebhookOptions) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/runtime/cert/ca.go b/pkg/runtime/cert/ca.go index 296a86af..8f194649 100644 --- a/pkg/runtime/cert/ca.go +++ b/pkg/runtime/cert/ca.go @@ -41,6 +41,33 @@ func NewCertificateAuthorityFromBytes(certBytes, keyBytes []byte) (*CapsuleCA, e }, nil } +func (c *CapsuleCA) ExpiresIn(now time.Time) (time.Duration, error) { + if c == nil || c.certificate == nil { + return 0, errors.New("CA certificate is nil") + } + + return c.certificate.NotAfter.Sub(now), nil +} + +func (c *CapsuleCA) ValidateCert(certificate *x509.Certificate) error { + if c == nil || c.certificate == nil { + return errors.New("CA certificate is nil") + } + + if certificate == nil { + return errors.New("certificate is nil") + } + + roots := x509.NewCertPool() + roots.AddCert(c.certificate) + + _, err := certificate.Verify(x509.VerifyOptions{ + Roots: roots, + }) + + return err +} + func (c CapsuleCA) CACertificatePem() (b *bytes.Buffer, err error) { var crtBytes []byte @@ -90,7 +117,7 @@ func GenerateCertificateAuthority() (s *CapsuleCA, err error) { certificate: &x509.Certificate{ SerialNumber: big.NewInt(2019), Subject: pkix.Name{ - Organization: []string{"Clastix"}, + Organization: []string{"Projectcapsule"}, Country: []string{"UK"}, Province: []string{""}, Locality: []string{"London"}, @@ -155,7 +182,7 @@ func (c *CapsuleCA) GenerateCertificate(opts CertificateOptions) (certificatePem cert := &x509.Certificate{ SerialNumber: big.NewInt(1658), Subject: pkix.Name{ - Organization: []string{"Clastix"}, + Organization: []string{"Projectcapsule"}, Country: []string{"UK"}, Province: []string{""}, Locality: []string{"London"}, diff --git a/pkg/runtime/cert/ca_test.go b/pkg/runtime/cert/ca_test.go index 48476ffd..8c89565d 100644 --- a/pkg/runtime/cert/ca_test.go +++ b/pkg/runtime/cert/ca_test.go @@ -1,7 +1,7 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package cert +package cert_test import ( "bytes" @@ -12,14 +12,16 @@ import ( "time" "github.com/stretchr/testify/assert" + + "github.com/projectcapsule/capsule/pkg/runtime/cert" ) func TestNewCertificateAuthorityFromBytes(t *testing.T) { - var ca *CapsuleCA + var ca *cert.CapsuleCA var err error - ca, err = GenerateCertificateAuthority() + ca, err = cert.GenerateCertificateAuthority() assert.Nil(t, err) var crt *bytes.Buffer @@ -30,7 +32,7 @@ func TestNewCertificateAuthorityFromBytes(t *testing.T) { key, err = ca.CAPrivateKeyPem() assert.Nil(t, err) - _, err = NewCertificateAuthorityFromBytes(crt.Bytes(), key.Bytes()) + _, err = cert.NewCertificateAuthorityFromBytes(crt.Bytes(), key.Bytes()) assert.Nil(t, err) } @@ -44,17 +46,17 @@ func TestCapsuleCa_GenerateCertificate(t *testing.T) { "SAN": {[]string{"capsule-webhook-service.capsule-system.svc", "capsule-webhook-service.capsule-system.default.cluster"}}, } { t.Run(name, func(t *testing.T) { - var ca *CapsuleCA + var ca *cert.CapsuleCA var err error e := time.Now().AddDate(1, 0, 0) - ca, err = GenerateCertificateAuthority() + ca, err = cert.GenerateCertificateAuthority() assert.Nil(t, err) var crt *bytes.Buffer var key *bytes.Buffer - crt, key, err = ca.GenerateCertificate(NewCertOpts(e, c.dnsNames...)) + crt, key, err = ca.GenerateCertificate(cert.NewCertOpts(e, c.dnsNames...)) assert.Nil(t, err) var b *pem.Block diff --git a/pkg/runtime/cert/errors.go b/pkg/runtime/cert/errors.go deleted file mode 100644 index fa0c9175..00000000 --- a/pkg/runtime/cert/errors.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package cert - -type CaNotYetValidError struct{} - -func (CaNotYetValidError) Error() string { - return "The current CA is not yet valid" -} - -type CaExpiredError struct{} - -func (CaExpiredError) Error() string { - return "The current CA is expired" -} diff --git a/pkg/runtime/client/apply.go b/pkg/runtime/client/apply.go index 0457634a..b37653ec 100644 --- a/pkg/runtime/client/apply.go +++ b/pkg/runtime/client/apply.go @@ -5,60 +5,26 @@ package client import ( "context" - "fmt" - apierr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/controller-runtime/pkg/client" ) -func CreateOrPatch( +func PatchApply( ctx context.Context, c client.Client, obj client.Object, fieldOwner string, overwrite bool, ) error { - gvks, _, err := c.Scheme().ObjectKinds(obj) - if err != nil { - return err - } - - if len(gvks) == 0 { - return fmt.Errorf("no GVK found for object %T", obj) - } - - obj.GetObjectKind().SetGroupVersionKind(gvks[0]) - - //nolint:forcetypeassert - actual := obj.DeepCopyObject().(client.Object) - - key := client.ObjectKeyFromObject(obj) - - err = c.Get(ctx, key, actual) - - notFound := apierr.IsNotFound(err) - if err != nil && !notFound { - return err - } - - if !notFound { - obj.SetResourceVersion(actual.GetResourceVersion()) - } else { - obj.SetResourceVersion("") - } - - patchOpts := []client.PatchOption{ - client.FieldOwner(fieldOwner), - } - + opts := []client.PatchOption{client.FieldOwner(fieldOwner)} if overwrite { - patchOpts = append(patchOpts, client.ForceOwnership) + opts = append(opts, client.ForceOwnership) } //nolint:staticcheck - return c.Patch(ctx, obj, client.Apply, patchOpts...) + return c.Patch(ctx, obj, client.Apply, opts...) } // Returns timestamp of last apply for a manager. diff --git a/pkg/runtime/configuration/client.go b/pkg/runtime/configuration/client.go index e0c7c608..4b127093 100644 --- a/pkg/runtime/configuration/client.go +++ b/pkg/runtime/configuration/client.go @@ -5,69 +5,103 @@ package configuration import ( "context" + "fmt" + "os" "regexp" + "time" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" capsuleapi "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) // capsuleConfiguration is the Capsule Configuration retrieval mode // using a closure that provides the desired configuration. type capsuleConfiguration struct { retrievalFn func() *capsulev1beta2.CapsuleConfiguration + rest *rest.Config + client client.Client } func DefaultCapsuleConfiguration() capsulev1beta2.CapsuleConfigurationSpec { + d, _ := time.ParseDuration("1h") + return capsulev1beta2.CapsuleConfigurationSpec{ - Users: []capsuleapi.UserSpec{ + Users: []rbac.UserSpec{ { Name: "projectcapsule.dev", - Kind: capsuleapi.GroupOwner, + Kind: rbac.GroupOwner, }, }, + CacheInvalidation: metav1.Duration{ + Duration: d, + }, + RBAC: &capsulev1beta2.RBACConfiguration{ + DeleterClusterRole: "capsule-namespace-deleter", + ProvisionerClusterRole: "capsule-namespace-provisioner", + }, ForceTenantPrefix: false, ProtectedNamespaceRegexpString: "", } } -func NewCapsuleConfiguration(ctx context.Context, c client.Client, name string) Configuration { - return &capsuleConfiguration{retrievalFn: func() *capsulev1beta2.CapsuleConfiguration { - cfg := &capsulev1beta2.CapsuleConfiguration{} - key := types.NamespacedName{Name: name} +func NewCapsuleConfiguration(ctx context.Context, c client.Client, rest *rest.Config, name string) Configuration { + return &capsuleConfiguration{ + client: c, + rest: rest, + retrievalFn: func() *capsulev1beta2.CapsuleConfiguration { + cfg := &capsulev1beta2.CapsuleConfiguration{} + key := types.NamespacedName{Name: name} - if err := c.Get(ctx, key, cfg); err == nil { - return cfg - } else if !apierrors.IsNotFound(err) { - panic(errors.Wrap(err, "cannot retrieve Capsule configuration with name "+name)) - } - - cfg = &capsulev1beta2.CapsuleConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: DefaultCapsuleConfiguration(), - } - - if err := c.Create(ctx, cfg); err != nil { - if apierrors.IsAlreadyExists(err) { - if err := c.Get(ctx, key, cfg); err != nil { - panic(errors.Wrap(err, "configuration created concurrently but cannot be retrieved")) - } + if err := c.Get(ctx, key, cfg); err == nil { + return cfg + } else if !apierrors.IsNotFound(err) { + panic(errors.Wrap(err, "cannot retrieve Capsule configuration with name "+name)) + } + err := c.Get(ctx, key, cfg) + if err == nil { return cfg } - panic(errors.Wrap(err, "cannot create Capsule configuration with name "+name)) - } + if !apierrors.IsNotFound(err) { + panic(errors.Wrap(err, "cannot retrieve Capsule configuration with name "+name)) + } - return cfg - }} + cfg = &capsulev1beta2.CapsuleConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: DefaultCapsuleConfiguration(), + } + + if err := c.Create(ctx, cfg); err != nil { + if apierrors.IsAlreadyExists(err) { + if err := c.Get(ctx, key, cfg); err != nil { + panic(errors.Wrap(err, "configuration created concurrently but cannot be retrieved")) + } + + return cfg + } + + panic(errors.Wrap(err, "cannot create Capsule configuration with name "+name)) + } + + return cfg + }, + } +} + +func (c *capsuleConfiguration) GetConfigObject() *capsulev1beta2.CapsuleConfiguration { + return c.retrievalFn() } func (c *capsuleConfiguration) ProtectedNamespaceRegexp() (*regexp.Regexp, error) { @@ -101,7 +135,7 @@ func (c *capsuleConfiguration) AllowServiceAccountPromotion() bool { } func (c *capsuleConfiguration) MutatingWebhookConfigurationName() (name string) { - return c.retrievalFn().Spec.CapsuleResources.MutatingWebhookConfigurationName + return string(c.retrievalFn().Spec.Admission.Mutating.Name) } func (c *capsuleConfiguration) TenantCRDName() string { @@ -109,32 +143,32 @@ func (c *capsuleConfiguration) TenantCRDName() string { } func (c *capsuleConfiguration) ValidatingWebhookConfigurationName() (name string) { - return c.retrievalFn().Spec.CapsuleResources.ValidatingWebhookConfigurationName + return string(c.retrievalFn().Spec.Admission.Validating.Name) } //nolint:staticcheck func (c *capsuleConfiguration) UserGroups() []string { - return append(c.retrievalFn().Spec.UserGroups, c.retrievalFn().Spec.Users.GetByKinds([]capsuleapi.OwnerKind{capsuleapi.GroupOwner})...) + return append(c.retrievalFn().Spec.UserGroups, c.retrievalFn().Spec.Users.GetByKinds([]rbac.OwnerKind{rbac.GroupOwner})...) } //nolint:staticcheck func (c *capsuleConfiguration) UserNames() []string { - return append(c.retrievalFn().Spec.UserNames, c.retrievalFn().Spec.Users.GetByKinds([]capsuleapi.OwnerKind{capsuleapi.UserOwner, capsuleapi.ServiceAccountOwner})...) + return append(c.retrievalFn().Spec.UserNames, c.retrievalFn().Spec.Users.GetByKinds([]rbac.OwnerKind{rbac.UserOwner, rbac.ServiceAccountOwner})...) } -func (c *capsuleConfiguration) Users() capsuleapi.UserListSpec { - out := capsuleapi.UserListSpec{} +func (c *capsuleConfiguration) Users() rbac.UserListSpec { + out := rbac.UserListSpec{} for _, user := range c.UserNames() { - out.Upsert(capsuleapi.UserSpec{ - Kind: capsuleapi.UserOwner, + out.Upsert(rbac.UserSpec{ + Kind: rbac.UserOwner, Name: user, }) } for _, group := range c.UserGroups() { - out.Upsert(capsuleapi.UserSpec{ - Kind: capsuleapi.GroupOwner, + out.Upsert(rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: group, }) } @@ -142,7 +176,7 @@ func (c *capsuleConfiguration) Users() capsuleapi.UserListSpec { return out } -func (c *capsuleConfiguration) GetUsersByStatus() capsuleapi.UserListSpec { +func (c *capsuleConfiguration) GetUsersByStatus() rbac.UserListSpec { return c.retrievalFn().Status.Users } @@ -166,7 +200,7 @@ func (c *capsuleConfiguration) ForbiddenUserNodeAnnotations() *capsuleapi.Forbid return &c.retrievalFn().Spec.NodeMetadata.ForbiddenAnnotations } -func (c *capsuleConfiguration) Administrators() capsuleapi.UserListSpec { +func (c *capsuleConfiguration) Administrators() rbac.UserListSpec { return c.retrievalFn().Spec.Administrators } @@ -181,3 +215,48 @@ func (c *capsuleConfiguration) RBAC() *capsulev1beta2.RBACConfiguration { func (c *capsuleConfiguration) CacheInvalidation() metav1.Duration { return c.retrievalFn().Spec.CacheInvalidation } + +func (c *capsuleConfiguration) ServiceAccountClientProperties() capsulev1beta2.ServiceAccountClient { + return c.retrievalFn().Spec.Impersonation +} + +func (c *capsuleConfiguration) ServiceAccountClient(ctx context.Context) (*rest.Config, error) { + props := c.ServiceAccountClientProperties() + + cfg := rest.CopyConfig(c.rest) + + if props.Endpoint != "" { + cfg.Host = props.Endpoint + } + + if props.SkipTLSVerify { + cfg.Insecure = true + cfg.CAData = nil + cfg.CAFile = "" + + return cfg, nil + } + + if props.CASecretName != "" { + namespace := props.CASecretNamespace + if namespace == "" { + namespace = meta.RFC1123SubdomainName(os.Getenv("NAMESPACE")) + } + + caData, err := fetchCACertFromSecret( + ctx, + c.client, + namespace.String(), + props.CASecretName.String(), + props.CASecretKey, + ) + if err != nil { + return nil, fmt.Errorf("could not fetch CA cert: %w", err) + } + + cfg.CAData = caData + cfg.CAFile = "" + } + + return cfg, nil +} diff --git a/pkg/runtime/configuration/configuration.go b/pkg/runtime/configuration/configuration.go index e4f84bd0..ff1e2ea4 100644 --- a/pkg/runtime/configuration/configuration.go +++ b/pkg/runtime/configuration/configuration.go @@ -4,12 +4,15 @@ package configuration import ( + "context" "regexp" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" capsuleapi "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) const ( @@ -17,6 +20,8 @@ const ( ) type Configuration interface { + GetConfigObject() *capsulev1beta2.CapsuleConfiguration + ProtectedNamespaceRegexp() (*regexp.Regexp, error) ForceTenantPrefix() bool // EnableTLSConfiguration enabled the TLS reconciler, responsible for creating CA and TLS certificate required @@ -29,12 +34,14 @@ type Configuration interface { TenantCRDName() string UserNames() []string UserGroups() []string - Users() capsuleapi.UserListSpec - GetUsersByStatus() capsuleapi.UserListSpec + Users() rbac.UserListSpec + GetUsersByStatus() rbac.UserListSpec IgnoreUserWithGroups() []string ForbiddenUserNodeLabels() *capsuleapi.ForbiddenListSpec ForbiddenUserNodeAnnotations() *capsuleapi.ForbiddenListSpec - Administrators() capsuleapi.UserListSpec + Administrators() rbac.UserListSpec + ServiceAccountClientProperties() capsulev1beta2.ServiceAccountClient + ServiceAccountClient(context.Context) (*rest.Config, error) Admission() capsulev1beta2.DynamicAdmission RBAC() *capsulev1beta2.RBACConfiguration CacheInvalidation() metav1.Duration diff --git a/pkg/runtime/configuration/env.go b/pkg/runtime/configuration/env.go new file mode 100644 index 00000000..050453e8 --- /dev/null +++ b/pkg/runtime/configuration/env.go @@ -0,0 +1,29 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package configuration + +import "os" + +const ( + EnvironmentServiceaccountName string = "SERVICE_ACCOUNT" + EnvironmentControllerNamespace string = "NAMESPACE" +) + +func ControllerNamespace() (namespace string) { + return os.Getenv("NAMESPACE") +} + +func ControllerServiceAccount() (name string, namespace string) { + return os.Getenv("SERVICE_ACCOUNT"), ControllerNamespace() +} + +func IsControllerServiceAccount(name string, namespace string) bool { + sa, ns := ControllerServiceAccount() + + if ns == "" || sa == "" { + return false + } + + return namespace == ns && name == sa +} diff --git a/pkg/runtime/configuration/utils.go b/pkg/runtime/configuration/utils.go new file mode 100644 index 00000000..eea2f4fc --- /dev/null +++ b/pkg/runtime/configuration/utils.go @@ -0,0 +1,29 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package configuration + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func fetchCACertFromSecret(ctx context.Context, k8sClient client.Client, namespace, secretName, secretCaKey string) ([]byte, error) { + var secret corev1.Secret + + key := client.ObjectKey{Namespace: namespace, Name: secretName} + + if err := k8sClient.Get(ctx, key, &secret); err != nil { + return nil, fmt.Errorf("unable to fetch CA secret %s/%s: %w", namespace, secretName, err) + } + + data, ok := secret.Data[secretCaKey] + if !ok { + return nil, fmt.Errorf("secret %s/%s does not contain key '%s'", namespace, secretName, secretCaKey) + } + + return data, nil +} diff --git a/pkg/runtime/events/reasons.go b/pkg/runtime/events/reasons.go index d39f02a7..1f6bf640 100644 --- a/pkg/runtime/events/reasons.go +++ b/pkg/runtime/events/reasons.go @@ -56,4 +56,8 @@ const ( // ResourcePools. ReasonDisassociated string = "Disassociated" + + // CustomQuotas. + ReasonUsageCalculationFailed = "UsageCalculationFailed" + ReasonQuotaExceeded = "QuotaExceeded" ) diff --git a/pkg/runtime/gvk/gk_types.go b/pkg/runtime/gvk/gk_types.go new file mode 100644 index 00000000..0abc03a3 --- /dev/null +++ b/pkg/runtime/gvk/gk_types.go @@ -0,0 +1,25 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package gvk + +import "k8s.io/apimachinery/pkg/runtime/schema" + +type VersionKind struct { + // Kind of the referent. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + Kind string `json:"kind" protobuf:"bytes,1,opt,name=kind"` + // API version of the referent. + APIVersion string `json:"apiVersion" protobuf:"bytes,5,opt,name=apiVersion"` +} + +func (s VersionKind) GroupVersionKind() schema.GroupVersionKind { + gv, err := schema.ParseGroupVersion(s.APIVersion) + if err != nil { + return schema.GroupVersionKind{ + Kind: s.Kind, + } + } + + return gv.WithKind(s.Kind) +} diff --git a/pkg/runtime/gvk/namespace_resources.go b/pkg/runtime/gvk/namespace_resources.go new file mode 100644 index 00000000..b246e4c9 --- /dev/null +++ b/pkg/runtime/gvk/namespace_resources.go @@ -0,0 +1,58 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package gvk + +import ( + "fmt" + "slices" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func NamespacedListableResources(resourceLists []*metav1.APIResourceList) ([]schema.GroupVersionResource, error) { + gvrs := make([]schema.GroupVersionResource, 0, 64) + seen := make(map[schema.GroupVersionResource]struct{}) + + for _, rl := range resourceLists { + gv, err := schema.ParseGroupVersion(rl.GroupVersion) + if err != nil { + return nil, fmt.Errorf("parse groupVersion %q: %w", rl.GroupVersion, err) + } + + for _, r := range rl.APIResources { + if !r.Namespaced { + continue + } + + if strings.Contains(r.Name, "/") { + continue + } + + if !SupportsVerb(r.Verbs, "list") { + continue + } + + if !SupportsVerb(r.Verbs, "patch") && !SupportsVerb(r.Verbs, "update") { + continue + } + + gvr := gv.WithResource(r.Name) + if _, ok := seen[gvr]; ok { + continue + } + + seen[gvr] = struct{}{} + + gvrs = append(gvrs, gvr) + } + } + + return gvrs, nil +} + +func SupportsVerb(verbs metav1.Verbs, want string) bool { + return slices.Contains(verbs, want) +} diff --git a/pkg/runtime/gvk/plural.go b/pkg/runtime/gvk/plural.go new file mode 100644 index 00000000..690c5d3f --- /dev/null +++ b/pkg/runtime/gvk/plural.go @@ -0,0 +1,29 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package gvk + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" +) + +// GetGVKByPlural returns the GroupVersionKind for a given plural name. +func ReplacePluralWithKind(discoveryClient *discovery.DiscoveryClient, gvk *schema.GroupVersionKind) error { + resourceList, err := discoveryClient.ServerResourcesForGroupVersion(gvk.Group + "/" + gvk.Version) + if err != nil { + return err + } + + for _, resource := range resourceList.APIResources { + if resource.Name == gvk.Kind { + gvk.Kind = resource.Kind + + return nil + } + } + + return fmt.Errorf("could not find GVK for plural name: %s", gvk.Kind) +} diff --git a/pkg/runtime/gvk/resource_id.go b/pkg/runtime/gvk/resource_id.go new file mode 100644 index 00000000..6af0c36f --- /dev/null +++ b/pkg/runtime/gvk/resource_id.go @@ -0,0 +1,113 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package gvk + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type TenantResourceID struct { + Tenant string `json:"tenant,omitempty"` +} + +type TenantResourceIDWithOrigin struct { + TenantResourceID `json:",inline"` + + Origin string `json:"origin,omitempty"` +} + +// ResourceID represents the decomposed parts of a Kubernetes resource identity. +type ResourceID struct { + TenantResourceIDWithOrigin `json:",inline"` + + Group string `json:"group,omitempty"` + Version string `json:"version,omitempty"` + Kind string `json:"kind,omitempty"` + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` +} + +// ResourceKey builds the canonical key string used for maps/sets. +// Non-namespaced objects will have "_" as the namespace component. +func NewResourceID(u *unstructured.Unstructured, tenant string, origin string) ResourceID { + gvk := u.GroupVersionKind() + + return ResourceID{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind, + Name: u.GetName(), + Namespace: u.GetNamespace(), + TenantResourceIDWithOrigin: TenantResourceIDWithOrigin{ + TenantResourceID: TenantResourceID{ + Tenant: tenant, + }, + Origin: origin, + }, + } +} + +func (r ResourceID) GetName() string { + return r.Name +} + +func (r ResourceID) GetNamespace() string { + return r.Namespace +} + +// GVK returns the schema.GroupVersionKind of the resource. +func (r ResourceID) GetGVK() schema.GroupVersionKind { + return schema.GroupVersionKind{ + Group: r.Group, + Version: r.Version, + Kind: r.Kind, + } +} + +func (r ResourceID) GetGVKKey(sep string) string { + if sep == "" { + sep = "\x1f" + } + + return fmt.Sprintf("%s%s%s%s%s%s%s%s%s%s", + r.Group, sep, + r.Version, sep, + r.Kind, sep, + r.Namespace, sep, + r.Name, sep, + ) +} + +func (r ResourceID) GetKey(sep string) string { + // Use a delimiter that won’t appear in fields normally; '\x1f' (unit separator) is great. + if sep == "" { + sep = "\x1f" + } + + return fmt.Sprintf("%s%s%s%s%s%s%s%s%s%s%s%s%s%s", + r.Group, sep, + r.Version, sep, + r.Kind, sep, + r.Namespace, sep, + r.Name, sep, + r.Tenant, sep, + r.Origin, sep, + ) +} + +func (r ResourceID) FieldOwner(sep string) string { + // Use a delimiter that won’t appear in fields normally; '\x1f' (unit separator) is great. + if sep == "" { + sep = "/" + } + + return fmt.Sprintf("%s%s%s%s%s%s", + r.Namespace, sep, + r.Tenant, sep, + r.Origin, sep, + ) +} diff --git a/pkg/runtime/gvk/resource_key.go b/pkg/runtime/gvk/resource_key.go new file mode 100644 index 00000000..54d1cbae --- /dev/null +++ b/pkg/runtime/gvk/resource_key.go @@ -0,0 +1,49 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package gvk + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type ResourceKey struct { + Group string + Version string + Kind string + Namespace string + Name string +} + +// keyFromUnstructured builds a stable identity for dedupe. +// Prefer UID if you want “same object even if renamed” semantics; for Kubernetes resources +// name+namespace+GVK is typically what you want for “don’t process duplicates”. +func KeyFromUnstructured(o *unstructured.Unstructured) (ResourceKey, bool) { + if o == nil { + return ResourceKey{}, false + } + + gvk := o.GroupVersionKind() + if gvk.Empty() { + gvk.Kind = o.GetKind() + + gv, err := schema.ParseGroupVersion(o.GetAPIVersion()) + if err == nil { + gvk.Group = gv.Group + gvk.Version = gv.Version + } + } + + if gvk.Kind == "" || o.GetName() == "" { + return ResourceKey{}, false + } + + return ResourceKey{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind, + Namespace: o.GetNamespace(), + Name: o.GetName(), + }, true +} diff --git a/pkg/runtime/gvk/resource_key_test.go b/pkg/runtime/gvk/resource_key_test.go new file mode 100644 index 00000000..7e035dd6 --- /dev/null +++ b/pkg/runtime/gvk/resource_key_test.go @@ -0,0 +1,103 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package gvk_test + +import ( + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/projectcapsule/capsule/pkg/runtime/gvk" +) + +func TestKeyFromUnstructured(t *testing.T) { + t.Parallel() + + t.Run("nil object returns false", func(t *testing.T) { + t.Parallel() + + _, ok := gvk.KeyFromUnstructured(nil) + if ok { + t.Fatalf("expected ok=false") + } + }) + + t.Run("missing kind returns false", func(t *testing.T) { + t.Parallel() + + u := &unstructured.Unstructured{} + u.SetAPIVersion("apps/v1") + u.SetName("demo") + u.SetNamespace("default") + + _, ok := gvk.KeyFromUnstructured(u) + if ok { + t.Fatalf("expected ok=false") + } + }) + + t.Run("missing name returns false", func(t *testing.T) { + t.Parallel() + + u := &unstructured.Unstructured{} + u.SetAPIVersion("apps/v1") + u.SetKind("Deployment") + u.SetNamespace("default") + + _, ok := gvk.KeyFromUnstructured(u) + if ok { + t.Fatalf("expected ok=false") + } + }) + + t.Run("returns key for namespaced object", func(t *testing.T) { + t.Parallel() + + u := &unstructured.Unstructured{} + u.SetAPIVersion("apps/v1") + u.SetKind("Deployment") + u.SetNamespace("default") + u.SetName("demo") + + key, ok := gvk.KeyFromUnstructured(u) + if !ok { + t.Fatalf("expected ok=true") + } + + if key.Group != "apps" { + t.Fatalf("expected group=apps, got %q", key.Group) + } + if key.Version != "v1" { + t.Fatalf("expected version=v1, got %q", key.Version) + } + if key.Kind != "Deployment" { + t.Fatalf("expected kind=Deployment, got %q", key.Kind) + } + if key.Namespace != "default" { + t.Fatalf("expected namespace=default, got %q", key.Namespace) + } + if key.Name != "demo" { + t.Fatalf("expected name=demo, got %q", key.Name) + } + }) + + t.Run("returns key for cluster-scoped object (empty namespace)", func(t *testing.T) { + t.Parallel() + + u := &unstructured.Unstructured{} + u.SetAPIVersion("rbac.authorization.k8s.io/v1") + u.SetKind("ClusterRole") + // no namespace + u.SetName("admin") + + key, ok := gvk.KeyFromUnstructured(u) + if !ok { + t.Fatalf("expected ok=true") + } + + if key.Namespace != "" { + t.Fatalf("expected empty namespace, got %q", key.Namespace) + } + }) +} diff --git a/pkg/runtime/handlers/admission_user.go b/pkg/runtime/handlers/admission_user.go new file mode 100644 index 00000000..6a8fa8a8 --- /dev/null +++ b/pkg/runtime/handlers/admission_user.go @@ -0,0 +1,49 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package handlers + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/users" +) + +type NewObjectFunc[T client.Object] func() T + +func ResolveAdmissionUser( + ctx context.Context, + c client.Client, + req admission.Request, + config configuration.Configuration, +) users.AdmissionUser { + user := users.NewAdmissionUser(users.AdmissionUserUnknown, req.UserInfo) + + if user.IsControllerServiceAccount() { + user.Type = users.AdmissionUserAdmin + + return user + } + + if users.HasIgnoredGroup(req.UserInfo.Groups, config.IgnoreUserWithGroups()) { + return user + } + + if config.Administrators().IsPresent(req.UserInfo.Username, req.UserInfo.Groups) { + user.Type = users.AdmissionUserAdmin + + return user + } + + if users.IsCapsuleUser(ctx, c, config, req.UserInfo.Username, req.UserInfo.Groups) { + user.Type = users.AdmissionUserCapsule + + return user + } + + return user +} diff --git a/pkg/runtime/handlers/handlers.go b/pkg/runtime/handlers/handlers.go index fae5ddd0..3ce8fef3 100644 --- a/pkg/runtime/handlers/handlers.go +++ b/pkg/runtime/handlers/handlers.go @@ -11,24 +11,31 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/users" ) type Func func(ctx context.Context, req admission.Request) *admission.Response type Handler interface { - OnCreate(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) Func - OnDelete(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) Func - OnUpdate(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) Func + OnCreate(client client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func + OnDelete(client client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func + OnUpdate(client client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func } type HanderWithTenant interface { - OnCreate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func - OnUpdate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func - OnDelete(c client.Client, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func + OnCreate(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func + OnUpdate(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func + OnDelete(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func } type TypedHandler[T client.Object] interface { - OnCreate(c client.Client, obj T, decoder admission.Decoder, recorder events.EventRecorder) Func - OnUpdate(c client.Client, obj T, old T, decoder admission.Decoder, recorder events.EventRecorder) Func - OnDelete(c client.Client, obj T, decoder admission.Decoder, recorder events.EventRecorder) Func + OnCreate(c client.Client, reader client.Reader, obj T, decoder admission.Decoder, recorder events.EventRecorder) Func + OnUpdate(c client.Client, reader client.Reader, obj T, old T, decoder admission.Decoder, recorder events.EventRecorder) Func + OnDelete(c client.Client, reader client.Reader, obj T, decoder admission.Decoder, recorder events.EventRecorder) Func +} + +type TypedHandlerWithUser[T client.Object] interface { + OnCreate(c client.Client, reader client.Reader, user users.AdmissionUser, obj T, decoder admission.Decoder, recorder events.EventRecorder) Func + OnUpdate(c client.Client, reader client.Reader, user users.AdmissionUser, obj T, old T, decoder admission.Decoder, recorder events.EventRecorder) Func + OnDelete(c client.Client, reader client.Reader, user users.AdmissionUser, obj T, decoder admission.Decoder, recorder events.EventRecorder) Func } diff --git a/pkg/runtime/handlers/in_capsule_groups.go b/pkg/runtime/handlers/in_capsule_groups.go index a590c8db..334f9387 100644 --- a/pkg/runtime/handlers/in_capsule_groups.go +++ b/pkg/runtime/handlers/in_capsule_groups.go @@ -28,14 +28,14 @@ type handler struct { } //nolint:dupl -func (h *handler) OnCreate(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) Func { +func (h *handler) OnCreate(client client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { if !users.IsCapsuleUser(ctx, client, h.configuration, req.UserInfo.Username, req.UserInfo.Groups) { return nil } for _, hndl := range h.handlers { - if response := hndl.OnCreate(client, decoder, recorder)(ctx, req); response != nil { + if response := hndl.OnCreate(client, reader, decoder, recorder)(ctx, req); response != nil { return response } } @@ -45,14 +45,14 @@ func (h *handler) OnCreate(client client.Client, decoder admission.Decoder, reco } //nolint:dupl -func (h *handler) OnDelete(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) Func { +func (h *handler) OnDelete(client client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { if !users.IsCapsuleUser(ctx, client, h.configuration, req.UserInfo.Username, req.UserInfo.Groups) { return nil } for _, hndl := range h.handlers { - if response := hndl.OnDelete(client, decoder, recorder)(ctx, req); response != nil { + if response := hndl.OnDelete(client, reader, decoder, recorder)(ctx, req); response != nil { return response } } @@ -62,14 +62,14 @@ func (h *handler) OnDelete(client client.Client, decoder admission.Decoder, reco } //nolint:dupl -func (h *handler) OnUpdate(client client.Client, decoder admission.Decoder, recorder events.EventRecorder) Func { +func (h *handler) OnUpdate(client client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { if !users.IsCapsuleUser(ctx, client, h.configuration, req.UserInfo.Username, req.UserInfo.Groups) { return nil } for _, hndl := range h.handlers { - if response := hndl.OnUpdate(client, decoder, recorder)(ctx, req); response != nil { + if response := hndl.OnUpdate(client, reader, decoder, recorder)(ctx, req); response != nil { return response } } diff --git a/pkg/runtime/handlers/is_not_privileged.go b/pkg/runtime/handlers/is_not_privileged.go new file mode 100644 index 00000000..047cb151 --- /dev/null +++ b/pkg/runtime/handlers/is_not_privileged.go @@ -0,0 +1,75 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package handlers + +import ( + "context" + + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/users" +) + +func IsNotPrivileged(configuration configuration.Configuration, handlers ...Handler) Handler { + return &isNotPrivileged{ + configuration: configuration, + handlers: handlers, + } +} + +type isNotPrivileged struct { + configuration configuration.Configuration + handlers []Handler +} + +func (h *isNotPrivileged) OnCreate(client client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + if users.IsAdminUser(req, h.configuration.Administrators()) { + return nil + } + + for _, hndl := range h.handlers { + if response := hndl.OnCreate(client, reader, decoder, recorder)(ctx, req); response != nil { + return response + } + } + + return nil + } +} + +func (h *isNotPrivileged) OnDelete(client client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + if users.IsAdminUser(req, h.configuration.Administrators()) { + return nil + } + + for _, hndl := range h.handlers { + if response := hndl.OnDelete(client, reader, decoder, recorder)(ctx, req); response != nil { + return response + } + } + + return nil + } +} + +func (h *isNotPrivileged) OnUpdate(client client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + if users.IsAdminUser(req, h.configuration.Administrators()) { + return nil + } + + for _, hndl := range h.handlers { + if response := hndl.OnUpdate(client, reader, decoder, recorder)(ctx, req); response != nil { + return response + } + } + + return nil + } +} diff --git a/pkg/runtime/handlers/typed_tenant_object.go b/pkg/runtime/handlers/typed_tenant_object.go index 1884ba37..453a4675 100644 --- a/pkg/runtime/handlers/typed_tenant_object.go +++ b/pkg/runtime/handlers/typed_tenant_object.go @@ -16,9 +16,9 @@ import ( ) type TypedHandlerWithTenant[T client.Object] interface { - OnCreate(c client.Client, obj T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func - OnUpdate(c client.Client, obj T, old T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func - OnDelete(c client.Client, obj T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func + OnCreate(c client.Client, reader client.Reader, obj T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func + OnUpdate(c client.Client, reader client.Reader, obj T, old T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func + OnDelete(c client.Client, reader client.Reader, obj T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func } type TypedTenantHandler[T client.Object] struct { @@ -26,9 +26,9 @@ type TypedTenantHandler[T client.Object] struct { Handlers []TypedHandlerWithTenant[T] } -func (h *TypedTenantHandler[T]) OnCreate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) Func { +func (h *TypedTenantHandler[T]) OnCreate(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { - tnt, err := h.resolveTenant(ctx, c, req) + tnt, err := h.resolveTenant(ctx, reader, req) if err != nil { return ErroredResponse(err) } @@ -43,7 +43,7 @@ func (h *TypedTenantHandler[T]) OnCreate(c client.Client, decoder admission.Deco } for _, hndl := range h.Handlers { - if response := hndl.OnCreate(c, obj, decoder, recorder, tnt)(ctx, req); response != nil { + if response := hndl.OnCreate(c, reader, obj, decoder, recorder, tnt)(ctx, req); response != nil { return response } } @@ -52,9 +52,9 @@ func (h *TypedTenantHandler[T]) OnCreate(c client.Client, decoder admission.Deco } } -func (h *TypedTenantHandler[T]) OnUpdate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) Func { +func (h *TypedTenantHandler[T]) OnUpdate(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { - tnt, err := h.resolveTenant(ctx, c, req) + tnt, err := h.resolveTenant(ctx, reader, req) if err != nil { return ErroredResponse(err) } @@ -74,7 +74,7 @@ func (h *TypedTenantHandler[T]) OnUpdate(c client.Client, decoder admission.Deco } for _, hndl := range h.Handlers { - if response := hndl.OnUpdate(c, oldObj, newObj, decoder, recorder, tnt)(ctx, req); response != nil { + if response := hndl.OnUpdate(c, reader, oldObj, newObj, decoder, recorder, tnt)(ctx, req); response != nil { return response } } @@ -83,9 +83,9 @@ func (h *TypedTenantHandler[T]) OnUpdate(c client.Client, decoder admission.Deco } } -func (h *TypedTenantHandler[T]) OnDelete(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) Func { +func (h *TypedTenantHandler[T]) OnDelete(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { - tnt, err := h.resolveTenant(ctx, c, req) + tnt, err := h.resolveTenant(ctx, reader, req) if err != nil { return ErroredResponse(err) } @@ -100,7 +100,7 @@ func (h *TypedTenantHandler[T]) OnDelete(c client.Client, decoder admission.Deco } for _, hndl := range h.Handlers { - if response := hndl.OnDelete(c, obj, decoder, recorder, tnt)(ctx, req); response != nil { + if response := hndl.OnDelete(c, reader, obj, decoder, recorder, tnt)(ctx, req); response != nil { return response } } @@ -109,10 +109,10 @@ func (h *TypedTenantHandler[T]) OnDelete(c client.Client, decoder admission.Deco } } -func (h *TypedTenantHandler[T]) resolveTenant(ctx context.Context, c client.Client, req admission.Request) (*capsulev1beta2.Tenant, error) { +func (h *TypedTenantHandler[T]) resolveTenant(ctx context.Context, c client.Reader, req admission.Request) (*capsulev1beta2.Tenant, error) { if req.Namespace == "" { return nil, nil } - return tenant.TenantByStatusNamespace(ctx, c, req.Namespace) + return tenant.GetTenantByNamespace(ctx, c, req.Namespace) } diff --git a/pkg/runtime/handlers/typed_tenant_ruleset.go b/pkg/runtime/handlers/typed_tenant_ruleset.go index f4628536..5568caa8 100644 --- a/pkg/runtime/handlers/typed_tenant_ruleset.go +++ b/pkg/runtime/handlers/typed_tenant_ruleset.go @@ -15,14 +15,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" "github.com/projectcapsule/capsule/pkg/tenant" ) type TypedHandlerWithTenantWithRuleset[T client.Object] interface { - OnCreate(c client.Client, obj T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, rule *capsulev1beta2.NamespaceRuleBody) Func - OnUpdate(c client.Client, obj T, old T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, rule *capsulev1beta2.NamespaceRuleBody) Func - OnDelete(c client.Client, obj T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, rule *capsulev1beta2.NamespaceRuleBody) Func + OnCreate(c client.Client, reader client.Reader, obj T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, rule *api.NamespaceRuleBodyNamespace) Func + OnUpdate(c client.Client, reader client.Reader, obj T, old T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, rule *api.NamespaceRuleBodyNamespace) Func + OnDelete(c client.Client, reader client.Reader, obj T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant, rule *api.NamespaceRuleBodyNamespace) Func } type TypedTenantWithRulesetHandler[T client.Object] struct { @@ -30,9 +31,9 @@ type TypedTenantWithRulesetHandler[T client.Object] struct { Handlers []TypedHandlerWithTenantWithRuleset[T] } -func (h *TypedTenantWithRulesetHandler[T]) OnCreate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) Func { +func (h *TypedTenantWithRulesetHandler[T]) OnCreate(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { - tnt, err := h.resolveTenant(ctx, c, req) + tnt, err := h.resolveTenant(ctx, reader, req) if err != nil { return ErroredResponse(err) } @@ -52,7 +53,7 @@ func (h *TypedTenantWithRulesetHandler[T]) OnCreate(c client.Client, decoder adm } for _, hndl := range h.Handlers { - if response := hndl.OnCreate(c, obj, decoder, recorder, tnt, rule)(ctx, req); response != nil { + if response := hndl.OnCreate(c, reader, obj, decoder, recorder, tnt, rule)(ctx, req); response != nil { return response } } @@ -61,7 +62,7 @@ func (h *TypedTenantWithRulesetHandler[T]) OnCreate(c client.Client, decoder adm } } -func (h *TypedTenantWithRulesetHandler[T]) OnUpdate(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) Func { +func (h *TypedTenantWithRulesetHandler[T]) OnUpdate(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { tnt, err := h.resolveTenant(ctx, c, req) if err != nil { @@ -88,7 +89,7 @@ func (h *TypedTenantWithRulesetHandler[T]) OnUpdate(c client.Client, decoder adm } for _, hndl := range h.Handlers { - if response := hndl.OnUpdate(c, oldObj, newObj, decoder, recorder, tnt, rule)(ctx, req); response != nil { + if response := hndl.OnUpdate(c, reader, oldObj, newObj, decoder, recorder, tnt, rule)(ctx, req); response != nil { return response } } @@ -97,9 +98,9 @@ func (h *TypedTenantWithRulesetHandler[T]) OnUpdate(c client.Client, decoder adm } } -func (h *TypedTenantWithRulesetHandler[T]) OnDelete(c client.Client, decoder admission.Decoder, recorder events.EventRecorder) Func { +func (h *TypedTenantWithRulesetHandler[T]) OnDelete(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { return func(ctx context.Context, req admission.Request) *admission.Response { - tnt, err := h.resolveTenant(ctx, c, req) + tnt, err := h.resolveTenant(ctx, reader, req) if err != nil { return ErroredResponse(err) } @@ -119,7 +120,7 @@ func (h *TypedTenantWithRulesetHandler[T]) OnDelete(c client.Client, decoder adm } for _, hndl := range h.Handlers { - if response := hndl.OnDelete(c, obj, decoder, recorder, tnt, rule)(ctx, req); response != nil { + if response := hndl.OnDelete(c, reader, obj, decoder, recorder, tnt, rule)(ctx, req); response != nil { return response } } @@ -128,23 +129,23 @@ func (h *TypedTenantWithRulesetHandler[T]) OnDelete(c client.Client, decoder adm } } -func (h *TypedTenantWithRulesetHandler[T]) resolveTenant(ctx context.Context, c client.Client, req admission.Request) (*capsulev1beta2.Tenant, error) { +func (h *TypedTenantWithRulesetHandler[T]) resolveTenant(ctx context.Context, c client.Reader, req admission.Request) (*capsulev1beta2.Tenant, error) { if req.Namespace == "" { return nil, nil } - return tenant.TenantByStatusNamespace(ctx, c, req.Namespace) + return tenant.GetTenantByNamespace(ctx, c, req.Namespace) } // Resolve the corresponding managed ruleset for this namespace // If not yet present try to calculate it. func (h *TypedTenantWithRulesetHandler[T]) resolveRuleset( ctx context.Context, - c client.Client, + c client.Reader, req admission.Request, namespace string, tnt *capsulev1beta2.Tenant, -) (*capsulev1beta2.NamespaceRuleBody, error) { +) (*api.NamespaceRuleBodyNamespace, error) { rs := &capsulev1beta2.RuleStatus{} key := types.NamespacedName{ Namespace: namespace, @@ -152,9 +153,7 @@ func (h *TypedTenantWithRulesetHandler[T]) resolveRuleset( } if err := c.Get(ctx, key, rs); err == nil { - rule := rs.Status.Rule - - return &rule, nil + return &rs.Status.Rule, nil } else if !apierrors.IsNotFound(err) { return nil, err } @@ -164,5 +163,5 @@ func (h *TypedTenantWithRulesetHandler[T]) resolveRuleset( return nil, err } - return tenant.BuildNamespaceRuleBodyForNamespace(ns, tnt) + return tenant.BuildNamespaceRuleBodyStatus(ctx, c, ns, tnt) } diff --git a/pkg/runtime/handlers/typed_tenant_user_object.go b/pkg/runtime/handlers/typed_tenant_user_object.go new file mode 100644 index 00000000..d9495eea --- /dev/null +++ b/pkg/runtime/handlers/typed_tenant_user_object.go @@ -0,0 +1,127 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +//nolint:dupl +package handlers + +import ( + "context" + + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/tenant" + "github.com/projectcapsule/capsule/pkg/users" +) + +type TypedHandlerWithTenantUser[T client.Object] interface { + OnCreate(c client.Client, reader client.Reader, user users.AdmissionUser, obj T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func + OnUpdate(c client.Client, reader client.Reader, user users.AdmissionUser, obj T, old T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func + OnDelete(c client.Client, reader client.Reader, user users.AdmissionUser, obj T, decoder admission.Decoder, recorder events.EventRecorder, tnt *capsulev1beta2.Tenant) Func +} + +type TypedTenantWithUserHandler[T client.Object] struct { + Factory NewObjectFunc[T] + Handlers []TypedHandlerWithTenantUser[T] + Configuration configuration.Configuration +} + +func (h *TypedTenantWithUserHandler[T]) OnCreate(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + user := ResolveAdmissionUser(ctx, c, req, h.Configuration) + + tnt, err := h.resolveTenant(ctx, reader, req) + if err != nil { + return ErroredResponse(err) + } + + if tnt == nil { + return nil + } + + obj := h.Factory() + if err := decoder.Decode(req, obj); err != nil { + return ErroredResponse(err) + } + + for _, hndl := range h.Handlers { + if response := hndl.OnCreate(c, reader, user, obj, decoder, recorder, tnt)(ctx, req); response != nil { + return response + } + } + + return nil + } +} + +func (h *TypedTenantWithUserHandler[T]) OnUpdate(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + user := ResolveAdmissionUser(ctx, c, req, h.Configuration) + + tnt, err := h.resolveTenant(ctx, reader, req) + if err != nil { + return ErroredResponse(err) + } + + if tnt == nil { + return nil + } + + newObj := h.Factory() + if err := decoder.Decode(req, newObj); err != nil { + return ErroredResponse(err) + } + + oldObj := h.Factory() + if err := decoder.DecodeRaw(req.OldObject, oldObj); err != nil { + return ErroredResponse(err) + } + + for _, hndl := range h.Handlers { + if response := hndl.OnUpdate(c, reader, user, oldObj, newObj, decoder, recorder, tnt)(ctx, req); response != nil { + return response + } + } + + return nil + } +} + +func (h *TypedTenantWithUserHandler[T]) OnDelete(c client.Client, reader client.Reader, decoder admission.Decoder, recorder events.EventRecorder) Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + user := ResolveAdmissionUser(ctx, c, req, h.Configuration) + + tnt, err := h.resolveTenant(ctx, reader, req) + if err != nil { + return ErroredResponse(err) + } + + if tnt == nil { + return nil + } + + obj := h.Factory() + if err := decoder.Decode(req, obj); err != nil { + return ErroredResponse(err) + } + + for _, hndl := range h.Handlers { + if response := hndl.OnDelete(c, reader, user, obj, decoder, recorder, tnt)(ctx, req); response != nil { + return response + } + } + + return nil + } +} + +func (h *TypedTenantWithUserHandler[T]) resolveTenant(ctx context.Context, c client.Reader, req admission.Request) (*capsulev1beta2.Tenant, error) { + if req.Namespace == "" { + return nil, nil + } + + return tenant.GetTenantByNamespace(ctx, c, req.Namespace) +} diff --git a/pkg/runtime/handlers/utils.go b/pkg/runtime/handlers/utils.go deleted file mode 100644 index 7fdf3ad1..00000000 --- a/pkg/runtime/handlers/utils.go +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 - -package handlers - -import "sigs.k8s.io/controller-runtime/pkg/client" - -type NewObjectFunc[T client.Object] func() T diff --git a/pkg/runtime/indexers/customquota/const.go b/pkg/runtime/indexers/customquota/const.go new file mode 100644 index 00000000..7b2cd3f5 --- /dev/null +++ b/pkg/runtime/indexers/customquota/const.go @@ -0,0 +1,9 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquota + +const ( + TargetIndexerFieldName string = ".status.target" + ObjectUIDIndexerFieldName string = ".status.objs.uid" +) diff --git a/pkg/runtime/indexers/customquota/global_target.go b/pkg/runtime/indexers/customquota/global_target.go new file mode 100644 index 00000000..55c8151f --- /dev/null +++ b/pkg/runtime/indexers/customquota/global_target.go @@ -0,0 +1,34 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquota + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +type GlobalTargetReference struct{} + +func (o GlobalTargetReference) Object() client.Object { + return &capsulev1beta2.GlobalCustomQuota{} +} + +func (o GlobalTargetReference) Field() string { + return TargetIndexerFieldName +} + +func (o GlobalTargetReference) Func() client.IndexerFunc { + return func(object client.Object) []string { + tr := object.(*capsulev1beta2.GlobalCustomQuota) //nolint:forcetypeassert + + targets := make([]string, 0, len(tr.Status.Targets)) + + for _, t := range tr.Status.Targets { + targets = append(targets, t.String()) + } + + return targets + } +} diff --git a/pkg/runtime/indexers/customquota/global_uid.go b/pkg/runtime/indexers/customquota/global_uid.go new file mode 100644 index 00000000..118e106f --- /dev/null +++ b/pkg/runtime/indexers/customquota/global_uid.go @@ -0,0 +1,34 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquota + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +type GlobalObjectUIDReference struct{} + +func (o GlobalObjectUIDReference) Object() client.Object { + return &capsulev1beta2.GlobalCustomQuota{} +} + +func (o GlobalObjectUIDReference) Field() string { + return ObjectUIDIndexerFieldName +} + +func (o GlobalObjectUIDReference) Func() client.IndexerFunc { + return func(object client.Object) []string { + tr := object.(*capsulev1beta2.GlobalCustomQuota) //nolint:forcetypeassert + + objs := make([]string, 0, len(tr.Status.Claims)) + + for _, obj := range tr.Status.Claims { + objs = append(objs, string(obj.UID)) + } + + return objs + } +} diff --git a/pkg/runtime/indexers/customquota/namespaces_target.go b/pkg/runtime/indexers/customquota/namespaces_target.go new file mode 100644 index 00000000..4178a1b4 --- /dev/null +++ b/pkg/runtime/indexers/customquota/namespaces_target.go @@ -0,0 +1,34 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquota + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +type NamespacedTargetReference struct{} + +func (o NamespacedTargetReference) Object() client.Object { + return &capsulev1beta2.CustomQuota{} +} + +func (o NamespacedTargetReference) Field() string { + return TargetIndexerFieldName +} + +func (o NamespacedTargetReference) Func() client.IndexerFunc { + return func(object client.Object) []string { + tr := object.(*capsulev1beta2.CustomQuota) //nolint:forcetypeassert + + targets := make([]string, 0, len(tr.Status.Targets)) + + for _, t := range tr.Status.Targets { + targets = append(targets, t.String()) + } + + return targets + } +} diff --git a/pkg/runtime/indexers/customquota/namespaces_uid.go b/pkg/runtime/indexers/customquota/namespaces_uid.go new file mode 100644 index 00000000..e0acf7d7 --- /dev/null +++ b/pkg/runtime/indexers/customquota/namespaces_uid.go @@ -0,0 +1,34 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package customquota + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +type NamespacedObjectUIDReference struct{} + +func (o NamespacedObjectUIDReference) Object() client.Object { + return &capsulev1beta2.CustomQuota{} +} + +func (o NamespacedObjectUIDReference) Field() string { + return ObjectUIDIndexerFieldName +} + +func (o NamespacedObjectUIDReference) Func() client.IndexerFunc { + return func(object client.Object) []string { + tr := object.(*capsulev1beta2.CustomQuota) //nolint:forcetypeassert + + objs := make([]string, 0, len(tr.Status.Claims)) + + for _, obj := range tr.Status.Claims { + objs = append(objs, string(obj.UID)) + } + + return objs + } +} diff --git a/pkg/runtime/indexers/indexer.go b/pkg/runtime/indexers/indexer.go index 2048fce9..1ca0ebb9 100644 --- a/pkg/runtime/indexers/indexer.go +++ b/pkg/runtime/indexers/indexer.go @@ -15,6 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/runtime/indexers/customquota" "github.com/projectcapsule/capsule/pkg/runtime/indexers/ingress" "github.com/projectcapsule/capsule/pkg/runtime/indexers/namespace" "github.com/projectcapsule/capsule/pkg/runtime/indexers/resourcepool" @@ -32,6 +33,17 @@ type CustomIndexer interface { func AddToManager(ctx context.Context, log logr.Logger, mgr manager.Manager) error { indexers := []CustomIndexer{ tenant.NamespacesReference{Obj: &capsulev1beta2.Tenant{}}, + tenantresource.GlobalServiceAccount{}, + tenantresource.GlobalProcessedItems{}, + tenantresource.GlobalCreatedItems{}, + tenantresource.NamespacedServiceAccount{}, + tenantresource.NamespacedProcessedItems{}, + tenantresource.NamespacedResourceNamespace{}, + tenantresource.NamespacedCreatedItems{}, + customquota.NamespacedTargetReference{}, + customquota.NamespacedObjectUIDReference{}, + customquota.GlobalTargetReference{}, + customquota.GlobalObjectUIDReference{}, resourcepool.NamespacesReference{Obj: &capsulev1beta2.ResourcePool{}}, resourcepool.PoolUIDReference{Obj: &capsulev1beta2.ResourcePoolClaim{}}, tenant.OwnerReference{}, @@ -39,8 +51,6 @@ func AddToManager(ctx context.Context, log logr.Logger, mgr manager.Manager) err ingress.HostnamePath{Obj: &extensionsv1beta1.Ingress{}}, ingress.HostnamePath{Obj: &networkingv1beta1.Ingress{}}, ingress.HostnamePath{Obj: &networkingv1.Ingress{}}, - tenantresource.GlobalProcessedItems{}, - tenantresource.LocalProcessedItems{}, } for _, f := range indexers { diff --git a/pkg/runtime/indexers/tenantresource/constants.go b/pkg/runtime/indexers/namespace/const.go similarity index 52% rename from pkg/runtime/indexers/tenantresource/constants.go rename to pkg/runtime/indexers/namespace/const.go index e750fb02..ca47a5ea 100644 --- a/pkg/runtime/indexers/tenantresource/constants.go +++ b/pkg/runtime/indexers/namespace/const.go @@ -1,8 +1,8 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package tenantresource +package namespace const ( - IndexerFieldName = "status.processedItems" + OwnerReferenceIndex string = ".metadata.ownerReferences[*].capsule" ) diff --git a/pkg/runtime/indexers/namespace/namespaces.go b/pkg/runtime/indexers/namespace/namespaces.go index 37a122d5..8dff8130 100644 --- a/pkg/runtime/indexers/namespace/namespaces.go +++ b/pkg/runtime/indexers/namespace/namespaces.go @@ -19,7 +19,7 @@ func (o OwnerReference) Object() client.Object { } func (o OwnerReference) Field() string { - return ".metadata.ownerReferences[*].capsule" + return OwnerReferenceIndex } func (o OwnerReference) Func() client.IndexerFunc { diff --git a/pkg/runtime/indexers/tenant/const.go b/pkg/runtime/indexers/tenant/const.go new file mode 100644 index 00000000..11030822 --- /dev/null +++ b/pkg/runtime/indexers/tenant/const.go @@ -0,0 +1,9 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenant + +const ( + NamespaceIndexerFieldName string = ".status.namespaces" + OwnerKindIndexerFieldName string = ".spec.owner.ownerkind" +) diff --git a/pkg/runtime/indexers/tenant/namespaces.go b/pkg/runtime/indexers/tenant/namespaces.go index 9da775f9..e30f9082 100644 --- a/pkg/runtime/indexers/tenant/namespaces.go +++ b/pkg/runtime/indexers/tenant/namespaces.go @@ -18,7 +18,7 @@ func (o NamespacesReference) Object() client.Object { } func (o NamespacesReference) Field() string { - return ".status.namespaces" + return NamespaceIndexerFieldName } func (o NamespacesReference) Func() client.IndexerFunc { diff --git a/pkg/runtime/indexers/tenant/owner.go b/pkg/runtime/indexers/tenant/owner.go index f4bd563b..db641b0f 100644 --- a/pkg/runtime/indexers/tenant/owner.go +++ b/pkg/runtime/indexers/tenant/owner.go @@ -19,7 +19,7 @@ func (o OwnerReference) Object() client.Object { } func (o OwnerReference) Field() string { - return ".spec.owner.ownerkind" + return OwnerKindIndexerFieldName } func (o OwnerReference) Func() client.IndexerFunc { diff --git a/pkg/runtime/indexers/tenantresource/const.go b/pkg/runtime/indexers/tenantresource/const.go new file mode 100644 index 00000000..b8470f19 --- /dev/null +++ b/pkg/runtime/indexers/tenantresource/const.go @@ -0,0 +1,11 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenantresource + +const ( + ServiceAccountIndexerFieldName string = "spec.serviceaccount" + ProcessedIndexerFieldName string = "status.items" + CreatedIndexerFieldName string = "status.items.created" + NamespaceIndexerFieldName string = "metadata.namespace" +) diff --git a/pkg/runtime/indexers/tenantresource/global_created_items.go b/pkg/runtime/indexers/tenantresource/global_created_items.go new file mode 100644 index 00000000..1b294cb9 --- /dev/null +++ b/pkg/runtime/indexers/tenantresource/global_created_items.go @@ -0,0 +1,36 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenantresource + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +type GlobalCreatedItems struct{} + +func (g GlobalCreatedItems) Object() client.Object { + return &capsulev1beta2.GlobalTenantResource{} +} + +func (g GlobalCreatedItems) Field() string { + return CreatedIndexerFieldName +} + +func (g GlobalCreatedItems) Func() client.IndexerFunc { + return func(object client.Object) []string { + tgr := object.(*capsulev1beta2.GlobalTenantResource) //nolint:forcetypeassert + + out := make([]string, 0, len(tgr.Status.ProcessedItems)) + + for _, pi := range tgr.Status.ProcessedItems { + if pi.Created { + out = append(out, pi.GetGVKKey("")) + } + } + + return out + } +} diff --git a/pkg/runtime/indexers/tenantresource/global.go b/pkg/runtime/indexers/tenantresource/global_items.go similarity index 91% rename from pkg/runtime/indexers/tenantresource/global.go rename to pkg/runtime/indexers/tenantresource/global_items.go index d91291d8..6eb095aa 100644 --- a/pkg/runtime/indexers/tenantresource/global.go +++ b/pkg/runtime/indexers/tenantresource/global_items.go @@ -1,7 +1,6 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -//nolint:dupl package tenantresource import ( @@ -17,7 +16,7 @@ func (g GlobalProcessedItems) Object() client.Object { } func (g GlobalProcessedItems) Field() string { - return IndexerFieldName + return ProcessedIndexerFieldName } func (g GlobalProcessedItems) Func() client.IndexerFunc { @@ -26,7 +25,7 @@ func (g GlobalProcessedItems) Func() client.IndexerFunc { out := make([]string, 0, len(tgr.Status.ProcessedItems)) for _, pi := range tgr.Status.ProcessedItems { - out = append(out, pi.String()) + out = append(out, pi.GetGVKKey("")) } return out diff --git a/pkg/runtime/indexers/tenantresource/global_serviceaccount.go b/pkg/runtime/indexers/tenantresource/global_serviceaccount.go new file mode 100644 index 00000000..427734a1 --- /dev/null +++ b/pkg/runtime/indexers/tenantresource/global_serviceaccount.go @@ -0,0 +1,41 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenantresource + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +type GlobalServiceAccount struct{} + +func (g GlobalServiceAccount) Object() client.Object { + return &capsulev1beta2.GlobalTenantResource{} +} + +func (g GlobalServiceAccount) Field() string { + return ServiceAccountIndexerFieldName +} + +func (g GlobalServiceAccount) Func() client.IndexerFunc { + return func(object client.Object) []string { + tgr := object.(*capsulev1beta2.GlobalTenantResource) //nolint:forcetypeassert + + imp := tgr.Status.ServiceAccount + if imp == nil { + return nil + } + + ns := imp.Namespace.String() + + name := imp.Name.String() + + if ns == "" || name == "" { + return nil + } + + return []string{ns + "/" + name} + } +} diff --git a/pkg/runtime/indexers/tenantresource/global_serviceaccount_test.go b/pkg/runtime/indexers/tenantresource/global_serviceaccount_test.go new file mode 100644 index 00000000..549f341d --- /dev/null +++ b/pkg/runtime/indexers/tenantresource/global_serviceaccount_test.go @@ -0,0 +1,86 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenantresource_test + +import ( + "reflect" + "testing" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/indexers/tenantresource" +) + +func TestGlobalServiceAccount_Object(t *testing.T) { + var idx tenantresource.GlobalServiceAccount + obj := idx.Object() + + _, ok := obj.(*capsulev1beta2.GlobalTenantResource) + if !ok { + t.Fatalf("expected *capsulev1beta2.GlobalTenantResource, got %T", obj) + } +} + +func TestGlobalServiceAccount_Field(t *testing.T) { + var idx tenantresource.GlobalServiceAccount + if idx.Field() != tenantresource.ServiceAccountIndexerFieldName { + t.Fatalf("unexpected field: got %q want %q", idx.Field(), tenantresource.ServiceAccountIndexerFieldName) + } +} + +func TestGlobalServiceAccount_Func(t *testing.T) { + var idx tenantresource.GlobalServiceAccount + fn := idx.Func() + + t.Run("nil serviceAccount => nil", func(t *testing.T) { + tgr := &capsulev1beta2.GlobalTenantResource{} + tgr.Status.ServiceAccount = nil + + got := fn(tgr) + if got != nil { + t.Fatalf("expected nil, got %#v", got) + } + }) + + t.Run("empty namespace => nil", func(t *testing.T) { + tgr := &capsulev1beta2.GlobalTenantResource{} + tgr.Status.ServiceAccount = &meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: meta.RFC1123Name("sa"), + Namespace: meta.RFC1123SubdomainName(""), + } + + got := fn(tgr) + if got != nil { + t.Fatalf("expected nil, got %#v", got) + } + }) + + t.Run("empty name => nil", func(t *testing.T) { + tgr := &capsulev1beta2.GlobalTenantResource{} + tgr.Status.ServiceAccount = &meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: meta.RFC1123Name(""), + Namespace: meta.RFC1123SubdomainName("kube-system"), + } + + got := fn(tgr) + if got != nil { + t.Fatalf("expected nil, got %#v", got) + } + }) + + t.Run("both set => returns ns/name key", func(t *testing.T) { + tgr := &capsulev1beta2.GlobalTenantResource{} + tgr.Status.ServiceAccount = &meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: meta.RFC1123Name("default"), + Namespace: meta.RFC1123SubdomainName("kube-system"), + } + + got := fn(tgr) + want := []string{"kube-system/default"} + + if !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected result\nwant=%#v\ngot =%#v", want, got) + } + }) +} diff --git a/pkg/runtime/indexers/tenantresource/namespaced_created_items.go b/pkg/runtime/indexers/tenantresource/namespaced_created_items.go new file mode 100644 index 00000000..83f4ef6a --- /dev/null +++ b/pkg/runtime/indexers/tenantresource/namespaced_created_items.go @@ -0,0 +1,36 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenantresource + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +type NamespacedCreatedItems struct{} + +func (g NamespacedCreatedItems) Object() client.Object { + return &capsulev1beta2.TenantResource{} +} + +func (g NamespacedCreatedItems) Field() string { + return CreatedIndexerFieldName +} + +func (g NamespacedCreatedItems) Func() client.IndexerFunc { + return func(object client.Object) []string { + tgr := object.(*capsulev1beta2.TenantResource) //nolint:forcetypeassert + + out := make([]string, 0, len(tgr.Status.ProcessedItems)) + + for _, pi := range tgr.Status.ProcessedItems { + if pi.Created { + out = append(out, pi.GetGVKKey("")) + } + } + + return out + } +} diff --git a/pkg/runtime/indexers/tenantresource/local.go b/pkg/runtime/indexers/tenantresource/namespaced_items.go similarity index 65% rename from pkg/runtime/indexers/tenantresource/local.go rename to pkg/runtime/indexers/tenantresource/namespaced_items.go index 60a14d9e..407d1270 100644 --- a/pkg/runtime/indexers/tenantresource/local.go +++ b/pkg/runtime/indexers/tenantresource/namespaced_items.go @@ -1,7 +1,6 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -//nolint:dupl package tenantresource import ( @@ -10,23 +9,23 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" ) -type LocalProcessedItems struct{} +type NamespacedProcessedItems struct{} -func (g LocalProcessedItems) Object() client.Object { +func (g NamespacedProcessedItems) Object() client.Object { return &capsulev1beta2.TenantResource{} } -func (g LocalProcessedItems) Field() string { - return IndexerFieldName +func (g NamespacedProcessedItems) Field() string { + return ProcessedIndexerFieldName } -func (g LocalProcessedItems) Func() client.IndexerFunc { +func (g NamespacedProcessedItems) Func() client.IndexerFunc { return func(object client.Object) []string { tgr := object.(*capsulev1beta2.TenantResource) //nolint:forcetypeassert out := make([]string, 0, len(tgr.Status.ProcessedItems)) for _, pi := range tgr.Status.ProcessedItems { - out = append(out, pi.String()) + out = append(out, pi.GetGVKKey("")) } return out diff --git a/pkg/runtime/indexers/tenantresource/namespaced_namespace.go b/pkg/runtime/indexers/tenantresource/namespaced_namespace.go new file mode 100644 index 00000000..44d1b790 --- /dev/null +++ b/pkg/runtime/indexers/tenantresource/namespaced_namespace.go @@ -0,0 +1,31 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenantresource + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +type NamespacedResourceNamespace struct{} + +func (g NamespacedResourceNamespace) Object() client.Object { + return &capsulev1beta2.TenantResource{} +} + +func (g NamespacedResourceNamespace) Field() string { + return NamespaceIndexerFieldName +} + +func (g NamespacedResourceNamespace) Func() client.IndexerFunc { + return func(object client.Object) []string { + tr := object.(*capsulev1beta2.TenantResource) //nolint:forcetypeassert + if tr.Namespace == "" { + return nil + } + + return []string{tr.Namespace} + } +} diff --git a/pkg/runtime/indexers/tenantresource/namespaced_serviceaccount.go b/pkg/runtime/indexers/tenantresource/namespaced_serviceaccount.go new file mode 100644 index 00000000..a8b687cd --- /dev/null +++ b/pkg/runtime/indexers/tenantresource/namespaced_serviceaccount.go @@ -0,0 +1,41 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenantresource + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +type NamespacedServiceAccount struct{} + +func (g NamespacedServiceAccount) Object() client.Object { + return &capsulev1beta2.TenantResource{} +} + +func (g NamespacedServiceAccount) Field() string { + return ServiceAccountIndexerFieldName +} + +func (g NamespacedServiceAccount) Func() client.IndexerFunc { + return func(object client.Object) []string { + tgr := object.(*capsulev1beta2.TenantResource) //nolint:forcetypeassert + + imp := tgr.Status.ServiceAccount + if imp == nil { + return nil + } + + ns := tgr.GetNamespace() + + name := imp.Name.String() + + if ns == "" || name == "" { + return nil + } + + return []string{ns + "/" + name} + } +} diff --git a/pkg/runtime/jsonpath/compiled.go b/pkg/runtime/jsonpath/compiled.go new file mode 100644 index 00000000..33a07ed3 --- /dev/null +++ b/pkg/runtime/jsonpath/compiled.go @@ -0,0 +1,52 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package jsonpath + +import ( + "bytes" + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/util/jsonpath" +) + +const maxJSONPathLength = 1024 + +// CompiledJSONPath wraps a parsed JSONPath expression for repeated use. +type CompiledJSONPath struct { + jp *jsonpath.JSONPath +} + +// CompileJSONPath parses and validates a JSONPath source path once. +// Example sourcePath: ".spec.resources.requests.cpu". +func CompileJSONPath(sourcePath string) (*CompiledJSONPath, error) { + sourcePath = strings.TrimSpace(sourcePath) + if err := validateSourcePath(sourcePath); err != nil { + return nil, err + } + + j := jsonpath.New("usagePath") + j.AllowMissingKeys(true) + + if err := j.Parse(wrapJSONPath(sourcePath)); err != nil { + return nil, fmt.Errorf("parse usage jsonpath %q: %w", sourcePath, err) + } + + return &CompiledJSONPath{jp: j}, nil +} + +// Execute applies a precompiled JSONPath to the given object and returns the extracted value. +func (c *CompiledJSONPath) Execute(u unstructured.Unstructured) (string, error) { + if c == nil || c.jp == nil { + return "", fmt.Errorf("compiled jsonpath is nil") + } + + var buf bytes.Buffer + if err := c.jp.Execute(&buf, u.Object); err != nil { + return "", fmt.Errorf("execute usage jsonpath: %w", err) + } + + return strings.TrimSpace(buf.String()), nil +} diff --git a/pkg/runtime/jsonpath/compiled_test.go b/pkg/runtime/jsonpath/compiled_test.go new file mode 100644 index 00000000..4c99dbd2 --- /dev/null +++ b/pkg/runtime/jsonpath/compiled_test.go @@ -0,0 +1,258 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package jsonpath + +import ( + "strings" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestCompileJSONPath(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + sourcePath string + wantErr bool + errMsg string + }{ + { + name: "valid simple path", + sourcePath: ".spec.resources.requests.cpu", + }, + { + name: "valid path with surrounding whitespace", + sourcePath: " .spec.resources.requests.memory ", + }, + { + name: "empty path", + sourcePath: "", + wantErr: true, + errMsg: "sourcePath must not be empty", + }, + { + name: "missing leading dot", + sourcePath: "spec.resources.requests.cpu", + wantErr: true, + errMsg: "sourcePath must start with '.'", + }, + { + name: "contains newline", + sourcePath: ".spec.\nresources.requests.cpu", + wantErr: true, + errMsg: "sourcePath must not contain control whitespace", + }, + { + name: "contains tab", + sourcePath: ".spec.\tresources.requests.cpu", + wantErr: true, + errMsg: "sourcePath must not contain control whitespace", + }, + { + name: "too long", + sourcePath: "." + strings.Repeat("a", maxJSONPathLength), + wantErr: true, + errMsg: "sourcePath exceeds max length", + }, + { + name: "invalid jsonpath syntax", + sourcePath: ".spec[", + wantErr: true, + errMsg: "parse usage jsonpath", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := CompileJSONPath(tt.sourcePath) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got nil") + } + if !strings.Contains(err.Error(), tt.errMsg) { + t.Fatalf("expected error to contain %q, got %q", tt.errMsg, err.Error()) + } + if got != nil { + t.Fatalf("expected compiled path to be nil on error, got %#v", got) + } + return + } + + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if got == nil { + t.Fatal("expected compiled path, got nil") + } + if got.jp == nil { + t.Fatal("expected compiled jsonpath to be initialized, got nil jp") + } + }) + } +} + +func TestCompiledJSONPathExecute(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + compiled *CompiledJSONPath + object unstructured.Unstructured + want string + wantErr bool + errMsg string + prepareJP string + }{ + { + name: "nil receiver", + compiled: nil, + object: unstructured.Unstructured{}, + wantErr: true, + errMsg: "compiled jsonpath is nil", + }, + { + name: "nil jsonpath", + compiled: &CompiledJSONPath{}, + object: unstructured.Unstructured{}, + wantErr: true, + errMsg: "compiled jsonpath is nil", + }, + { + name: "extract string value", + prepareJP: ".spec.resources.requests.cpu", + object: unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "resources": map[string]interface{}{ + "requests": map[string]interface{}{ + "cpu": "250m", + }, + }, + }, + }, + }, + want: "250m", + }, + { + name: "trim surrounding whitespace", + prepareJP: ".spec.value", + object: unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "value": " hello world ", + }, + }, + }, + want: "hello world", + }, + { + name: "extract numeric value", + prepareJP: ".spec.replicas", + object: unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "replicas": int64(3), + }, + }, + }, + want: "3", + }, + { + name: "missing path returns empty string", + prepareJP: ".spec.resources.requests.memory", + object: unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "resources": map[string]interface{}{ + "requests": map[string]interface{}{ + "cpu": "250m", + }, + }, + }, + }, + }, + want: "", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + compiled := tt.compiled + if tt.prepareJP != "" { + var err error + compiled, err = CompileJSONPath(tt.prepareJP) + if err != nil { + t.Fatalf("failed to compile jsonpath for test: %v", err) + } + } + + got, err := compiled.Execute(tt.object) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got nil (value=%q)", got) + } + if !strings.Contains(err.Error(), tt.errMsg) { + t.Fatalf("expected error to contain %q, got %q", tt.errMsg, err.Error()) + } + return + } + + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if got != tt.want { + t.Fatalf("expected %q, got %q", tt.want, got) + } + }) + } +} + +func TestCompileUsageJSONPath_Execute_Success(t *testing.T) { + compiled, err := CompileJSONPath(".spec.resources.requests.memory") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + u := unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "resources": map[string]interface{}{ + "requests": map[string]interface{}{ + "memory": "1Gi", + }, + }, + }, + }, + } + + got, err := compiled.Execute(u) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if got != "1Gi" { + t.Fatalf("expected %q, got %q", "1Gi", got) + } +} + +func TestCompiledJSONPath_Execute_NilReceiver(t *testing.T) { + var compiled *CompiledJSONPath + + u := unstructured.Unstructured{ + Object: map[string]interface{}{}, + } + + _, err := compiled.Execute(u) + if err == nil { + t.Fatal("expected error, got nil") + } +} diff --git a/pkg/runtime/jsonpath/truth.go b/pkg/runtime/jsonpath/truth.go new file mode 100644 index 00000000..36de0b8b --- /dev/null +++ b/pkg/runtime/jsonpath/truth.go @@ -0,0 +1,41 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package jsonpath + +import ( + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// EvaluateTruthyFromCompiled evaluates a compiled JSONPath expression and interprets the result +// using "truthy" semantics: +// +// - empty result => false +// - "false" (case-insensitive) => false +// - "0" => false +// - anything else non-empty => true +func EvaluateTruthyFromCompiled(u unstructured.Unstructured, compiled *CompiledJSONPath) (bool, error) { + if compiled == nil { + return false, fmt.Errorf("compiled jsonpath is nil") + } + + value, err := compiled.Execute(u) + if err != nil { + return false, err + } + + value = strings.TrimSpace(value) + if value == "" { + return false, nil + } + + switch strings.ToLower(value) { + case "false", "0": + return false, nil + default: + return true, nil + } +} diff --git a/pkg/runtime/jsonpath/utils.go b/pkg/runtime/jsonpath/utils.go new file mode 100644 index 00000000..10c6b2c2 --- /dev/null +++ b/pkg/runtime/jsonpath/utils.go @@ -0,0 +1,33 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package jsonpath + +import ( + "fmt" + "strings" +) + +func wrapJSONPath(sourcePath string) string { + return fmt.Sprintf("{%s}", sourcePath) +} + +func validateSourcePath(sourcePath string) error { + if sourcePath == "" { + return fmt.Errorf("sourcePath must not be empty") + } + + if len(sourcePath) > maxJSONPathLength { + return fmt.Errorf("sourcePath exceeds max length of %d", maxJSONPathLength) + } + + if !strings.HasPrefix(sourcePath, ".") { + return fmt.Errorf("sourcePath must start with '.'") + } + + if strings.ContainsAny(sourcePath, "\r\n\t") { + return fmt.Errorf("sourcePath must not contain control whitespace") + } + + return nil +} diff --git a/pkg/runtime/jsonpath/utils_test.go b/pkg/runtime/jsonpath/utils_test.go new file mode 100644 index 00000000..1639e076 --- /dev/null +++ b/pkg/runtime/jsonpath/utils_test.go @@ -0,0 +1,120 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package jsonpath + +import ( + "strings" + "testing" +) + +func TestWrapJSONPath(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + sourcePath string + want string + }{ + { + name: "simple path", + sourcePath: ".spec.resources.requests.cpu", + want: "{.spec.resources.requests.cpu}", + }, + { + name: "empty path", + sourcePath: "", + want: "{}", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := wrapJSONPath(tt.sourcePath) + if got != tt.want { + t.Fatalf("expected %q, got %q", tt.want, got) + } + }) + } +} + +func TestValidateSourcePath(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + sourcePath string + wantErr bool + errMsg string + }{ + { + name: "valid path", + sourcePath: ".spec.resources.requests.cpu", + }, + { + name: "valid minimal path", + sourcePath: ".a", + }, + { + name: "empty path", + sourcePath: "", + wantErr: true, + errMsg: "sourcePath must not be empty", + }, + { + name: "too long", + sourcePath: "." + strings.Repeat("a", maxJSONPathLength), + wantErr: true, + errMsg: "sourcePath exceeds max length", + }, + { + name: "missing dot prefix", + sourcePath: "spec.value", + wantErr: true, + errMsg: "sourcePath must start with '.'", + }, + { + name: "contains carriage return", + sourcePath: ".spec\r.value", + wantErr: true, + errMsg: "sourcePath must not contain control whitespace", + }, + { + name: "contains newline", + sourcePath: ".spec\nvalue", + wantErr: true, + errMsg: "sourcePath must not contain control whitespace", + }, + { + name: "contains tab", + sourcePath: ".spec\tvalue", + wantErr: true, + errMsg: "sourcePath must not contain control whitespace", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := validateSourcePath(tt.sourcePath) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got nil") + } + if !strings.Contains(err.Error(), tt.errMsg) { + t.Fatalf("expected error to contain %q, got %q", tt.errMsg, err.Error()) + } + return + } + + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + }) + } +} diff --git a/pkg/runtime/predicates/config_change.go b/pkg/runtime/predicates/config_change.go index 7cdb46a0..87114670 100644 --- a/pkg/runtime/predicates/config_change.go +++ b/pkg/runtime/predicates/config_change.go @@ -9,13 +9,13 @@ import ( capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" ) -type CapsuleConfigSpecChangedPredicate struct{} +type CapsuleConfigSpecAdministratorsChangedPredicate struct{} -func (CapsuleConfigSpecChangedPredicate) Create(event.CreateEvent) bool { return false } -func (CapsuleConfigSpecChangedPredicate) Delete(event.DeleteEvent) bool { return false } -func (CapsuleConfigSpecChangedPredicate) Generic(event.GenericEvent) bool { return false } +func (CapsuleConfigSpecAdministratorsChangedPredicate) Create(event.CreateEvent) bool { return false } +func (CapsuleConfigSpecAdministratorsChangedPredicate) Delete(event.DeleteEvent) bool { return false } +func (CapsuleConfigSpecAdministratorsChangedPredicate) Generic(event.GenericEvent) bool { return false } -func (CapsuleConfigSpecChangedPredicate) Update(e event.UpdateEvent) bool { +func (CapsuleConfigSpecAdministratorsChangedPredicate) Update(e event.UpdateEvent) bool { oldObj, ok1 := e.ObjectOld.(*capsulev1beta2.CapsuleConfiguration) newObj, ok2 := e.ObjectNew.(*capsulev1beta2.CapsuleConfiguration) @@ -25,3 +25,43 @@ func (CapsuleConfigSpecChangedPredicate) Update(e event.UpdateEvent) bool { return len(oldObj.Spec.Administrators) != len(newObj.Spec.Administrators) } + +type CapsuleConfigSpecImpersonationChangedPredicate struct{} + +func (CapsuleConfigSpecImpersonationChangedPredicate) Create(event.CreateEvent) bool { return false } +func (CapsuleConfigSpecImpersonationChangedPredicate) Delete(event.DeleteEvent) bool { return false } +func (CapsuleConfigSpecImpersonationChangedPredicate) Generic(event.GenericEvent) bool { return false } + +func (CapsuleConfigSpecImpersonationChangedPredicate) Update(e event.UpdateEvent) bool { + oldCfg, ok1 := e.ObjectOld.(*capsulev1beta2.CapsuleConfiguration) + newCfg, ok2 := e.ObjectNew.(*capsulev1beta2.CapsuleConfiguration) + + if !ok1 || !ok2 { + return false + } + + oldSpec := oldCfg.Spec + newSpec := newCfg.Spec + + return oldSpec.Impersonation != newSpec.Impersonation +} + +type CapsuleConfigSpecAdmissionChangedPredicate struct{} + +func (CapsuleConfigSpecAdmissionChangedPredicate) Create(event.CreateEvent) bool { return false } +func (CapsuleConfigSpecAdmissionChangedPredicate) Delete(event.DeleteEvent) bool { return false } +func (CapsuleConfigSpecAdmissionChangedPredicate) Generic(event.GenericEvent) bool { return false } + +func (CapsuleConfigSpecAdmissionChangedPredicate) Update(e event.UpdateEvent) bool { + oldCfg, ok1 := e.ObjectOld.(*capsulev1beta2.CapsuleConfiguration) + newCfg, ok2 := e.ObjectNew.(*capsulev1beta2.CapsuleConfiguration) + + if !ok1 || !ok2 { + return false + } + + oldSpec := oldCfg.Spec + newSpec := newCfg.Spec + + return oldSpec.Admission != newSpec.Admission +} diff --git a/pkg/runtime/predicates/config_change_test.go b/pkg/runtime/predicates/config_change_test.go index 30891603..91743a2c 100644 --- a/pkg/runtime/predicates/config_change_test.go +++ b/pkg/runtime/predicates/config_change_test.go @@ -1,4 +1,4 @@ -// Copyright 2020-2025 Project Capsule Authors +// Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 package predicates_test @@ -9,14 +9,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/runtime/predicates" ) -func TestCapsuleConfigSpecChangedPredicate_StaticFuncs(t *testing.T) { +func TestCapsuleConfigSpecAdministratorsChangedPredicate_StaticFuncs(t *testing.T) { t.Parallel() - p := predicates.CapsuleConfigSpecChangedPredicate{} + p := predicates.CapsuleConfigSpecAdministratorsChangedPredicate{} if got := p.Create(event.CreateEvent{}); got { t.Fatalf("Create() = %v, want false", got) @@ -29,10 +29,10 @@ func TestCapsuleConfigSpecChangedPredicate_StaticFuncs(t *testing.T) { } } -func TestCapsuleConfigSpecChangedPredicate_Update(t *testing.T) { +func TestCapsuleConfigSpecAdministratorsChangedPredicate_Update(t *testing.T) { t.Parallel() - p := predicates.CapsuleConfigSpecChangedPredicate{} + p := predicates.CapsuleConfigSpecAdministratorsChangedPredicate{} t.Run("returns false when types are not CapsuleConfiguration", func(t *testing.T) { t.Parallel() @@ -60,12 +60,12 @@ func TestCapsuleConfigSpecChangedPredicate_Update(t *testing.T) { } // same length (2) - oldObj.Spec.Administrators = []api.UserSpec{ + oldObj.Spec.Administrators = []rbac.UserSpec{ {Name: "a"}, {Name: "b"}, } - newObj.Spec.Administrators = []api.UserSpec{ + newObj.Spec.Administrators = []rbac.UserSpec{ {Name: "x"}, {Name: "y"}, } @@ -82,11 +82,11 @@ func TestCapsuleConfigSpecChangedPredicate_Update(t *testing.T) { oldObj := &capsulev1beta2.CapsuleConfiguration{} newObj := &capsulev1beta2.CapsuleConfiguration{} - oldObj.Spec.Administrators = []api.UserSpec{ + oldObj.Spec.Administrators = []rbac.UserSpec{ {Name: "a"}, } - newObj.Spec.Administrators = []api.UserSpec{ + newObj.Spec.Administrators = []rbac.UserSpec{ {Name: "a"}, {Name: "b"}, } diff --git a/pkg/runtime/predicates/label_present.go b/pkg/runtime/predicates/label_present.go new file mode 100644 index 00000000..5a1829ec --- /dev/null +++ b/pkg/runtime/predicates/label_present.go @@ -0,0 +1,45 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package predicates + +import ( + "sigs.k8s.io/controller-runtime/pkg/event" +) + +type LabelPresentPredicate struct { + Label string +} + +func (p LabelPresentPredicate) Create(e event.CreateEvent) bool { + if e.Object == nil { + return false + } + + _, ok := e.Object.GetLabels()[p.Label] + + return ok +} + +func (p LabelPresentPredicate) Delete(e event.DeleteEvent) bool { + if e.Object == nil { + return false + } + + _, ok := e.Object.GetLabels()[p.Label] + + return ok +} + +func (p LabelPresentPredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + return false + } + + oldVal := e.ObjectOld.GetLabels()[p.Label] + newVal := e.ObjectNew.GetLabels()[p.Label] + + return oldVal != newVal +} + +func (p LabelPresentPredicate) Generic(event.GenericEvent) bool { return false } diff --git a/pkg/runtime/predicates/label_present_test.go b/pkg/runtime/predicates/label_present_test.go new file mode 100644 index 00000000..15bad0d9 --- /dev/null +++ b/pkg/runtime/predicates/label_present_test.go @@ -0,0 +1,298 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package predicates_test + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/event" + + "github.com/projectcapsule/capsule/pkg/runtime/predicates" +) + +func TestLabelPresentPredicate_Create(t *testing.T) { + t.Parallel() + + p := predicates.LabelPresentPredicate{Label: "example.com/watch"} + + tests := []struct { + name string + event event.CreateEvent + want bool + }{ + { + name: "nil object", + event: event.CreateEvent{}, + want: false, + }, + { + name: "label present", + event: event.CreateEvent{ + Object: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Labels: map[string]string{ + "example.com/watch": "true", + }, + }, + }, + }, + want: true, + }, + { + name: "label missing", + event: event.CreateEvent{ + Object: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Labels: map[string]string{"other": "value"}, + }, + }, + }, + want: false, + }, + { + name: "labels map nil", + event: event.CreateEvent{ + Object: &corev1.Namespace{}, + }, + want: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := p.Create(tt.event) + if got != tt.want { + t.Fatalf("Create() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLabelPresentPredicate_Delete(t *testing.T) { + t.Parallel() + + p := predicates.LabelPresentPredicate{Label: "example.com/watch"} + + tests := []struct { + name string + event event.DeleteEvent + want bool + }{ + { + name: "nil object", + event: event.DeleteEvent{}, + want: false, + }, + { + name: "label present", + event: event.DeleteEvent{ + Object: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Labels: map[string]string{ + "example.com/watch": "true", + }, + }, + }, + }, + want: true, + }, + { + name: "label missing", + event: event.DeleteEvent{ + Object: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Labels: map[string]string{"other": "value"}, + }, + }, + }, + want: false, + }, + { + name: "labels map nil", + event: event.DeleteEvent{ + Object: &corev1.Namespace{}, + }, + want: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := p.Delete(tt.event) + if got != tt.want { + t.Fatalf("Delete() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLabelPresentPredicate_Update(t *testing.T) { + t.Parallel() + + p := predicates.LabelPresentPredicate{Label: "example.com/watch"} + + tests := []struct { + name string + event event.UpdateEvent + want bool + }{ + { + name: "nil old object", + event: event.UpdateEvent{ObjectNew: &corev1.Namespace{}}, + want: false, + }, + { + name: "nil new object", + event: event.UpdateEvent{ObjectOld: &corev1.Namespace{}}, + want: false, + }, + { + name: "both objects nil", + event: event.UpdateEvent{}, + want: false, + }, + { + name: "label unchanged and present", + event: event.UpdateEvent{ + ObjectOld: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "example.com/watch": "true", + }, + }, + }, + ObjectNew: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "example.com/watch": "true", + }, + }, + }, + }, + want: false, + }, + { + name: "label changed value", + event: event.UpdateEvent{ + ObjectOld: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "example.com/watch": "old", + }, + }, + }, + ObjectNew: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "example.com/watch": "new", + }, + }, + }, + }, + want: true, + }, + { + name: "label added", + event: event.UpdateEvent{ + ObjectOld: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "other": "value", + }, + }, + }, + ObjectNew: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "example.com/watch": "true", + }, + }, + }, + }, + want: true, + }, + { + name: "label removed", + event: event.UpdateEvent{ + ObjectOld: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "example.com/watch": "true", + }, + }, + }, + ObjectNew: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "other": "value", + }, + }, + }, + }, + want: true, + }, + { + name: "label absent in both", + event: event.UpdateEvent{ + ObjectOld: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "other": "one", + }, + }, + }, + ObjectNew: &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "other": "two", + }, + }, + }, + }, + want: false, + }, + { + name: "nil labels in both", + event: event.UpdateEvent{ + ObjectOld: &corev1.Namespace{}, + ObjectNew: &corev1.Namespace{}, + }, + want: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := p.Update(tt.event) + if got != tt.want { + t.Fatalf("Update() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLabelPresentPredicate_Generic(t *testing.T) { + t.Parallel() + + p := predicates.LabelPresentPredicate{Label: "example.com/watch"} + + if got := p.Generic(event.GenericEvent{}); got { + t.Fatalf("Generic() = %v, want false", got) + } +} diff --git a/pkg/runtime/predicates/promoted_serviceaccount.go b/pkg/runtime/predicates/promoted_serviceaccount.go index 0715ccbf..6f521ca2 100644 --- a/pkg/runtime/predicates/promoted_serviceaccount.go +++ b/pkg/runtime/predicates/promoted_serviceaccount.go @@ -7,6 +7,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/utils" ) type PromotedServiceaccountPredicate struct{} @@ -18,9 +19,15 @@ func (PromotedServiceaccountPredicate) Create(e event.CreateEvent) bool { return false } - v, ok := e.Object.GetLabels()[meta.OwnerPromotionLabel] + if v := e.Object.GetLabels()[meta.OwnerPromotionLabel]; v == meta.ValueTrue { + return true + } - return ok && v == meta.ValueTrue + if v := e.Object.GetLabels()[meta.ServiceAccountPromotionLabel]; v == meta.ValueTrue { + return true + } + + return false } func (PromotedServiceaccountPredicate) Delete(e event.DeleteEvent) bool { @@ -28,9 +35,15 @@ func (PromotedServiceaccountPredicate) Delete(e event.DeleteEvent) bool { return false } - v, ok := e.Object.GetLabels()[meta.OwnerPromotionLabel] + if v := e.Object.GetLabels()[meta.OwnerPromotionLabel]; v == meta.ValueTrue { + return true + } - return ok && v == meta.ValueTrue + if v := e.Object.GetLabels()[meta.ServiceAccountPromotionLabel]; v == meta.ValueTrue { + return true + } + + return false } func (PromotedServiceaccountPredicate) Update(e event.UpdateEvent) bool { @@ -38,8 +51,9 @@ func (PromotedServiceaccountPredicate) Update(e event.UpdateEvent) bool { return false } - oldVal, oldOK := e.ObjectOld.GetLabels()[meta.OwnerPromotionLabel] - newVal, newOK := e.ObjectNew.GetLabels()[meta.OwnerPromotionLabel] + if !utils.MapEqual(e.ObjectOld.GetLabels(), e.ObjectNew.GetLabels()) { + return true + } - return oldOK != newOK || oldVal != newVal + return false } diff --git a/pkg/runtime/predicates/tenant_change.go b/pkg/runtime/predicates/tenant_change.go new file mode 100644 index 00000000..13e38b51 --- /dev/null +++ b/pkg/runtime/predicates/tenant_change.go @@ -0,0 +1,42 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package predicates + +import ( + "sigs.k8s.io/controller-runtime/pkg/event" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/rbac" +) + +type TenantStatusOwnersChangedPredicate struct{} + +func (TenantStatusOwnersChangedPredicate) Create(event.CreateEvent) bool { return false } +func (TenantStatusOwnersChangedPredicate) Delete(event.DeleteEvent) bool { return false } +func (TenantStatusOwnersChangedPredicate) Generic(event.GenericEvent) bool { return false } + +func (TenantStatusOwnersChangedPredicate) Update(e event.UpdateEvent) bool { + oldObj, ok1 := e.ObjectOld.(*capsulev1beta2.Tenant) + newObj, ok2 := e.ObjectNew.(*capsulev1beta2.Tenant) + + if !ok1 || !ok2 { + return false + } + + return ownersChanged(oldObj.Status.Owners, newObj.Status.Owners) +} + +func ownersChanged(a, b rbac.OwnerStatusListSpec) bool { + if len(a) != len(b) { + return true + } + + for i := range a { + if a[i].Name == b[i].Name && a[i].Kind == b[i].Kind { + return true + } + } + + return false +} diff --git a/pkg/runtime/predicates/utils.go b/pkg/runtime/predicates/utils.go new file mode 100644 index 00000000..e1cef2f0 --- /dev/null +++ b/pkg/runtime/predicates/utils.go @@ -0,0 +1,31 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package predicates + +func LabelsEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + + for k, v := range a { + if bv, ok := b[k]; !ok || bv != v { + return false + } + } + + return true +} + +func LabelsChanged(keys []string, oldLabels, newLabels map[string]string) bool { + for _, key := range keys { + oldVal, oldOK := oldLabels[key] + newVal, newOK := newLabels[key] + + if oldOK != newOK || oldVal != newVal { + return true + } + } + + return false +} diff --git a/pkg/runtime/predicates/utils_test.go b/pkg/runtime/predicates/utils_test.go new file mode 100644 index 00000000..c455e286 --- /dev/null +++ b/pkg/runtime/predicates/utils_test.go @@ -0,0 +1,149 @@ +// Copyright 2020-2025 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package predicates_test + +import ( + "testing" + + "github.com/projectcapsule/capsule/pkg/runtime/predicates" +) + +func TestLabelsEqual(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + a, b map[string]string + want bool + }{ + {"both nil", nil, nil, true}, + {"nil and empty", nil, map[string]string{}, true}, + {"same single label", map[string]string{"a": "1"}, map[string]string{"a": "1"}, true}, + {"different value", map[string]string{"a": "1"}, map[string]string{"a": "2"}, false}, + {"different key", map[string]string{"a": "1"}, map[string]string{"b": "1"}, false}, + {"missing key in b", map[string]string{"a": "1", "b": "2"}, map[string]string{"a": "1"}, false}, + {"same multiple labels (order independent)", map[string]string{"b": "2", "a": "1"}, map[string]string{"a": "1", "b": "2"}, true}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := predicates.LabelsEqual(tt.a, tt.b); got != tt.want { + t.Fatalf("labelsEqual(%v, %v) = %v, want %v", tt.a, tt.b, got, tt.want) + } + }) + } +} + +func TestLabelsChanged(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + keys []string + oldLabels map[string]string + newLabels map[string]string + want bool + }{ + { + name: "no keys returns false", + keys: nil, + oldLabels: map[string]string{"a": "1"}, + newLabels: map[string]string{"a": "2"}, + want: false, + }, + { + name: "key absent in both returns false", + keys: []string{"x"}, + oldLabels: map[string]string{"a": "1"}, + newLabels: map[string]string{"a": "2"}, + want: false, + }, + { + name: "key added returns true", + keys: []string{"x"}, + oldLabels: map[string]string{"a": "1"}, + newLabels: map[string]string{"a": "1", "x": "v"}, + want: true, + }, + { + name: "key removed returns true", + keys: []string{"x"}, + oldLabels: map[string]string{"a": "1", "x": "v"}, + newLabels: map[string]string{"a": "1"}, + want: true, + }, + { + name: "key value changed returns true", + keys: []string{"x"}, + oldLabels: map[string]string{"x": "v1"}, + newLabels: map[string]string{"x": "v2"}, + want: true, + }, + { + name: "key unchanged returns false", + keys: []string{"x"}, + oldLabels: map[string]string{"x": "v"}, + newLabels: map[string]string{"x": "v"}, + want: false, + }, + { + name: "multiple keys returns true if any tracked key changed", + keys: []string{"a", "b", "c"}, + oldLabels: map[string]string{"a": "1", "b": "2", "c": "3"}, + newLabels: map[string]string{"a": "1", "b": "CHANGED", "c": "3"}, + want: true, + }, + { + name: "multiple keys returns false if none of tracked keys changed even if other labels changed", + keys: []string{"a", "b"}, + oldLabels: map[string]string{"a": "1", "b": "2", "other": "x"}, + newLabels: map[string]string{"a": "1", "b": "2", "other": "y"}, + want: false, + }, + { + name: "nil maps behave like empty maps (added triggers true)", + keys: []string{"x"}, + oldLabels: nil, + newLabels: map[string]string{"x": "v"}, + want: true, + }, + { + name: "nil maps behave like empty maps (both nil no change -> false)", + keys: []string{"x"}, + oldLabels: nil, + newLabels: nil, + want: false, + }, + { + name: "empty string value vs missing key counts as change", + keys: []string{"x"}, + oldLabels: map[string]string{"x": ""}, + newLabels: map[string]string{}, + want: true, + }, + { + name: "duplicate keys still works (change detected once)", + keys: []string{"x", "x"}, + oldLabels: map[string]string{"x": "v1"}, + newLabels: map[string]string{"x": "v2"}, + want: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := predicates.LabelsChanged(tt.keys, tt.oldLabels, tt.newLabels) + if got != tt.want { + t.Fatalf("LabelsChanged(%v, %v, %v) = %v, want %v", + tt.keys, tt.oldLabels, tt.newLabels, got, tt.want) + } + }) + } +} diff --git a/pkg/runtime/quota/custom_quota.go b/pkg/runtime/quota/custom_quota.go new file mode 100644 index 00000000..7671c339 --- /dev/null +++ b/pkg/runtime/quota/custom_quota.go @@ -0,0 +1,32 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package quota + +import ( + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/projectcapsule/capsule/pkg/runtime/jsonpath" +) + +// Overlay for Global/Namespace CustomQuotas. +type MatchedQuota struct { + Key string + Name string + Namespace string + Path string + CompiledPath *jsonpath.CompiledJSONPath + Operation Operation + Limit resource.Quantity + Used resource.Quantity + IsGlobal bool + SourceRank int +} + +func MakeCustomQuotaCacheKey(namespace, name string) string { + return namespace + "/" + name +} + +func MakeGlobalCustomQuotaCacheKey(name string) string { + return "C/" + name +} diff --git a/pkg/runtime/quota/operation.go b/pkg/runtime/quota/operation.go new file mode 100644 index 00000000..8a18d743 --- /dev/null +++ b/pkg/runtime/quota/operation.go @@ -0,0 +1,13 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package quota + +// +kubebuilder:validation:Enum=add;sub;count +type Operation string + +const ( + OpAdd Operation = "add" + OpSub Operation = "sub" + OpCount Operation = "count" +) diff --git a/pkg/runtime/quota/parse.go b/pkg/runtime/quota/parse.go new file mode 100644 index 00000000..d504136a --- /dev/null +++ b/pkg/runtime/quota/parse.go @@ -0,0 +1,89 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package quota + +import ( + "fmt" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/projectcapsule/capsule/pkg/runtime/jsonpath" +) + +func ParseBoolFromUnstructured(u unstructured.Unstructured, compiled *jsonpath.CompiledJSONPath) (bool, error) { + value, err := ParseUsageFromUnstructured(u, compiled) + if err != nil { + return false, err + } + + value = strings.TrimSpace(value) + if value == "" { + return false, nil + } + + parsed, err := strconv.ParseBool(value) + if err != nil { + return false, fmt.Errorf("condition path %v did not resolve to a boolean, got %q: %w", compiled, value, err) + } + + return parsed, nil +} + +func ConditionsMatch(u unstructured.Unstructured, conditions []*jsonpath.CompiledJSONPath) (bool, error) { + for _, cond := range conditions { + ok, err := ParseBoolFromUnstructured(u, cond) + if err != nil { + return false, err + } + + if !ok { + return false, nil + } + } + + return true, nil +} + +func ParseQuantities(value string) (resource.Quantity, error) { + fields := strings.Fields(value) + if len(fields) == 0 { + return resource.Quantity{}, fmt.Errorf("no quantity values found") + } + + total := resource.Quantity{} + + for _, v := range fields { + q, err := resource.ParseQuantity(v) + if err != nil { + return total, fmt.Errorf("invalid quantity %q: %w", v, err) + } + + total.Add(q) + } + + return total, nil +} + +// GetUsageFromUnstructured extracts a value from an unstructured object using a JSONPath source path. +// It is convenient for one-off calls. For repeated calls with the same sourcePath, prefer +// CompileUsageJSONPath(...) and then Execute(...) to avoid reparsing. +func ParseUsageFromUnstructured(u unstructured.Unstructured, compiled *jsonpath.CompiledJSONPath) (string, error) { + return compiled.Execute(u) +} + +func ParseQuantityFromUnstructured(u unstructured.Unstructured, compiled *jsonpath.CompiledJSONPath) (resource.Quantity, error) { + usage, err := ParseUsageFromUnstructured(u, compiled) + if err != nil { + return resource.Quantity{}, err + } + + if strings.TrimSpace(usage) == "" { + return resource.Quantity{}, fmt.Errorf("quantity path did not resolve to any value") + } + + return ParseQuantities(usage) +} diff --git a/pkg/runtime/quota/parse_test.go b/pkg/runtime/quota/parse_test.go new file mode 100644 index 00000000..45643dd1 --- /dev/null +++ b/pkg/runtime/quota/parse_test.go @@ -0,0 +1,324 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package quota_test + +import ( + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/projectcapsule/capsule/pkg/runtime/jsonpath" + "github.com/projectcapsule/capsule/pkg/runtime/quota" +) + +func contains(s, substr string) bool { + return len(substr) == 0 || (len(s) >= len(substr) && stringContains(s, substr)) +} + +func stringContains(s, substr string) bool { + for i := 0; i+len(substr) <= len(s); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} +func TestParseQuantities(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + want resource.Quantity + wantErr bool + errContains string + }{ + { + name: "empty string returns error", + input: "", + wantErr: true, + errContains: "no quantity values found", + }, + { + name: "whitespace only returns error", + input: " \n\t ", + wantErr: true, + errContains: "no quantity values found", + }, + { + name: "single quantity", + input: "100m", + want: resource.MustParse("100m"), + wantErr: false, + }, + { + name: "multiple cpu quantities", + input: "100m 200m 300m", + want: resource.MustParse("600m"), + wantErr: false, + }, + { + name: "multiple memory quantities", + input: "128Mi 256Mi 1Gi", + want: resource.MustParse("1408Mi"), + wantErr: false, + }, + { + name: "mixed whitespace separators", + input: "100m\t200m\n300m", + want: resource.MustParse("600m"), + wantErr: false, + }, + { + name: "equivalent units", + input: "1Gi 512Mi 512Mi", + want: resource.MustParse("2Gi"), + wantErr: false, + }, + { + name: "invalid quantity returns error", + input: "100m invalid 200m", + wantErr: true, + errContains: `invalid quantity "invalid"`, + }, + { + name: "completely invalid input returns error", + input: "nope", + wantErr: true, + errContains: `invalid quantity "nope"`, + }, + { + name: "decimal SI quantities", + input: "1 2 3", + want: resource.MustParse("6"), + wantErr: false, + }, + { + name: "binary SI quantities", + input: "1Ki 2Ki 3Ki", + want: resource.MustParse("6Ki"), + wantErr: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := quota.ParseQuantities(tt.input) + + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got nil") + } + if tt.errContains != "" && !contains(err.Error(), tt.errContains) { + t.Fatalf("expected error to contain %q, got %q", tt.errContains, err.Error()) + } + return + } + + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if got.Cmp(tt.want) != 0 { + t.Fatalf("expected quantity %q, got %q", tt.want.String(), got.String()) + } + }) + } +} + +func TestParseQuantityFromUnstructured_Success(t *testing.T) { + t.Parallel() + + u := unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "resources": map[string]interface{}{ + "requests": map[string]interface{}{ + "cpu": "250m", + }, + }, + }, + }, + } + + jp, err := jsonpath.CompileJSONPath(".spec.resources.requests.cpu") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + got, err := quota.ParseQuantityFromUnstructured(u, jp) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + want := resource.MustParse("250m") + if got.Cmp(want) != 0 { + t.Fatalf("expected quantity %q, got %q", want.String(), got.String()) + } +} + +func TestParseQuantityFromUnstructured_MissingPathReturnsError(t *testing.T) { + t.Parallel() + + u := unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{}, + }, + } + + jp, err := jsonpath.CompileJSONPath(".spec.resources.requests.cpu") + if err != nil { + t.Fatalf("expected no error compiling jsonpath, got %v", err) + } + + _, err = quota.ParseQuantityFromUnstructured(u, jp) + if err == nil { + t.Fatal("expected error, got nil") + } + + if !contains(err.Error(), "did not resolve to any value") && + !contains(err.Error(), "no quantity values found") { + t.Fatalf("expected missing quantity error, got %q", err.Error()) + } +} + +func TestParseQuantityFromUnstructured_WhitespaceOnlyReturnsError(t *testing.T) { + t.Parallel() + + u := unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "value": " \n\t ", + }, + }, + } + + jp, err := jsonpath.CompileJSONPath(".spec.value") + if err != nil { + t.Fatalf("expected no error compiling jsonpath, got %v", err) + } + + _, err = quota.ParseQuantityFromUnstructured(u, jp) + if err == nil { + t.Fatal("expected error, got nil") + } + + if !contains(err.Error(), "did not resolve to any value") && + !contains(err.Error(), "no quantity values found") { + t.Fatalf("expected empty quantity error, got %q", err.Error()) + } +} + +func TestParseUsageFromUnstructured_Success(t *testing.T) { + u := unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "resources": map[string]interface{}{ + "requests": map[string]interface{}{ + "cpu": "250m", + "memory": "512Mi", + }, + }, + }, + }, + } + + jp, err := jsonpath.CompileJSONPath(".spec.resources.requests.cpu") + if err != nil { + t.Fatal("expected no error, got error", err) + } + + got, err := quota.ParseUsageFromUnstructured(u, jp) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if got != "250m" { + t.Fatalf("expected %q, got %q", "250m", got) + } +} + +func TestParseUsageFromUnstructured_TrimsWhitespace(t *testing.T) { + u := unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "value": " hello ", + }, + }, + } + + jp, err := jsonpath.CompileJSONPath(" .spec.value") + if err != nil { + t.Fatal("expected no error, got error", err) + } + + got, err := quota.ParseUsageFromUnstructured(u, jp) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if got != "hello" { + t.Fatalf("expected %q, got %q", "hello", got) + } +} + +func TestParseUsageFromUnstructured_MissingPath(t *testing.T) { + t.Parallel() + + u := unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{}, + }, + } + + jp, err := jsonpath.CompileJSONPath(".spec.resources.requests.cpu") + if err != nil { + t.Fatalf("expected no error compiling jsonpath, got %v", err) + } + + _, err = quota.ParseUsageFromUnstructured(u, jp) + if err != nil { + t.Fatal("expected no error, got err") + } +} + +func TestParseUsageFromUnstructured_InvalidJSONPath(t *testing.T) { + t.Parallel() + + _, err := jsonpath.CompileJSONPath(".spec[") + if err == nil { + t.Fatal("expected compile error, got nil") + } +} + +func TestParseUsageFromUnstructured_EmptySourcePath(t *testing.T) { + t.Parallel() + + _, err := jsonpath.CompileJSONPath("") + if err == nil { + t.Fatal("expected compile error, got nil") + } +} + +func TestParseUsageFromUnstructured_SourcePathMustStartWithDot(t *testing.T) { + t.Parallel() + + _, err := jsonpath.CompileJSONPath("spec.resources.requests.cpu") + if err == nil { + t.Fatal("expected compile error, got nil") + } +} + +func TestParseUsageFromUnstructured_RejectsControlWhitespace(t *testing.T) { + t.Parallel() + + _, err := jsonpath.CompileJSONPath(".spec.\nrequests.cpu") + if err == nil { + t.Fatal("expected compile error, got nil") + } +} diff --git a/pkg/runtime/quota/utils.go b/pkg/runtime/quota/utils.go new file mode 100644 index 00000000..9a35d8a3 --- /dev/null +++ b/pkg/runtime/quota/utils.go @@ -0,0 +1,36 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package quota + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/api/resource" +) + +func ValidateQuantity(q resource.Quantity) error { + parsed, err := resource.ParseQuantity(q.String()) + if err != nil { + return fmt.Errorf("invalid quantity %q: %w", q.String(), err) + } + + if parsed.Sign() <= 0 { + return fmt.Errorf("quantity must not be negative or 0: %q", q.String()) + } + + return nil +} + +func ClampQuantityToZero(q *resource.Quantity) { + if q.Sign() < 0 { + *q = resource.MustParse("0") + } +} + +func NegateQuantity(in resource.Quantity) resource.Quantity { + out := in.DeepCopy() + out.Neg() + + return out +} diff --git a/pkg/runtime/sanitize/object.go b/pkg/runtime/sanitize/object.go index 3382c46e..b6d1c745 100644 --- a/pkg/runtime/sanitize/object.go +++ b/pkg/runtime/sanitize/object.go @@ -25,6 +25,18 @@ func SanitizeObject(obj client.Object, scheme *runtime.Scheme, opts SanitizeOpti obj.SetUID("") } + if opts.StripResourceVersion { + obj.SetResourceVersion("") + } + + if opts.StripOwnerreferences { + obj.SetOwnerReferences(nil) + } + + if opts.StripGeneration { + obj.SetGeneration(0) + } + if opts.StripManagedFields { accessor, err := apiMeta.Accessor(obj) if err == nil { diff --git a/pkg/runtime/sanitize/options.go b/pkg/runtime/sanitize/options.go index 400082d4..fd5e6ac2 100644 --- a/pkg/runtime/sanitize/options.go +++ b/pkg/runtime/sanitize/options.go @@ -4,17 +4,23 @@ package sanitize type SanitizeOptions struct { - StripUID bool - StripManagedFields bool - StripLastApplied bool - StripStatus bool + StripResourceVersion bool + StripOwnerreferences bool + StripGeneration bool + StripUID bool + StripManagedFields bool + StripLastApplied bool + StripStatus bool } func DefaultSanitizeOptions() SanitizeOptions { return SanitizeOptions{ - StripUID: true, - StripManagedFields: true, - StripLastApplied: true, - StripStatus: true, + StripResourceVersion: true, + StripOwnerreferences: true, + StripGeneration: true, + StripUID: true, + StripManagedFields: true, + StripLastApplied: true, + StripStatus: true, } } diff --git a/pkg/runtime/selectors/combine.go b/pkg/runtime/selectors/combine.go index c032db12..981ddcc5 100644 --- a/pkg/runtime/selectors/combine.go +++ b/pkg/runtime/selectors/combine.go @@ -15,7 +15,6 @@ func CombineSelectors(selectors ...labels.Selector) labels.Selector { reqs, selectable := sel.Requirements() if !selectable { - // Defensive: if selector can't be expressed as requirements, match nothing. return labels.Nothing() } diff --git a/pkg/runtime/selectors/fields.go b/pkg/runtime/selectors/fields.go new file mode 100644 index 00000000..0dc78bef --- /dev/null +++ b/pkg/runtime/selectors/fields.go @@ -0,0 +1,27 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package selectors + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + "github.com/projectcapsule/capsule/pkg/runtime/jsonpath" +) + +// +kubebuilder:object:generate=true +type SelectorWithFields struct { + // Select Items based on their labels. + *metav1.LabelSelector `json:",inline"` + + // Additional boolean JSONPath expressions. + // All must evaluate to true for this selector to match. + // +optional + FieldSelectors []string `json:"fieldSelectors,omitempty"` +} + +type CompiledSelectorWithFields struct { + LabelSelector labels.Selector + FieldMatchers []*jsonpath.CompiledJSONPath +} diff --git a/pkg/runtime/selectors/list_selectors.go b/pkg/runtime/selectors/list_selectors.go index fe1e1bba..fe282344 100644 --- a/pkg/runtime/selectors/list_selectors.go +++ b/pkg/runtime/selectors/list_selectors.go @@ -17,7 +17,7 @@ import ( // match ANY of the provided LabelSelectors. The result is unique by namespace/name. func ListBySelectors[T client.Object]( ctx context.Context, - c client.Client, + c client.Reader, list client.ObjectList, selectors []*metav1.LabelSelector, ) ([]T, error) { diff --git a/pkg/runtime/selectors/matches.go b/pkg/runtime/selectors/matches.go new file mode 100644 index 00000000..286ee1c4 --- /dev/null +++ b/pkg/runtime/selectors/matches.go @@ -0,0 +1,45 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package selectors + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" +) + +// Attempt so verify multiple selector objects against a labels.Set +// If selectors are not set, it is always considered a match. +func MatchesSelectors(objLabels labels.Set, selectors []metav1.LabelSelector) bool { + if len(selectors) == 0 { + return true + } + + for _, selector := range selectors { + sel, err := metav1.LabelSelectorAsSelector(&selector) + if err != nil { + continue + } + + if sel.Matches(objLabels) { + return true + } + } + + return false +} + +// Attempt so verify multiple selector objects against a labels.Set +// If selectors are not set, it is always considered a match. +func MatchesSelector(objLabels labels.Set, selector metav1.LabelSelector) (bool, error) { + sel, err := metav1.LabelSelectorAsSelector(&selector) + if err != nil { + return false, err + } + + if sel.Matches(objLabels) { + return true, nil + } + + return false, nil +} diff --git a/pkg/runtime/selectors/namespaced_selectors.go b/pkg/runtime/selectors/namespaced_selectors.go index f75e0745..69f97101 100644 --- a/pkg/runtime/selectors/namespaced_selectors.go +++ b/pkg/runtime/selectors/namespaced_selectors.go @@ -6,6 +6,7 @@ package selectors import ( "context" "fmt" + "sort" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -24,10 +25,10 @@ type NamespaceSelector struct { // GetMatchingNamespaces retrieves the list of namespaces that match the NamespaceSelector. func (s *NamespaceSelector) GetMatchingNamespaces( ctx context.Context, - c client.Client, + c client.Reader, ) ([]corev1.Namespace, error) { if s.LabelSelector == nil { - return nil, nil // No namespace selector means all namespaces + return nil, nil } nsSelector, err := metav1.LabelSelectorAsSelector(s.LabelSelector) @@ -51,6 +52,62 @@ func (s *NamespaceSelector) GetMatchingNamespaces( return matchingNamespaces, nil } +// Takes a list of NamespaceSelectors and returns unique ordered Namespaces. +func GetNamespacesMatchingSelectors( + ctx context.Context, + c client.Reader, + namespaceSelector []NamespaceSelector, +) (namespaces []corev1.Namespace, err error) { + if len(namespaceSelector) == 0 { + return nil, nil + } + + byName := make(map[string]corev1.Namespace) + + for _, selector := range namespaceSelector { + matches, err := selector.GetMatchingNamespaces(ctx, c) + if err != nil { + return nil, err + } + + for _, ns := range matches { + byName[ns.Name] = ns + } + } + + names := make([]string, 0, len(byName)) + for name := range byName { + names = append(names, name) + } + + sort.Strings(names) + + result := make([]corev1.Namespace, 0, len(names)) + for _, name := range names { + result = append(result, byName[name]) + } + + return result, nil +} + +func GetNamespacesMatchingSelectorsStrings( + ctx context.Context, + c client.Reader, + namespaceSelector []NamespaceSelector, +) ([]string, error) { + namespaces, err := GetNamespacesMatchingSelectors(ctx, c, namespaceSelector) + if err != nil { + return nil, err + } + + result := make([]string, 0, len(namespaces)) + for _, ns := range namespaces { + result = append(result, ns.Name) + } + + return result, nil +} + // Selector for resources and their labels or selecting origin namespaces // +kubebuilder:object:generate=true type SelectorWithNamespaceSelector struct { @@ -64,7 +121,7 @@ type SelectorWithNamespaceSelector struct { func (s *SelectorWithNamespaceSelector) MatchObjects( ctx context.Context, - c client.Client, + c client.Reader, objects []metav1.Object, ) ([]metav1.Object, error) { if s == nil { diff --git a/pkg/runtime/selectors/zz_generated.deepcopy.go b/pkg/runtime/selectors/zz_generated.deepcopy.go index 5a2d9983..6138f3c7 100644 --- a/pkg/runtime/selectors/zz_generated.deepcopy.go +++ b/pkg/runtime/selectors/zz_generated.deepcopy.go @@ -31,6 +31,31 @@ func (in *NamespaceSelector) DeepCopy() *NamespaceSelector { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SelectorWithFields) DeepCopyInto(out *SelectorWithFields) { + *out = *in + if in.LabelSelector != nil { + in, out := &in.LabelSelector, &out.LabelSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.FieldSelectors != nil { + in, out := &in.FieldSelectors, &out.FieldSelectors + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SelectorWithFields. +func (in *SelectorWithFields) DeepCopy() *SelectorWithFields { + if in == nil { + return nil + } + out := new(SelectorWithFields) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SelectorWithNamespaceSelector) DeepCopyInto(out *SelectorWithNamespaceSelector) { *out = *in diff --git a/pkg/template/context.go b/pkg/template/context.go new file mode 100644 index 00000000..32703226 --- /dev/null +++ b/pkg/template/context.go @@ -0,0 +1,98 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package template + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + + k8smeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/projectcapsule/capsule/pkg/runtime/sanitize" +) + +// Additional Context to enhance templating +// +kubebuilder:object:generate=true +type TemplateContext struct { + Resources []*TemplateResourceReference `json:"resources,omitempty"` +} + +func (t *TemplateContext) GatherContext( + ctx context.Context, + kubeClient client.Client, + restMapper k8smeta.RESTMapper, + templateContext map[string]string, + namespace string, + additionSelectors []labels.Selector, + validateNamespace NamespaceValidator, +) (ReferenceContext, error) { + result := ReferenceContext{} + + if t.Resources == nil { + return result, nil + } + + var errs []error + + // Load external resources + for index, resource := range t.Resources { + res, err := resource.LoadResources( + ctx, + kubeClient, + restMapper, + namespace, + additionSelectors, + templateContext, + true, + validateNamespace, + ) + if err != nil { + errs = append(errs, err) + + continue + } + + if len(res) == 0 { + continue + } + + resourceIndex := resource.Index + if resourceIndex == "" { + resourceIndex = strconv.Itoa(index) + } + + items := make([]map[string]any, 0, len(res)) + + for _, u := range res { + sanitize.SanitizeUnstructured(u, sanitize.DefaultSanitizeOptions()) + + items = append(items, u.UnstructuredContent()) + } + + result[resourceIndex] = items + } + + return result, errors.Join(errs...) +} + +// +kubebuilder:object:generate=false +type ReferenceContext map[string]any + +func (t *ReferenceContext) String() (string, error) { + dataBytes, err := json.Marshal(t) + if err != nil { + return "", fmt.Errorf("error marshaling TemplateContext: %w", err) + } + + if err := json.Unmarshal(dataBytes, t); err != nil { + return "", fmt.Errorf("error unmarshaling TemplateContext into map: %w", err) + } + + return string(dataBytes), nil +} diff --git a/pkg/template/fast.go b/pkg/template/fast.go index 4b611aa1..dcf6fd33 100644 --- a/pkg/template/fast.go +++ b/pkg/template/fast.go @@ -6,18 +6,109 @@ package template import ( "fmt" "io" + "regexp" "slices" "strings" "github.com/valyala/fasttemplate" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" ) +var AllowedNamespaceMetadataTemplates = sets.New[string]( + "tenant.name", + "namespace", +) + +var FastTemplateExpression = regexp.MustCompile(`{{\s*([^{}]+)\s*}}`) + +func ValidateKubernetesStringOrAllowedTemplates( + fieldPath string, + value string, + validate func(string) []string, +) []string { + checkValue, errs := validateAllowedTemplatesAndReplace(fieldPath, value) + if len(errs) > 0 { + return errs + } + + return prefixValidationErrors(fieldPath, validate(checkValue)) +} + +func ValidateAllowedTemplatesOnly( + fieldPath string, + value string, +) []string { + _, errs := validateAllowedTemplatesAndReplace(fieldPath, value) + + return errs +} + +func validateAllowedTemplatesAndReplace( + fieldPath string, + value string, +) (string, []string) { + if !ContainsFastTemplateSyntax(value) { + return value, nil + } + + matches := FastTemplateExpression.FindAllStringSubmatch(value, -1) + if len(matches) == 0 { + return value, []string{ + fmt.Sprintf("%s: malformed template %q", fieldPath, value), + } + } + + checkValue := value + + for _, match := range matches { + raw := match[0] + name := strings.TrimSpace(match[1]) + + if !AllowedNamespaceMetadataTemplates.Has(name) { + return value, []string{ + fmt.Sprintf( + "%s: unsupported template %q in %q, allowed templates are {{tenant.name}} and {{namespace}}", + fieldPath, + name, + value, + ), + } + } + + checkValue = strings.ReplaceAll(checkValue, raw, "template") + } + + if strings.Contains(checkValue, "{{") || strings.Contains(checkValue, "}}") { + return value, []string{ + fmt.Sprintf("%s: malformed template %q", fieldPath, value), + } + } + + return checkValue, nil +} + +func prefixValidationErrors(fieldPath string, messages []string) []string { + if len(messages) == 0 { + return nil + } + + errs := make([]string, 0, len(messages)) + + for _, msg := range messages { + errs = append(errs, fmt.Sprintf("%s: %s", fieldPath, msg)) + } + + return errs +} + +func ContainsFastTemplateSyntax(value string) bool { + return strings.Contains(value, "{{") || strings.Contains(value, "}}") +} + // RequiresFastTemplate evaluates if given string requires templating. -func RequiresFastTemplate( - template string, -) bool { - return strings.Contains(template, "{{") && strings.Contains(template, "}}") +func RequiresFastTemplate(value string) bool { + return strings.Contains(value, "{{") && strings.Contains(value, "}}") } // FastTemplate applies templating to the provided string. @@ -51,6 +142,7 @@ func FastTemplateMap( } out := make(map[string]string, len(m)) + for k, v := range m { out[FastTemplate(k, templateContext)] = FastTemplate(v, templateContext) } @@ -58,7 +150,6 @@ func FastTemplateMap( return out } -// FastTemplateMap evaluates if given LabelSelector requires templating. func SelectorRequiresTemplating(sel *metav1.LabelSelector) bool { if sel == nil { return false @@ -83,7 +174,6 @@ func SelectorRequiresTemplating(sel *metav1.LabelSelector) bool { return false } -// FastTemplateMap templates a Labelselector (all keys and values). func FastTemplateLabelSelector( in *metav1.LabelSelector, templateContext map[string]string, @@ -93,7 +183,6 @@ func FastTemplateLabelSelector( } out := in.DeepCopy() - out.MatchLabels = FastTemplateMap(in.MatchLabels, templateContext) for i := range out.MatchExpressions { diff --git a/pkg/template/funcmap.go b/pkg/template/funcmap.go deleted file mode 100644 index 2d4f4627..00000000 --- a/pkg/template/funcmap.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2020-2026 Project Capsule Authors -// SPDX-License-Identifier: Apache-2.0 -package template - -import ( - "bytes" - "encoding/json" - "maps" - "strings" - "text/template" - - "github.com/BurntSushi/toml" - "github.com/go-sprout/sprout/sprigin" - "sigs.k8s.io/yaml" -) - -// TxtFuncMap returns an aggregated template function map. Currently (custom functions + sprig). -func ExtraFuncMap() template.FuncMap { - funcMap := sprigin.FuncMap() - - extraFuncs := template.FuncMap{ - "toToml": toTOML, - "fromToml": fromTOML, - "toYaml": toYAML, - "fromYaml": fromYAML, - "fromYamlArray": fromYAMLArray, - "toJson": toJSON, - "fromJson": fromJSON, - "fromJsonArray": fromJSONArray, - } - - maps.Copy(funcMap, extraFuncs) - - return funcMap -} - -// toYAML takes an interface, marshals it to yaml, and returns a string. It will -// always return a string, even on marshal error (empty string). -// -// This is designed to be called from a template. -func toYAML(v any) string { - data, err := yaml.Marshal(v) - if err != nil { - // Swallow errors inside of a template. - return "" - } - - return strings.TrimSuffix(string(data), "\n") -} - -// fromYAML converts a YAML document into a map[string]interface{}. -// -// This is not a general-purpose YAML parser, and will not parse all valid -// YAML documents. Additionally, because its intended use is within templates -// it tolerates errors. It will insert the returned error message string into -// m["Error"] in the returned map. -func fromYAML(str string) map[string]any { - m := map[string]any{} - - if err := yaml.Unmarshal([]byte(str), &m); err != nil { - m["Error"] = err.Error() - } - - return m -} - -// fromYAMLArray converts a YAML array into a []interface{}. -// -// This is not a general-purpose YAML parser, and will not parse all valid -// YAML documents. Additionally, because its intended use is within templates -// it tolerates errors. It will insert the returned error message string as -// the first and only item in the returned array. -func fromYAMLArray(str string) []any { - a := []any{} - - if err := yaml.Unmarshal([]byte(str), &a); err != nil { - a = []any{err.Error()} - } - - return a -} - -// toTOML takes an interface, marshals it to toml, and returns a string. It will -// always return a string, even on marshal error (empty string). -// -// This is designed to be called from a template. -func toTOML(v any) string { - b := bytes.NewBuffer(nil) - e := toml.NewEncoder(b) - - err := e.Encode(v) - if err != nil { - return err.Error() - } - - return b.String() -} - -// fromTOML converts a TOML document into a map[string]interface{}. -// -// This is not a general-purpose TOML parser, and will not parse all valid -// TOML documents. Additionally, because its intended use is within templates -// it tolerates errors. It will insert the returned error message string into -// m["Error"] in the returned map. -func fromTOML(str string) map[string]any { - m := make(map[string]any) - - if err := toml.Unmarshal([]byte(str), &m); err != nil { - m["Error"] = err.Error() - } - - return m -} - -// toJSON takes an interface, marshals it to json, and returns a string. It will -// always return a string, even on marshal error (empty string). -// -// This is designed to be called from a template. -func toJSON(v any) string { - data, err := json.Marshal(v) - if err != nil { - // Swallow errors inside of a template. - return "" - } - - return string(data) -} - -// fromJSON converts a JSON document into a map[string]interface{}. -// -// This is not a general-purpose JSON parser, and will not parse all valid -// JSON documents. Additionally, because its intended use is within templates -// it tolerates errors. It will insert the returned error message string into -// m["Error"] in the returned map. -func fromJSON(str string) map[string]any { - m := make(map[string]any) - - if err := json.Unmarshal([]byte(str), &m); err != nil { - m["Error"] = err.Error() - } - - return m -} - -// fromJSONArray converts a JSON array into a []interface{}. -// -// This is not a general-purpose JSON parser, and will not parse all valid -// JSON documents. Additionally, because its intended use is within templates -// it tolerates errors. It will insert the returned error message string as -// the first and only item in the returned array. -func fromJSONArray(str string) []any { - a := []any{} - - if err := json.Unmarshal([]byte(str), &a); err != nil { - a = []any{err.Error()} - } - - return a -} diff --git a/pkg/template/functions/age.go b/pkg/template/functions/age.go new file mode 100644 index 00000000..ddd3e663 --- /dev/null +++ b/pkg/template/functions/age.go @@ -0,0 +1,43 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 +package functions + +import ( + "filippo.io/age" +) + +type AgeKeyPair struct { + // Identity is the private key, e.g. AGE-SECRET-KEY-1... + Identity string `json:"identity" yaml:"identity"` + + // Recipient is the public key, e.g. age1... + Recipient string `json:"recipient" yaml:"recipient"` +} + +func generateAgeKey() any { + identity, err := age.GenerateX25519Identity() + if err != nil { + return map[string]any{ + "Error": err.Error(), + } + } + + return AgeKeyPair{ + Identity: identity.String(), + Recipient: identity.Recipient().String(), + } +} + +func generateAgePQKey() any { + identity, err := age.GenerateHybridIdentity() + if err != nil { + return map[string]any{ + "Error": err.Error(), + } + } + + return AgeKeyPair{ + Identity: identity.String(), + Recipient: identity.Recipient().String(), + } +} diff --git a/pkg/template/functions/conversion.go b/pkg/template/functions/conversion.go new file mode 100644 index 00000000..01aafa47 --- /dev/null +++ b/pkg/template/functions/conversion.go @@ -0,0 +1,78 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 +package functions + +import ( + "bytes" + "encoding/json" + + "github.com/BurntSushi/toml" + "sigs.k8s.io/yaml" +) + +// fromYAMLArray converts a YAML array into a []interface{}. +// +// This is not a general-purpose YAML parser, and will not parse all valid +// YAML documents. Additionally, because its intended use is within templates +// it tolerates errors. It will insert the returned error message string as +// the first and only item in the returned array. +func fromYAMLArray(str string) []any { + a := []any{} + + if err := yaml.Unmarshal([]byte(str), &a); err != nil { + a = []any{err.Error()} + } + + return a +} + +// toTOML takes an interface, marshals it to toml, and returns a string. It will +// always return a string, even on marshal error (empty string). +// +// This is designed to be called from a template. +func toTOML(v any) string { + if v == nil { + return "" + } + + b := bytes.NewBuffer(nil) + e := toml.NewEncoder(b) + + if err := e.Encode(v); err != nil { + return err.Error() + } + + return b.String() +} + +// fromTOML converts a TOML document into a map[string]interface{}. +// +// This is not a general-purpose TOML parser, and will not parse all valid +// TOML documents. Additionally, because its intended use is within templates +// it tolerates errors. It will insert the returned error message string into +// m["Error"] in the returned map. +func fromTOML(str string) map[string]any { + m := make(map[string]any) + + if err := toml.Unmarshal([]byte(str), &m); err != nil { + m["Error"] = err.Error() + } + + return m +} + +// fromJSONArray converts a JSON array into a []interface{}. +// +// This is not a general-purpose JSON parser, and will not parse all valid +// JSON documents. Additionally, because its intended use is within templates +// it tolerates errors. It will insert the returned error message string as +// the first and only item in the returned array. +func fromJSONArray(str string) []any { + a := []any{} + + if err := json.Unmarshal([]byte(str), &a); err != nil { + a = []any{err.Error()} + } + + return a +} diff --git a/pkg/template/functions/conversion_test.go b/pkg/template/functions/conversion_test.go new file mode 100644 index 00000000..f4422058 --- /dev/null +++ b/pkg/template/functions/conversion_test.go @@ -0,0 +1,153 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package functions + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestToTOML(t *testing.T) { + tests := []struct { + name string + in any + expectError bool + }{ + { + name: "encodes simple map", + in: map[string]any{ + "a": "b", + "n": int64(3), + }, + expectError: false, + }, + { + name: "encodes nil as empty string or valid toml", + in: nil, + expectError: false, + }, + { + name: "returns error string on unsupported type (function)", + in: func() {}, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := toTOML(tt.in) + require.NotNil(t, got) + + if tt.expectError { + // Encoder errors are returned as strings, so just assert it's non-empty and looks like an error. + assert.NotEmpty(t, got) + return + } + + // For successful encodes, output should be non-empty for most values. + // (nil may encode to empty) + if tt.in != nil { + assert.NotEmpty(t, got) + } + }) + } +} + +func TestFromTOML(t *testing.T) { + tests := []struct { + name string + in string + expectError bool + wantKeys map[string]any + }{ + { + name: "valid toml", + in: "a = \"b\"\nn = 3\n", + wantKeys: map[string]any{ + "a": "b", + // go-toml commonly decodes numbers as int64 + "n": int64(3), + }, + }, + { + name: "invalid toml sets Error key", + in: "a = ", + expectError: true, + }, + { + name: "empty string yields empty map (no Error)", + in: "", + wantKeys: map[string]any{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := fromTOML(tt.in) + + if tt.expectError { + errVal, ok := got["Error"] + require.True(t, ok, "expected Error key") + s, ok := errVal.(string) + require.True(t, ok, "expected Error value to be string") + require.NotEmpty(t, s) + return + } + + // Must NOT contain Error + _, ok := got["Error"] + require.False(t, ok, "did not expect Error key") + + assert.Equal(t, tt.wantKeys, got) + }) + } +} + +func TestFromJSONArray(t *testing.T) { + tests := []struct { + name string + in string + expectError bool + want []any + }{ + { + name: "valid json array", + in: `["a","b",3,true]`, + want: []any{"a", "b", float64(3), true}, // encoding/json uses float64 for numbers in interface{} + }, + { + name: "invalid json returns single error string element", + in: `[`, + expectError: true, + }, + { + name: "empty string returns error string element (invalid json)", + in: "", + // json.Unmarshal("", &a) => error ("unexpected end of JSON input") + expectError: true, + }, + { + name: "whitespace is invalid json (still error)", + in: " ", + // json.Unmarshal(" ", &a) => error + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := fromJSONArray(tt.in) + if tt.expectError { + require.Len(t, got, 1) + _, ok := got[0].(string) + require.True(t, ok, "expected error string in array") + require.NotEmpty(t, got[0].(string)) + return + } + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/template/functions/funcmap.go b/pkg/template/functions/funcmap.go new file mode 100644 index 00000000..85b6272e --- /dev/null +++ b/pkg/template/functions/funcmap.go @@ -0,0 +1,36 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 +package functions + +import ( + "maps" + "text/template" + + "github.com/go-sprout/sprout/sprigin" +) + +// TxtFuncMap returns an aggregated template function map. Currently (custom functions + sprig). +func ExtraFuncMap() template.FuncMap { + funcMap := sprigin.FuncMap() + + maps.Copy(funcMap, CustomFuncMap()) + + // Remove unsafe methods + delete(funcMap, "env") + delete(funcMap, "expandEnv") + + return funcMap +} + +// CustomFuncMap return our custom templates. +func CustomFuncMap() template.FuncMap { + return template.FuncMap{ + "toToml": toTOML, + "fromToml": fromTOML, + "fromYamlArray": fromYAMLArray, + "fromJsonArray": fromJSONArray, + "deterministicUUID": deterministicUUID, + "generateAgeKey": generateAgeKey, + "generateAgePQKey": generateAgePQKey, + } +} diff --git a/pkg/template/functions/uuid.go b/pkg/template/functions/uuid.go new file mode 100644 index 00000000..a37f3e2b --- /dev/null +++ b/pkg/template/functions/uuid.go @@ -0,0 +1,36 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 +package functions + +import ( + "crypto/sha256" + "encoding/hex" + "strings" +) + +func deterministicUUID(parts ...string) string { + // Normalize: trim whitespace; keep empty strings if caller passes them intentionally. + // If you prefer to skip empties, filter them out here. + clean := make([]string, 0, len(parts)) + for _, p := range parts { + clean = append(clean, strings.TrimSpace(p)) + } + + // Deterministic stable separator. Using a non-printable delimiter reduces accidental collisions. + // "|" is also fine; \x1f is "unit separator" and common for this use. + msg := strings.Join(clean, "\x1f") + + sum := sha256.Sum256([]byte(msg)) + b := sum[:16] // 128-bit UUID material + + // Set RFC4122 variant (10xxxxxx) + b[8] = (b[8] & 0x3f) | 0x80 + // Set version 5 (0101xxxx) + b[6] = (b[6] & 0x0f) | 0x50 + + // Format as 8-4-4-4-12 hex + hex32 := hex.EncodeToString(b) // 32 lowercase hex chars + uuid := hex32[0:8] + "-" + hex32[8:12] + "-" + hex32[12:16] + "-" + hex32[16:20] + "-" + hex32[20:32] + + return strings.ToUpper(uuid) +} diff --git a/pkg/template/functions/uuid_test.go b/pkg/template/functions/uuid_test.go new file mode 100644 index 00000000..d1903b1b --- /dev/null +++ b/pkg/template/functions/uuid_test.go @@ -0,0 +1,80 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package functions + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDeterministicUUID(t *testing.T) { + tests := []struct { + name string + partsA []string + partsB []string + sameAsA bool + wantUpper bool + }{ + { + name: "same inputs produce same uuid", + partsA: []string{"tenant", "wind", "ns", "wind-test"}, + partsB: []string{"tenant", "wind", "ns", "wind-test"}, + sameAsA: true, + wantUpper: true, + }, + { + name: "whitespace is trimmed (same logical inputs)", + partsA: []string{" tenant ", " wind", "ns ", " wind-test "}, + partsB: []string{"tenant", "wind", "ns", "wind-test"}, + sameAsA: true, + wantUpper: true, + }, + { + name: "different inputs produce different uuid", + partsA: []string{"tenant", "wind", "ns", "wind-test"}, + partsB: []string{"tenant", "wind", "ns", "other-ns"}, + sameAsA: false, + wantUpper: true, + }, + { + name: "empty strings are kept as separators (affects output)", + partsA: []string{"a", "", "b"}, + partsB: []string{"a", "b"}, + sameAsA: false, + wantUpper: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + uA := deterministicUUID(tt.partsA...) + uB := deterministicUUID(tt.partsB...) + + // Basic UUID formatting checks + assert.Len(t, uA, 36) + assert.Equal(t, byte('-'), uA[8]) + assert.Equal(t, byte('-'), uA[13]) + assert.Equal(t, byte('-'), uA[18]) + assert.Equal(t, byte('-'), uA[23]) + + if tt.wantUpper { + assert.Equal(t, strings.ToUpper(uA), uA) + } + + // Version 5 and RFC4122 variant checks: + // UUID format: xxxxxxxx-xxxx-Mxxx-Nxxx-xxxxxxxxxxxx + // M must be '5' (version 5). N must be one of 8,9,A,B (variant 10xx) + assert.Equal(t, byte('5'), uA[14], "expected version 5 at position 14") + assert.Contains(t, "89AB", string(uA[19]), "expected RFC4122 variant at position 19") + + if tt.sameAsA { + assert.Equal(t, uA, uB) + } else { + assert.NotEqual(t, uA, uB) + } + }) + } +} diff --git a/pkg/template/reference.go b/pkg/template/reference.go index 847d1326..0ee395ce 100644 --- a/pkg/template/reference.go +++ b/pkg/template/reference.go @@ -13,19 +13,17 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" - "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/gvk" "github.com/projectcapsule/capsule/pkg/runtime/selectors" ) // Reference // +kubebuilder:object:generate=true type ResourceReference struct { - // Kind of the referent. - // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - Kind string `json:"kind" protobuf:"bytes,1,opt,name=kind"` - // API version of the referent. - APIVersion string `json:"apiVersion" protobuf:"bytes,5,opt,name=apiVersion"` + gvk.VersionKind `json:",inline"` + // Name of the values referent. This is useful // when you traying to get a specific resource // +kubebuilder:validation:MinLength=1 @@ -34,7 +32,7 @@ type ResourceReference struct { Name string `json:"name,omitempty"` // Namespace of the values referent. // +optional - Namespace meta.RFC1123SubdomainName `json:"namespace,omitempty"` + Namespace string `json:"namespace,omitempty"` // Selector which allows to get any amount of these resources based on labels // +optional Selector *metav1.LabelSelector `json:"selector,omitempty"` @@ -48,7 +46,7 @@ func (t ResourceReference) RequiresTemplating() bool { return true } - if RequiresFastTemplate(string(t.Namespace)) { + if RequiresFastTemplate(t.Namespace) { return true } @@ -72,9 +70,7 @@ func (t ResourceReference) LoadTemplated(templateContext map[string]string) (Res } if out.Namespace != "" { - out.Namespace = meta.RFC1123SubdomainName( - FastTemplate(string(out.Namespace), templateContext), - ) + out.Namespace = FastTemplate(out.Namespace, templateContext) } // Selector @@ -98,6 +94,7 @@ func (t ResourceReference) LoadResources( additionSelectors []labels.Selector, templateContext map[string]string, allowClusterScoped bool, + validateNamespace NamespaceValidator, ) ([]*unstructured.Unstructured, error) { isNamespaced, err := t.IsNamespacedGVK(restMapper) if err != nil { @@ -113,7 +110,18 @@ func (t ResourceReference) LoadResources( return nil, err } - return ref.loadResources(ctx, kubeClient, restMapper, namespace, additionSelectors) + ns := ref.Namespace + if ns == "" && namespace != "" { + ns = namespace + } + + if validateNamespace != nil && ns != "" { + if err := validateNamespace(ns); err != nil { + return nil, err + } + } + + return ref.loadResources(ctx, kubeClient, ns, additionSelectors) } func (t ResourceReference) IsNamespacedGVK( @@ -139,15 +147,17 @@ func (t ResourceReference) IsNamespacedGVK( func (t ResourceReference) loadResources( ctx context.Context, kubeClient client.Client, - restMapper k8smeta.RESTMapper, namespace string, additionSelectors []labels.Selector, ) ([]*unstructured.Unstructured, error) { - ns := t.Namespace + log := log.FromContext(ctx) - if namespace != "" { - ns = meta.RFC1123SubdomainName(namespace) - } + log.V(5).Info("gathering resources", + "apiVersion", t.APIVersion, + "kind", t.Kind, + "name", t.Name, + "namespace", namespace, + ) // GET path (single object) if t.Name != "" { @@ -157,7 +167,7 @@ func (t ResourceReference) loadResources( key := client.ObjectKey{ Name: t.Name, - Namespace: string(ns), + Namespace: namespace, } if err := kubeClient.Get(ctx, key, obj); err != nil { @@ -176,11 +186,10 @@ func (t ResourceReference) loadResources( list.SetKind(t.Kind + "List") var opts []client.ListOption - if ns != "" { - opts = append(opts, client.InNamespace(string(ns))) + if namespace != "" { + opts = append(opts, client.InNamespace(namespace)) } - // Convert t.Selector (metav1) to labels.Selector if present var tenantSel labels.Selector if t.Selector != nil { @@ -206,7 +215,26 @@ func (t ResourceReference) loadResources( if len(all) > 0 { combined := selectors.CombineSelectors(all...) + + selectorStrings := make([]string, 0, len(all)) + + for _, s := range all { + if s != nil { + selectorStrings = append(selectorStrings, s.String()) + } + } + + log.V(5).Info("applying combined label selector", + "namespace", namespace, + "selectors", selectorStrings, + "combinedSelector", combined.String(), + ) + opts = append(opts, client.MatchingLabelsSelector{Selector: combined}) + } else { + log.V(5).Info("listing without label selector", + "namespace", namespace, + ) } if err := kubeClient.List(ctx, list, opts...); err != nil { @@ -218,5 +246,10 @@ func (t ResourceReference) loadResources( results = append(results, list.Items[i].DeepCopy()) } + log.V(5).Info("gathered resources", + "namespace", namespace, + "count", len(results), + ) + return results, nil } diff --git a/pkg/template/reference_context.go b/pkg/template/reference_context.go index ef2dd266..ebc95d6c 100644 --- a/pkg/template/reference_context.go +++ b/pkg/template/reference_context.go @@ -3,27 +3,6 @@ package template -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "strconv" - "text/template" - - k8smeta "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/projectcapsule/capsule/pkg/runtime/sanitize" -) - -// Additional Context to enhance templating -// +kubebuilder:object:generate=true -type TemplateContext struct { - Resources []*TemplateResourceReference `json:"resources,omitempty"` -} - // +kubebuilder:object:generate=true type TemplateResourceReference struct { ResourceReference `json:",inline"` @@ -31,100 +10,3 @@ type TemplateResourceReference struct { // Index to mount the resource in the template context Index string `json:"index,omitempty"` } - -func (t *TemplateContext) GatherContext( - ctx context.Context, - kubeClient client.Client, - restMapper k8smeta.RESTMapper, - data map[string]any, - namespace string, - additionSelectors []labels.Selector, -) (context ReferenceContext, errors []error) { - context = ReferenceContext{} - - if t.Resources == nil { - return - } - - // Template Context for Tenant - if len(data) != 0 { - if err := t.selfTemplate(data); err != nil { - return context, []error{fmt.Errorf("cloud not template: %w", err)} - } - } - - // Load external Resources - for index, resource := range t.Resources { - res, err := resource.LoadResources(ctx, kubeClient, restMapper, namespace, additionSelectors, map[string]string{}, true) - if err != nil { - errors = append(errors, err) - - continue - } - - if len(res) > 0 { - resourceIndex := resource.Index - if resourceIndex == "" { - resourceIndex = strconv.Itoa(index) - } - - for _, u := range res { - sanitize.SanitizeUnstructured(u, sanitize.DefaultSanitizeOptions()) - } - - context[resourceIndex] = res - } - } - - return -} - -// Templates itself with the option to populate tenant fields. -func (t *TemplateContext) selfTemplate( - data map[string]any, -) (err error) { - dataBytes, err := json.Marshal(t) - if err != nil { - return fmt.Errorf("error marshaling TemplateContext: %w", err) - } - - if err := json.Unmarshal(dataBytes, &data); err != nil { - return fmt.Errorf("error unmarshaling TemplateContext into map: %w", err) - } - - tmpl, err := template.New("tpl").Option("missingkey=error").Funcs(ExtraFuncMap()).Parse(string(dataBytes)) - if err != nil { - return fmt.Errorf("error parsing template: %w", err) - } - - var rendered bytes.Buffer - if err := tmpl.Execute(&rendered, data); err != nil { - return fmt.Errorf("error executing template: %w", err) - } - - tplContext := &TemplateContext{} - if err := json.Unmarshal(rendered.Bytes(), tplContext); err != nil { - return fmt.Errorf("error unmarshaling JSON into TemplateContext: %w", err) - } - - // Reassing templated context - *t = *tplContext - - return nil -} - -// +kubebuilder:object:generate=false -type ReferenceContext map[string]any - -func (t *ReferenceContext) String() (string, error) { - dataBytes, err := json.Marshal(t) - if err != nil { - return "", fmt.Errorf("error marshaling TemplateContext: %w", err) - } - - if err := json.Unmarshal(dataBytes, t); err != nil { - return "", fmt.Errorf("error unmarshaling TemplateContext into map: %w", err) - } - - return string(dataBytes), nil -} diff --git a/pkg/template/types.go b/pkg/template/types.go index f041d736..89448d36 100644 --- a/pkg/template/types.go +++ b/pkg/template/types.go @@ -3,7 +3,7 @@ package template -// +kubebuilder:validation:Enum=default;zero;error +// +kubebuilder:validation:Enum=invalid;zero;error type MissingKeyOption string func (p MissingKeyOption) String() string { @@ -11,7 +11,7 @@ func (p MissingKeyOption) String() string { } const ( - MissingKeyDefault MissingKeyOption = "default" + MissingKeyInvalid MissingKeyOption = "invalid" MissingKeyZero MissingKeyOption = "zero" MissingKeyError MissingKeyOption = "error" ) diff --git a/pkg/template/unstructured.go b/pkg/template/unstructured.go index 7b4a0a66..2b50c53b 100644 --- a/pkg/template/unstructured.go +++ b/pkg/template/unstructured.go @@ -8,10 +8,13 @@ import ( "errors" "fmt" "io" + "strings" "text/template" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" kyaml "k8s.io/apimachinery/pkg/util/yaml" + + "github.com/projectcapsule/capsule/pkg/template/functions" ) // RenderUnstructuredItems attempts to render a given string template into a list of unstructured resources. @@ -20,7 +23,7 @@ func RenderUnstructuredItems( key MissingKeyOption, tplString string, ) (items []*unstructured.Unstructured, err error) { - tmpl, err := template.New("tpl").Option("missingkey=" + key.String()).Funcs(ExtraFuncMap()).Parse(tplString) + tmpl, err := template.New("tpl").Option("missingkey=" + key.String()).Funcs(functions.ExtraFuncMap()).Parse(tplString) if err != nil { return } @@ -42,7 +45,7 @@ func RenderUnstructuredItems( } // Skip pure whitespace/--- separators that decode to nil/empty. - return nil, fmt.Errorf("decode yaml: %w", err) + return nil, fmt.Errorf("decode yaml: %w\nrendered template:\n%s", err, withLineNumbers(rendered.String())) } if len(obj) == 0 { @@ -59,3 +62,17 @@ func RenderUnstructuredItems( return out, nil } + +func withLineNumbers(s string) string { + lines := strings.Split(s, "\n") + + width := len(fmt.Sprintf("%d", len(lines))) + + var b strings.Builder + + for i, line := range lines { + fmt.Fprintf(&b, "%*d | %s\n", width, i+1, line) + } + + return b.String() +} diff --git a/pkg/template/validator_namespaces.go b/pkg/template/validator_namespaces.go new file mode 100644 index 00000000..d8b7d318 --- /dev/null +++ b/pkg/template/validator_namespaces.go @@ -0,0 +1,34 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package template + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/util/sets" +) + +type NamespaceValidator func(namespace string) error + +func NewNamespaceValidator(allowCrossNamespaceSelection bool, allowed sets.Set[string]) NamespaceValidator { + return func(namespace string) error { + if namespace == "" { + return nil + } + + if allowCrossNamespaceSelection { + return nil + } + + if allowed.Has(namespace) { + return nil + } + + return fmt.Errorf( + "cross-namespace selection is not allowed. Referring a Namespace (%s) that is not part of the allowed namespaces %v", + namespace, + allowed.UnsortedList(), + ) + } +} diff --git a/pkg/template/validator_namespaces_test.go b/pkg/template/validator_namespaces_test.go new file mode 100644 index 00000000..115fbb68 --- /dev/null +++ b/pkg/template/validator_namespaces_test.go @@ -0,0 +1,100 @@ +package template + +import ( + "testing" + + "k8s.io/apimachinery/pkg/util/sets" +) + +func TestNewNamespaceValidator(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + allowCrossNamespaceSelection bool + allowed sets.Set[string] + namespace string + wantErr bool + wantErrMsg string + }{ + { + name: "allows empty namespace", + allowCrossNamespaceSelection: false, + allowed: sets.New[string]("solar-one", "solar-two"), + namespace: "", + wantErr: false, + }, + { + name: "allows any namespace when cross namespace selection is enabled", + allowCrossNamespaceSelection: true, + allowed: sets.New[string]("solar-one"), + namespace: "kube-system", + wantErr: false, + }, + { + name: "allows namespace contained in allowed set", + allowCrossNamespaceSelection: false, + allowed: sets.New[string]("solar-one", "solar-two"), + namespace: "solar-two", + wantErr: false, + }, + { + name: "rejects namespace not contained in allowed set", + allowCrossNamespaceSelection: false, + allowed: sets.New[string]("solar-one", "solar-two"), + namespace: "kube-system", + wantErr: true, + wantErrMsg: "cross-namespace selection is not allowed. Referring a Namespace (kube-system) that is not part of the allowed namespaces", + }, + { + name: "rejects namespace when allowed set is nil", + allowCrossNamespaceSelection: false, + allowed: nil, + namespace: "kube-system", + wantErr: true, + wantErrMsg: "cross-namespace selection is not allowed. Referring a Namespace (kube-system) that is not part of the allowed namespaces", + }, + { + name: "allows namespace when allowed set has exactly that namespace", + allowCrossNamespaceSelection: false, + allowed: sets.New[string]("kube-system"), + namespace: "kube-system", + wantErr: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + validate := NewNamespaceValidator(tt.allowCrossNamespaceSelection, tt.allowed) + err := validate(tt.namespace) + + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got nil") + } + if got := err.Error(); got == "" || !contains(got, tt.wantErrMsg) { + t.Fatalf("unexpected error message: %q", got) + } + return + } + + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + }) + } +} + +func contains(s, sub string) bool { + return len(sub) == 0 || (len(s) >= len(sub) && func() bool { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return true + } + } + return false + }()) +} diff --git a/pkg/template/zz_generated.deepcopy.go b/pkg/template/zz_generated.deepcopy.go index aabd2450..9842f061 100644 --- a/pkg/template/zz_generated.deepcopy.go +++ b/pkg/template/zz_generated.deepcopy.go @@ -14,6 +14,7 @@ import ( // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceReference) DeepCopyInto(out *ResourceReference) { *out = *in + out.VersionKind = in.VersionKind if in.Selector != nil { in, out := &in.Selector, &out.Selector *out = new(v1.LabelSelector) diff --git a/pkg/tenant/get_by.go b/pkg/tenant/get_by.go index 629ccd78..a6c2497e 100644 --- a/pkg/tenant/get_by.go +++ b/pkg/tenant/get_by.go @@ -9,9 +9,9 @@ import ( "sort" "strings" - authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -74,11 +74,86 @@ func IsNamespaceInTenant( return true, nil } +func GetTenantNameByNamespace( + ctx context.Context, + c client.Reader, + namespace string, +) (tnt string, err error) { + var ns corev1.Namespace + if err := c.Get(ctx, client.ObjectKey{Name: namespace}, &ns); err != nil { + if apierrors.IsNotFound(err) { + return "", nil + } + + return "", err + } + + tntName, ok := GetTenantNameByOwnerreferences(ns.OwnerReferences) + if !ok { + return "", nil + } + + return tntName, nil +} + +func GetTenantByNamespace( + ctx context.Context, + r client.Reader, + namespace string, +) (*capsulev1beta2.Tenant, error) { + var ns corev1.Namespace + if err := r.Get(ctx, client.ObjectKey{Name: namespace}, &ns); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + + return nil, err + } + + for _, or := range ns.GetOwnerReferences() { + if !IsTenantOwnerReference(or) { + continue + } + + tnt := &capsulev1beta2.Tenant{} + if err := r.Get(ctx, client.ObjectKey{Name: or.Name}, tnt); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + + return nil, err + } + + if or.UID != "" && tnt.UID != or.UID { + return nil, fmt.Errorf( + "tenant ownerReference UID mismatch for %q: namespace references UID %q but tenant has UID %q", + or.Name, or.UID, tnt.UID, + ) + } + + return tnt, nil + } + + return nil, nil +} + +func GetTenantNameByOwnerreferences( + refs []metav1.OwnerReference, +) (string, bool) { + for _, or := range refs { + if IsTenantOwnerReference(or) { + return or.Name, true + } + } + + return "", false +} + // getNamespaceTenant returns namespace owner tenant. func GetTenantByOwnerreferences( ctx context.Context, - c client.Client, - refs []v1.OwnerReference, + c client.Reader, + refs []metav1.OwnerReference, ) (tnt *capsulev1beta2.Tenant, err error) { for _, or := range refs { if !IsTenantOwnerReference(or) { @@ -101,15 +176,14 @@ func GetTenantByUserInfo( c client.Client, cfg configuration.Configuration, ns *corev1.Namespace, - username string, - groups []string, + user users.AdmissionUser, ) (sortedTenants, error) { var tenants sortedTenants // User tenants. userTntList := &capsulev1beta2.TenantList{} fields := client.MatchingFields{ - ".spec.owner.ownerkind": fmt.Sprintf("User:%s", username), + ".spec.owner.ownerkind": fmt.Sprintf("User:%s", user.Username), } err := c.List(ctx, userTntList, fields) @@ -120,10 +194,10 @@ func GetTenantByUserInfo( tenants = userTntList.Items // ServiceAccount tenants. - if strings.HasPrefix(username, "system:serviceaccount:") { + if strings.HasPrefix(user.Username, "system:serviceaccount:") { saTntList := &capsulev1beta2.TenantList{} fields = client.MatchingFields{ - ".spec.owner.ownerkind": fmt.Sprintf("ServiceAccount:%s", username), + ".spec.owner.ownerkind": fmt.Sprintf("ServiceAccount:%s", user.Username), } err = c.List(ctx, saTntList, fields) @@ -137,7 +211,7 @@ func GetTenantByUserInfo( // Group tenants. groupTntList := &capsulev1beta2.TenantList{} - for _, group := range groups { + for _, group := range user.Groups { fields = client.MatchingFields{ ".spec.owner.ownerkind": fmt.Sprintf("Group:%s", group), } @@ -158,7 +232,7 @@ func GetTenantByUserInfo( // getTenantByLabels returns tenant from labels. func GetTenantByLabels( ctx context.Context, - c client.Client, + c client.Reader, ns *corev1.Namespace, ) (*capsulev1beta2.Tenant, error) { if label, ok := ns.Labels[meta.TenantLabel]; ok { @@ -176,10 +250,10 @@ func GetTenantByLabels( func GetTenantByLabelsAndUser( ctx context.Context, - c client.Client, + c client.Reader, cfg configuration.Configuration, ns *corev1.Namespace, - userInfo authenticationv1.UserInfo, + user users.AdmissionUser, ) (*capsulev1beta2.Tenant, error) { tnt, err := GetTenantByLabels(ctx, c, ns) if err != nil { @@ -187,7 +261,7 @@ func GetTenantByLabelsAndUser( } if tnt != nil { - if ok := users.IsTenantOwnerByStatus(ctx, c, cfg, tnt, userInfo); !ok { + if ok := users.IsTenantOwnerByStatus(tnt, user); !ok { return nil, fmt.Errorf("can not assign the desired namespace to a non-owned Tenant") } diff --git a/pkg/tenant/metdata.go b/pkg/tenant/metdata.go index 9b13b5ef..8f3e1fa3 100644 --- a/pkg/tenant/metdata.go +++ b/pkg/tenant/metdata.go @@ -8,6 +8,7 @@ import ( "strings" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api/meta" @@ -15,6 +16,64 @@ import ( "github.com/projectcapsule/capsule/pkg/utils" ) +func TenanLabelValue(ns *corev1.Namespace) string { + if ns.GetLabels() == nil { + return "" + } + + return ns.GetLabels()[meta.TenantLabel] +} + +func HasTenantReference(ns *corev1.Namespace) bool { + if ns.Labels != nil && ns.Labels[meta.TenantLabel] != "" { + return true + } + + //nolint:modernize + for _, ref := range ns.OwnerReferences { + if IsTenantOwnerReference(ref) { + return true + } + } + + return false +} + +func TenantOwnerReferences(ns *corev1.Namespace) []metav1.OwnerReference { + if ns == nil { + return nil + } + + refs := make([]metav1.OwnerReference, 0) + + for _, ref := range ns.GetOwnerReferences() { + if IsTenantOwnerReference(ref) { + refs = append(refs, ref) + } + } + + return refs +} + +func TenantOwnerReferenceName(ns *corev1.Namespace) string { + for _, ref := range ns.GetOwnerReferences() { + if IsTenantOwnerReference(ref) { + return ref.Name + } + } + + return "" +} + +func HasTenantOwnership(ns *corev1.Namespace) bool { + return TenanLabelValue(ns) != "" || TenantOwnerReferenceName(ns) != "" +} + +func TenantOwnershipChanged(oldNs, newNs *corev1.Namespace) bool { + return TenanLabelValue(oldNs) != TenanLabelValue(newNs) || + HasTenantOwnership(oldNs) != HasTenantOwnership(newNs) +} + func AddNamespaceNameLabels(labels map[string]string, ns *corev1.Namespace) { labels["kubernetes.io/metadata.name"] = ns.GetName() } @@ -58,7 +117,7 @@ func BuildNamespaceMetadataForTenant(ns *corev1.Namespace, tnt *capsulev1beta2.T annotations = BuildNamespaceAnnotationsForTenant(tnt) labels = BuildNamespaceLabelsForTenant(tnt) - fastContext := ContextForTenantAndNamespace(tnt, ns) + fastContext := FastContextForTenantAndNamespace(tnt, ns) if opts := tnt.Spec.NamespaceOptions; opts != nil && len(opts.AdditionalMetadataList) > 0 { for _, md := range opts.AdditionalMetadataList { diff --git a/pkg/tenant/namespaces.go b/pkg/tenant/namespaces.go new file mode 100644 index 00000000..dd3de992 --- /dev/null +++ b/pkg/tenant/namespaces.go @@ -0,0 +1,309 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenant + +import ( + "context" + "errors" + "fmt" + "sync" + + "golang.org/x/sync/errgroup" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" +) + +type NamespacedResourceCache interface { + Get(disco discovery.DiscoveryInterface) ([]schema.GroupVersionResource, error) +} + +func NamespaceIsPendingPodTerminating( + ctx context.Context, + c client.Reader, + ns *corev1.Namespace, +) (pending bool, err error) { + // Pods behave differently, we manually check if they are still present as they are the largest attack vector + var podList corev1.PodList + if err := c.List(ctx, &podList, client.InNamespace(ns.Name)); err != nil { + return false, fmt.Errorf("list pods in namespace %q: %w", ns.Name, err) + } + + if len(podList.Items) > 0 { + return true, nil + } + + return false, nil +} + +func NamespaceIsPendingUnmanagedTerminationByStatus(ctx context.Context, c client.Reader, ns *corev1.Namespace) (bool, error) { + tnt, err := GetTenantByNamespace(ctx, c, ns.GetName()) + if err != nil { + return false, err + } + + if tnt == nil { + return false, nil + } + + instance := tnt.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{ + Name: ns.GetName(), + }) + if instance == nil { + return false, nil + } + + cond := instance.Conditions.GetConditionByType(meta.TerminatingCondition) + if cond == nil || cond.Status != metav1.ConditionTrue { + return false, nil + } + + return true, nil +} + +// First deletes (remove finalizers and deleted) all resources which are not managed by capsule +// Once fulfilled, remove all managed resources. +func NamespacedCascadingCleanup( + ctx context.Context, + c client.Reader, + disco discovery.DiscoveryInterface, + resourceCache NamespacedResourceCache, + dyn dynamic.Interface, + ns *corev1.Namespace, +) (cleaned bool, err error) { + _, err = removeFinalizersFromRemainingNamespacedResources(ctx, disco, dyn, resourceCache, ns.Name, []string{}) + if err != nil { + return true, err + } + + return false, nil +} + +func removeFinalizersFromRemainingNamespacedResources( + ctx context.Context, + disco discovery.DiscoveryInterface, + dyn dynamic.Interface, + resourceCache NamespacedResourceCache, + namespace string, + ignoredFinalizers []string, +) (bool, error) { + var errs []error + + gvrs, err := resourceCache.Get(disco) + if err != nil { + return false, err + } + + ignored := make(map[string]struct{}, len(ignoredFinalizers)) + for _, f := range ignoredFinalizers { + ignored[f] = struct{}{} + } + + var ( + mu sync.Mutex + cleanedAny bool + ) + + g, ctx := errgroup.WithContext(ctx) + g.SetLimit(4) + + for _, gvr := range gvrs { + g.Go(func() error { + cleaned, err := processResourceType(ctx, dyn, gvr, namespace, ignored) + + mu.Lock() + defer mu.Unlock() + + if cleaned { + cleanedAny = true + } + + if err != nil { + errs = append(errs, fmt.Errorf("process %s in namespace %q: %w", gvr.String(), namespace, err)) + } + + return nil + }) + } + + _ = g.Wait() + + return cleanedAny, errors.Join(errs...) +} + +func processResourceType( + ctx context.Context, + dyn dynamic.Interface, + gvr schema.GroupVersionResource, + namespace string, + ignoredFinalizers map[string]struct{}, +) (bool, error) { + list, err := dyn.Resource(gvr).Namespace(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + if apierrors.IsNotFound(err) || apierrors.IsMethodNotSupported(err) { + return false, nil + } + + return false, fmt.Errorf("list %s in namespace %q: %w", gvr.String(), namespace, err) + } + + var errs []error + + cleanedAny := false + + for i := range list.Items { + obj := &list.Items[i] + + if gvr.Group == "" && gvr.Resource == "pods" { + continue + } + + // If the object is not yet deleting, issue a delete first. + if obj.GetDeletionTimestamp() == nil { + if err := dyn.Resource(gvr).Namespace(namespace).Delete(ctx, obj.GetName(), metav1.DeleteOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + errs = append(errs, fmt.Errorf( + "delete %s %q/%q: %w", + gvr.String(), + namespace, + obj.GetName(), + err, + )) + + continue + } + } else { + cleanedAny = true + } + } + + finalizers := obj.GetFinalizers() + if len(finalizers) == 0 { + continue + } + + remainingFinalizers, removed := meta.FilterFinalizers(finalizers, ignoredFinalizers) + if !removed { + continue + } + + patch := meta.BuildFinalizersMergePatch(remainingFinalizers) + + _, err := dyn.Resource(gvr).Namespace(namespace).Patch( + ctx, + obj.GetName(), + types.MergePatchType, + patch, + metav1.PatchOptions{}, + ) + if err != nil { + if !apierrors.IsNotFound(err) { + errs = append(errs, fmt.Errorf( + "patch finalizers on %s %q/%q: %w", + gvr.String(), + namespace, + obj.GetName(), + err, + )) + + continue + } + } else { + cleanedAny = true + } + } + + return cleanedAny, errors.Join(errs...) +} + +func ResolveNamespaceTenant( + ctx context.Context, + reader client.Reader, + ns *corev1.Namespace, +) (*capsulev1beta2.Tenant, error) { + if ns == nil { + return nil, nil + } + + label := TenanLabelValue(ns) + refs := TenantOwnerReferences(ns) + + switch { + case label == "" && len(refs) == 0: + return nil, nil + + case len(refs) > 1: + return nil, fmt.Errorf("namespace can not have multiple Tenant ownerReferences") + + case label == "" && len(refs) == 1: + return nil, fmt.Errorf("namespace has Tenant ownerReference %q but no tenant label", refs[0].Name) + + case label != "" && len(refs) == 0: + return nil, fmt.Errorf("namespace has tenant label %q but no Tenant ownerReference", label) + } + + tnt, err := GetTenantByOwnerreferences(ctx, reader, refs) + if err != nil { + return nil, err + } + + if tnt == nil { + return nil, fmt.Errorf("namespace references unknown Tenant %q", refs[0].Name) + } + + if tnt.GetName() != label { + return nil, fmt.Errorf("namespace label %q does not match owner reference %q", label, tnt.GetName()) + } + + return tnt, nil +} + +func CollectTenantNamespaceByLabel( + ctx context.Context, + c client.Client, + tnt capsulev1beta2.Tenant, + additionalSelector *metav1.LabelSelector, +) (namespaces []corev1.Namespace, err error) { + // Creating Namespace selector + var selector labels.Selector + + if additionalSelector != nil { + selector, err = metav1.LabelSelectorAsSelector(additionalSelector) + if err != nil { + return nil, err + } + } else { + selector = labels.NewSelector() + } + + // Resources can be replicated only on Namespaces belonging to the same Global: + // preventing a boundary cross by enforcing the selection. + tntRequirement, err := labels.NewRequirement(meta.TenantLabel, selection.Equals, []string{tnt.GetName()}) + if err != nil { + err = fmt.Errorf("unable to create requirement for Namespace filtering and resource replication: %w", err) + + return nil, err + } + + selector = selector.Add(*tntRequirement) + // Selecting the targeted Namespace according to the TenantResource specification. + ns := corev1.NamespaceList{} + if err = c.List(ctx, &ns, client.MatchingLabelsSelector{Selector: selector}); err != nil { + err = fmt.Errorf("cannot retrieve Namespaces for resource: %w", err) + + return nil, err + } + + return ns.Items, nil +} diff --git a/pkg/tenant/owned.go b/pkg/tenant/owned.go index e24ef4be..777fffa1 100644 --- a/pkg/tenant/owned.go +++ b/pkg/tenant/owned.go @@ -6,7 +6,6 @@ package tenant import ( "context" - authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -17,18 +16,18 @@ import ( func NamespaceIsOwned( ctx context.Context, - c client.Client, + c client.Reader, cfg configuration.Configuration, ns *corev1.Namespace, tnt *capsulev1beta2.Tenant, - userInfo authenticationv1.UserInfo, + user users.AdmissionUser, ) bool { for _, ownerRef := range ns.OwnerReferences { if !IsTenantOwnerReferenceForTenant(ownerRef, tnt) { continue } - return users.IsTenantOwnerByStatus(ctx, c, cfg, tnt, userInfo) + return users.IsTenantOwnerByStatus(tnt, user) } return false diff --git a/pkg/tenant/owners.go b/pkg/tenant/owners.go index c1ae6a80..3b4b703c 100644 --- a/pkg/tenant/owners.go +++ b/pkg/tenant/owners.go @@ -12,39 +12,42 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/runtime/configuration" ) func CollectOwners( ctx context.Context, - c client.Client, + c client.Reader, tnt *capsulev1beta2.Tenant, cfg configuration.Configuration, -) (api.OwnerStatusListSpec, error) { +) (rbac.OwnerStatusListSpec, error) { owners := tnt.Spec.Owners.ToStatusOwners() - // Promoted ServiceAccounts - if cfg.AllowServiceAccountPromotion() && len(tnt.Status.Namespaces) > 0 { - saList := &corev1.ServiceAccountList{} - if err := c.List(ctx, saList, - client.MatchingLabels{ - meta.OwnerPromotionLabel: meta.ValueTrue, - }, - ); err != nil { - return nil, err + if cfg.AllowServiceAccountPromotion() && + tnt.Spec.Permissions.AllowOwnerPromotion && + len(tnt.Status.Namespaces) > 0 { + nsSet := make(map[string]struct{}, len(tnt.Status.Namespaces)) + for _, ns := range tnt.Status.Namespaces { + nsSet[ns] = struct{}{} } - for _, sa := range saList.Items { - for _, ns := range tnt.Status.Namespaces { - if sa.GetNamespace() != ns { - continue - } + for ns := range nsSet { + saList := &corev1.ServiceAccountList{} + if err := c.List(ctx, saList, + client.InNamespace(ns), + client.MatchingLabels{ + meta.OwnerPromotionLabel: meta.ValueTrue, + }, + ); err != nil { + return owners, err + } - owners.Upsert(api.CoreOwnerSpec{ - UserSpec: api.UserSpec{ - Kind: api.ServiceAccountOwner, + for _, sa := range saList.Items { + owners.Upsert(rbac.CoreOwnerSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, Name: serviceaccount.ServiceAccountUsernamePrefix + sa.Namespace + ":" + sa.Name, }, ClusterRoles: cfg.RBAC().PromotionClusterRoles, @@ -53,18 +56,16 @@ func CollectOwners( } } - // Administrators for _, a := range cfg.Administrators() { - owners.Upsert(api.CoreOwnerSpec{ + owners.Upsert(rbac.CoreOwnerSpec{ UserSpec: a, ClusterRoles: cfg.RBAC().AdministrationClusterRoles, }) } - // Dedicated Owner Objects listed, err := tnt.Spec.Permissions.ListMatchingOwners(ctx, c, tnt.GetName()) if err != nil { - return nil, err + return owners, err } for _, o := range listed { diff --git a/pkg/tenant/owners_test.go b/pkg/tenant/owners_test.go index 705aaa27..9e998a87 100644 --- a/pkg/tenant/owners_test.go +++ b/pkg/tenant/owners_test.go @@ -8,7 +8,7 @@ import ( "testing" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/tenant" ) @@ -26,10 +26,10 @@ func TestGetOwnersWithKinds_SingleOwner(t *testing.T) { tnt := &capsulev1beta2.Tenant{ Status: capsulev1beta2.TenantStatus{ - Owners: []api.CoreOwnerSpec{ + Owners: []rbac.CoreOwnerSpec{ { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "alice", }, }, @@ -48,16 +48,16 @@ func TestGetOwnersWithKinds_SingleOwner(t *testing.T) { func TestGetOwnersWithKinds_MultipleOwners_PreservesOrder(t *testing.T) { tnt := &capsulev1beta2.Tenant{ Status: capsulev1beta2.TenantStatus{ - Owners: []api.CoreOwnerSpec{ + Owners: []rbac.CoreOwnerSpec{ { - UserSpec: api.UserSpec{ - Kind: api.GroupOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.GroupOwner, Name: "admins", }, }, { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "bob", }, }, @@ -79,10 +79,10 @@ func TestGetOwnersWithKinds_MultipleOwners_PreservesOrder(t *testing.T) { func TestGetOwnersWithKinds_EmptyNameStillIncluded(t *testing.T) { tnt := &capsulev1beta2.Tenant{ Status: capsulev1beta2.TenantStatus{ - Owners: []api.CoreOwnerSpec{ + Owners: []rbac.CoreOwnerSpec{ { - UserSpec: api.UserSpec{ - Kind: api.UserOwner, + UserSpec: rbac.UserSpec{ + Kind: rbac.UserOwner, Name: "", }, }, diff --git a/pkg/tenant/promotions.go b/pkg/tenant/promotions.go new file mode 100644 index 00000000..b21600ff --- /dev/null +++ b/pkg/tenant/promotions.go @@ -0,0 +1,129 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package tenant + +import ( + "context" + "fmt" + "slices" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/api/rbac" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" + "github.com/projectcapsule/capsule/pkg/users" +) + +func CollectPromotions( + ctx context.Context, + c client.Client, + tnt *capsulev1beta2.Tenant, + cfg configuration.Configuration, +) (promotions rbac.PromotionStatusListSpec, err error) { + if len(tnt.Status.Namespaces) == 0 { + return nil, nil + } + + promotions = rbac.PromotionStatusListSpec{} + + promoReq, err := labels.NewRequirement(meta.ServiceAccountPromotionLabel, selection.Equals, []string{meta.ValueTrue}) + if err != nil { + return nil, err + } + + staticSel := labels.NewSelector().Add(*promoReq) + + namespaces, err := tnt.GetNamespaceObjects(ctx, c) + if err != nil { + return nil, err + } + + for _, ruleset := range tnt.Spec.Rules { + var namespaceSelector labels.Selector + + if ruleset.NamespaceSelector != nil { + namespaceSelector, err = metav1.LabelSelectorAsSelector(ruleset.NamespaceSelector) + if err != nil { + return nil, fmt.Errorf("invalid promotion namespaceSelector for tenant %s: %w", tnt.Name, err) + } + } + + var ( + matchingNamespaces []corev1.Namespace + targetNamespaces []string + ) + + for _, ns := range namespaces { + if namespaceSelector != nil && !namespaceSelector.Matches(labels.Set(ns.Labels)) { + continue + } + + matchingNamespaces = append(matchingNamespaces, ns) + targetNamespaces = append(targetNamespaces, ns.GetName()) + } + + if len(matchingNamespaces) == 0 { + continue + } + + for _, promotion := range ruleset.Permissions.Promotions { + combinedSel := staticSel + + if promotion.Selector != nil { + ruleSel, err := metav1.LabelSelectorAsSelector(promotion.Selector) + if err != nil { + return nil, fmt.Errorf("invalid promotion selector for tenant %s: %w", tnt.Name, err) + } + + combinedSel = selectors.CombineSelectors(staticSel, ruleSel) + } + + for _, ns := range matchingNamespaces { + saList := &corev1.ServiceAccountList{} + + if err := c.List(ctx, saList, + client.InNamespace(ns.GetName()), + client.MatchingLabelsSelector{Selector: combinedSel}, + ); err != nil { + return nil, err + } + + for _, sa := range saList.Items { + targets := appendTargetNamespace(targetNamespaces, sa.Namespace) + + promotions.Upsert(rbac.PromotionSpec{ + UserSpec: rbac.UserSpec{ + Kind: rbac.ServiceAccountOwner, + Name: users.GetServiceAccountFullName(meta.NamespacedRFC1123ObjectReferenceWithNamespace{ + Name: meta.RFC1123Name(sa.Name), + Namespace: meta.RFC1123SubdomainName(sa.Namespace), + }), + }, + ClusterRoles: promotion.ClusterRoles, + Targets: targets, + }) + } + } + } + } + + return promotions, nil +} + +func appendTargetNamespace(targets []string, namespace string) []string { + result := append([]string(nil), targets...) + + if slices.Contains(result, namespace) { + return result + } + + return append(result, namespace) +} diff --git a/pkg/tenant/rules.go b/pkg/tenant/rules.go index 5f975634..a333c443 100644 --- a/pkg/tenant/rules.go +++ b/pkg/tenant/rules.go @@ -4,28 +4,45 @@ package tenant import ( + "context" "fmt" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/meta" + "github.com/projectcapsule/capsule/pkg/runtime/selectors" ) +func GetManagedRuleStatus( + ctx context.Context, + c client.Reader, + ns *corev1.Namespace, +) (*capsulev1beta2.RuleStatus, error) { + obj := &capsulev1beta2.RuleStatus{} + + err := c.Get(ctx, types.NamespacedName{Name: meta.NameForManagedRuleStatus(), Namespace: ns.GetName()}, obj) + if err != nil { + return nil, err + } + + return obj, err +} + // BuildNamespaceRuleBodyForNamespace returns the aggregated rule body that applies to `ns`. // - Rules with nil NamespaceSelector match all namespaces. // - Matching rules are combined in the order they appear in tnt.Spec.Rules (important for "later wins" semantics). -func BuildNamespaceRuleBodyForNamespace( +func BuildNamespaceRuleBodyStatus( + ctx context.Context, + c client.Reader, ns *corev1.Namespace, tnt *capsulev1beta2.Tenant, -) (*capsulev1beta2.NamespaceRuleBody, error) { - out := &capsulev1beta2.NamespaceRuleBody{ - Enforce: capsulev1beta2.NamespaceRuleEnforceBody{ - Registries: make([]api.OCIRegistry, 0), - }, - } +) (*api.NamespaceRuleBodyNamespace, error) { + out := &api.NamespaceRuleBodyNamespace{} if tnt == nil || ns == nil { return out, nil @@ -44,13 +61,15 @@ func BuildNamespaceRuleBodyForNamespace( continue } - matches, err := namespaceRuleMatches(nsLabels, rule.NamespaceSelector) - if err != nil { - return nil, fmt.Errorf("invalid namespaceSelector in rules[%d]: %w", i, err) - } + if rule.NamespaceSelector != nil { + matches, err := selectors.MatchesSelector(nsLabels, *rule.NamespaceSelector) + if err != nil { + return nil, fmt.Errorf("invalid namespaceSelector in rules[%d]: %w", i, err) + } - if !matches { - continue + if !matches { + continue + } } // Merge enforce body (for now: only registries) @@ -62,17 +81,3 @@ func BuildNamespaceRuleBodyForNamespace( return out, nil } - -func namespaceRuleMatches(nsLabels labels.Set, sel *metav1.LabelSelector) (bool, error) { - // nil selector => match all - if sel == nil { - return true, nil - } - - s, err := metav1.LabelSelectorAsSelector(sel) - if err != nil { - return false, err - } - - return s.Matches(nsLabels), nil -} diff --git a/pkg/tenant/template.go b/pkg/tenant/template.go index 5d779a17..96684539 100644 --- a/pkg/tenant/template.go +++ b/pkg/tenant/template.go @@ -5,12 +5,31 @@ package tenant import ( corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/runtime/sanitize" + "github.com/projectcapsule/capsule/pkg/utils" ) +// NewTenantContext returns the context for the tenant. +func NewTenantContext(tnt *capsulev1beta2.Tenant, scheme *runtime.Scheme, opts sanitize.SanitizeOptions) (map[string]any, error) { + if err := sanitize.SanitizeObject(tnt, scheme, opts); err != nil { + return nil, err + } + + context, err := utils.ToUnstructuredMap(tnt) + if err != nil { + return nil, err + } + + context["rbac"] = tnt.GetClusterRolesBySubject(nil) + + return context, nil +} + // TemplateForTenantAndNamespace applies templatingto the provided string. -func ContextForTenantAndNamespace(tnt *capsulev1beta2.Tenant, ns *corev1.Namespace) map[string]string { +func FastContextForTenantAndNamespace(tnt *capsulev1beta2.Tenant, ns *corev1.Namespace) map[string]string { values := map[string]string{} if tnt != nil { diff --git a/pkg/tenant/template_test.go b/pkg/tenant/template_test.go index d940c368..14f28d3f 100644 --- a/pkg/tenant/template_test.go +++ b/pkg/tenant/template_test.go @@ -14,7 +14,7 @@ import ( ) func TestContextForTenantAndNamespace_BothNil(t *testing.T) { - ctx := tenant.ContextForTenantAndNamespace(nil, nil) + ctx := tenant.FastContextForTenantAndNamespace(nil, nil) if ctx == nil { t.Fatalf("expected non-nil map") @@ -31,7 +31,7 @@ func TestContextForTenantAndNamespace_OnlyTenant(t *testing.T) { }, } - ctx := tenant.ContextForTenantAndNamespace(tnt, nil) + ctx := tenant.FastContextForTenantAndNamespace(tnt, nil) if got := ctx["tenant.name"]; got != "wind" { t.Fatalf("expected tenant.name=wind, got %q", got) @@ -51,7 +51,7 @@ func TestContextForTenantAndNamespace_OnlyNamespace(t *testing.T) { }, } - ctx := tenant.ContextForTenantAndNamespace(nil, ns) + ctx := tenant.FastContextForTenantAndNamespace(nil, ns) if got := ctx["namespace"]; got != "wind-prod" { t.Fatalf("expected namespace=wind-prod, got %q", got) @@ -76,7 +76,7 @@ func TestContextForTenantAndNamespace_BothSet(t *testing.T) { }, } - ctx := tenant.ContextForTenantAndNamespace(tnt, ns) + ctx := tenant.FastContextForTenantAndNamespace(tnt, ns) if got := ctx["tenant.name"]; got != "wind" { t.Fatalf("expected tenant.name=wind, got %q", got) diff --git a/pkg/users/admission_user.go b/pkg/users/admission_user.go new file mode 100644 index 00000000..b1dd9e98 --- /dev/null +++ b/pkg/users/admission_user.go @@ -0,0 +1,101 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package users + +import ( + authenticationv1 "k8s.io/api/authentication/v1" + "k8s.io/apiserver/pkg/authentication/serviceaccount" + + "github.com/projectcapsule/capsule/pkg/runtime/configuration" +) + +type AdmissionUserType string + +const ( + AdmissionUserUnknown AdmissionUserType = "Unknown" + AdmissionUserAdmin AdmissionUserType = "Admin" + AdmissionUserCapsule AdmissionUserType = "Capsule" +) + +type AdmissionUser struct { + Type AdmissionUserType + Username string + Groups []string + + ServiceAccount *AdmissionServiceAccount +} + +type AdmissionServiceAccount struct { + Namespace string + Name string +} + +func NewAdmissionUser(userType AdmissionUserType, info authenticationv1.UserInfo) AdmissionUser { + return AdmissionUser{ + Type: userType, + Username: info.Username, + Groups: info.Groups, + ServiceAccount: ToServiceAccount(info.Username), + } +} + +func (u AdmissionUser) IsAdmin() bool { + return u.Type == AdmissionUserAdmin +} + +func (u AdmissionUser) IsCapsule() bool { + return u.Type == AdmissionUserCapsule +} + +func (u AdmissionUser) IsUnknown() bool { + return u.Type == AdmissionUserUnknown +} + +func (u AdmissionUser) UserInfo() authenticationv1.UserInfo { + return authenticationv1.UserInfo{ + Username: u.Username, + Groups: u.Groups, + } +} + +func (u AdmissionUser) IsControllerServiceAccount() bool { + if u.ServiceAccount == nil { + return false + } + + name, namespace := configuration.ControllerServiceAccount() + + if namespace == "" || name == "" { + return false + } + + return u.ServiceAccount.Namespace == namespace && u.ServiceAccount.Name == name +} + +func ToServiceAccount(username string) *AdmissionServiceAccount { + namespace, name, err := serviceaccount.SplitUsername(username) + if err != nil { + return nil + } + + return &AdmissionServiceAccount{ + Namespace: namespace, + Name: name, + } +} + +func ServiceAccountUsername(namespace, name string) string { + return serviceaccount.MakeUsername(namespace, name) +} + +func ServiceAccountGroups(namespace string) []string { + return GetServiceAccountGroups(namespace) +} + +func ServiceAccountUserInfo(namespace, name string) authenticationv1.UserInfo { + return authenticationv1.UserInfo{ + Username: ServiceAccountUsername(namespace, name), + Groups: ServiceAccountGroups(namespace), + } +} diff --git a/pkg/users/groups.go b/pkg/users/groups.go new file mode 100644 index 00000000..32bde08d --- /dev/null +++ b/pkg/users/groups.go @@ -0,0 +1,23 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package users + +func HasIgnoredGroup(userGroups []string, ignoredGroups []string) bool { + if len(userGroups) == 0 || len(ignoredGroups) == 0 { + return false + } + + ignored := make(map[string]struct{}, len(ignoredGroups)) + for _, group := range ignoredGroups { + ignored[group] = struct{}{} + } + + for _, group := range userGroups { + if _, ok := ignored[group]; ok { + return true + } + } + + return false +} diff --git a/pkg/users/is_admin_user.go b/pkg/users/is_admin_user.go index 76d97e8b..a7dcfab0 100644 --- a/pkg/users/is_admin_user.go +++ b/pkg/users/is_admin_user.go @@ -4,11 +4,28 @@ package users import ( + "k8s.io/apiserver/pkg/authentication/serviceaccount" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" + "github.com/projectcapsule/capsule/pkg/runtime/configuration" ) -func IsAdminUser(req admission.Request, administrators api.UserListSpec) bool { +func IsAdminUser(req admission.Request, administrators rbac.UserListSpec) bool { + if IsControllerServiceAccount(req.UserInfo.Username) { + return true + } + return administrators.IsPresent(req.UserInfo.Username, req.UserInfo.Groups) } + +func IsControllerServiceAccount(username string) bool { + namespace, name, err := serviceaccount.SplitUsername(username) + if err != nil { + return false + } + + controllerName, controllerNamespace := configuration.ControllerServiceAccount() + + return namespace == controllerNamespace && name == controllerName +} diff --git a/pkg/users/is_capsule_user.go b/pkg/users/is_capsule_user.go index 4e82e6ca..7c8326a8 100644 --- a/pkg/users/is_capsule_user.go +++ b/pkg/users/is_capsule_user.go @@ -5,14 +5,13 @@ package users import ( "context" - "os" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/serviceaccount" "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/runtime/configuration" ) @@ -31,14 +30,21 @@ func IsCapsuleUser( return false } + capsuleUsers := cfg.GetUsersByStatus() + //nolint:nestif if sets.NewString(groups...).Has("system:serviceaccounts") { namespace, name, err := serviceaccount.SplitUsername(user) if err == nil { - if namespace == os.Getenv("NAMESPACE") && name == os.Getenv("SERVICE_ACCOUNT") { + if configuration.IsControllerServiceAccount(name, namespace) { return false } + serviceaccounts := capsuleUsers.GetByKinds([]rbac.OwnerKind{rbac.ServiceAccountOwner}) + if len(serviceaccounts) > 0 && sets.New[string](serviceaccounts...).Has(user) { + return true + } + var tl capsulev1beta2.TenantList if err := c.List(ctx, &tl, client.MatchingFields{".status.namespaces": namespace}); err != nil { return false @@ -50,10 +56,8 @@ func IsCapsuleUser( } } - capsuleUsers := cfg.GetUsersByStatus() - //nolint:modernize - for _, group := range capsuleUsers.GetByKinds([]api.OwnerKind{api.GroupOwner}) { + for _, group := range capsuleUsers.GetByKinds([]rbac.OwnerKind{rbac.GroupOwner}) { if groupList.Find(group) { if len(cfg.IgnoreUserWithGroups()) > 0 { for _, ignoreGroup := range cfg.IgnoreUserWithGroups() { @@ -67,7 +71,7 @@ func IsCapsuleUser( } } - users := capsuleUsers.GetByKinds([]api.OwnerKind{api.UserOwner}) + users := capsuleUsers.GetByKinds([]rbac.OwnerKind{rbac.UserOwner}) if len(users) > 0 && sets.New[string](users...).Has(user) { return true } diff --git a/pkg/users/is_tenant_owner.go b/pkg/users/is_tenant_owner.go index 8afae192..a73cf203 100644 --- a/pkg/users/is_tenant_owner.go +++ b/pkg/users/is_tenant_owner.go @@ -19,7 +19,7 @@ import ( func IsTenantOwner( ctx context.Context, - c client.Client, + c client.Reader, cfg configuration.Configuration, tnt *capsulev1beta2.Tenant, userInfo authenticationv1.UserInfo, @@ -32,18 +32,19 @@ func IsTenantOwner( } func IsTenantOwnerByStatus( - ctx context.Context, - c client.Client, - cfg configuration.Configuration, tnt *capsulev1beta2.Tenant, - userInfo authenticationv1.UserInfo, + user AdmissionUser, ) bool { - return tnt.Status.Owners.IsOwner(userInfo.Username, userInfo.Groups) + if user.IsAdmin() { + return true + } + + return tnt.Status.Owners.IsOwner(user.Username, user.Groups) } func IsCommonOwner( ctx context.Context, - c client.Client, + c client.Reader, cfg configuration.Configuration, tnt *capsulev1beta2.Tenant, userInfo authenticationv1.UserInfo, diff --git a/pkg/users/serviceaccounts.go b/pkg/users/serviceaccounts.go index dca088d7..e44d708d 100644 --- a/pkg/users/serviceaccounts.go +++ b/pkg/users/serviceaccounts.go @@ -5,10 +5,14 @@ package users import ( "context" + "fmt" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/authentication/serviceaccount" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" @@ -20,7 +24,7 @@ import ( // if a serviceaccount is in a tenant namespace they will return the tenant. func ResolveServiceAccountActor( ctx context.Context, - c client.Client, + c client.Reader, ns *corev1.Namespace, username string, cfg configuration.Configuration, @@ -54,3 +58,36 @@ func ResolveServiceAccountActor( return tnt, err } + +// GetServiceAccountFullName return the full qualified name for the serviceaccount. +func GetServiceAccountFullName(ref meta.NamespacedRFC1123ObjectReferenceWithNamespace) string { + return serviceaccount.ServiceAccountUsernamePrefix + string(ref.Namespace) + ":" + string(ref.Name) +} + +// GetServiceAccountGroups returns all groups associated with a ServiceAccount. +func GetServiceAccountGroups(namespace string) []string { + return []string{ + fmt.Sprintf("%s%s", serviceaccount.ServiceAccountGroupPrefix, namespace), + serviceaccount.AllServiceAccountsGroup, + user.AllAuthenticated, + } +} + +// ImpersonatedKubernetesClientForServiceAccount returns a controller-runtime client.Client that impersonates a given ServiceAccount. +func ImpersonatedKubernetesClientForServiceAccount( + base *rest.Config, + scheme *runtime.Scheme, + reference meta.NamespacedRFC1123ObjectReferenceWithNamespace, +) (client.Client, error) { + imp := rest.CopyConfig(base) + imp.Impersonate = rest.ImpersonationConfig{ + UserName: GetServiceAccountFullName(reference), + } + + k8sClient, err := client.New(imp, client.Options{Scheme: scheme}) + if err != nil { + return nil, fmt.Errorf("failed to create impersonated client: %w", err) + } + + return k8sClient, nil +} diff --git a/pkg/users/user_group_test.go b/pkg/users/user_group_test.go index ab0d020a..c682f24b 100644 --- a/pkg/users/user_group_test.go +++ b/pkg/users/user_group_test.go @@ -1,12 +1,14 @@ // Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 -package users +package users_test import ( "testing" "github.com/stretchr/testify/assert" + + "github.com/projectcapsule/capsule/pkg/users" ) func TestIsInCapsuleGroups(t *testing.T) { @@ -22,5 +24,5 @@ func TestIsInCapsuleGroups(t *testing.T) { capsuleGroup := "kubernetes-abilitytologin" - assert.True(t, NewUserGroupList(groups).Find(capsuleGroup), nil) + assert.True(t, users.NewUserGroupList(groups).Find(capsuleGroup), nil) } diff --git a/pkg/utils/errors.go b/pkg/utils/errors.go index c6350fb6..fb8a8d1a 100644 --- a/pkg/utils/errors.go +++ b/pkg/utils/errors.go @@ -4,7 +4,8 @@ package utils import ( - "github.com/pkg/errors" + gherrors "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/client-go/discovery" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -13,5 +14,21 @@ import ( func IsUnsupportedAPI(err error) bool { missingAPIError, discoveryGropuError, discoveryResourceError := &meta.NoKindMatchError{}, &discovery.ErrGroupDiscoveryFailed{}, &apiutil.ErrResourceDiscoveryFailed{} - return errors.As(err, &missingAPIError) || errors.As(err, &discoveryGropuError) || errors.As(err, &discoveryResourceError) + return gherrors.As(err, &missingAPIError) || gherrors.As(err, &discoveryGropuError) || gherrors.As(err, &discoveryResourceError) +} + +func IgnoreWrappedNotFound(err error) error { + if err == nil { + return nil + } + + if apierrors.IsNotFound(err) { + return nil + } + + if apierrors.IsNotFound(gherrors.Cause(err)) { + return nil + } + + return err } diff --git a/pkg/utils/hashes.go b/pkg/utils/hashes.go index b8569c8c..2860598b 100644 --- a/pkg/utils/hashes.go +++ b/pkg/utils/hashes.go @@ -7,10 +7,10 @@ import ( "fmt" "hash/fnv" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" ) -func RoleBindingHashFunc(binding api.AdditionalRoleBindingsSpec) string { +func RoleBindingHashFunc(binding rbac.AdditionalRoleBindingsSpec) string { h := fnv.New64a() _, _ = h.Write([]byte(binding.ClusterRoleName)) diff --git a/pkg/utils/hashes_test.go b/pkg/utils/hashes_test.go index 38ecbd01..0a4df6e6 100644 --- a/pkg/utils/hashes_test.go +++ b/pkg/utils/hashes_test.go @@ -8,12 +8,12 @@ import ( rbacv1 "k8s.io/api/rbac/v1" - "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/utils" ) func TestRoleBindingHashFunc_Deterministic(t *testing.T) { - b := api.AdditionalRoleBindingsSpec{ + b := rbac.AdditionalRoleBindingsSpec{ ClusterRoleName: "admin", Subjects: []rbacv1.Subject{ {Kind: "User", Name: "alice"}, @@ -33,11 +33,11 @@ func TestRoleBindingHashFunc_Deterministic(t *testing.T) { } func TestRoleBindingHashFunc_ChangesWhenClusterRoleChanges(t *testing.T) { - b1 := api.AdditionalRoleBindingsSpec{ + b1 := rbac.AdditionalRoleBindingsSpec{ ClusterRoleName: "admin", Subjects: []rbacv1.Subject{{Kind: "User", Name: "alice"}}, } - b2 := api.AdditionalRoleBindingsSpec{ + b2 := rbac.AdditionalRoleBindingsSpec{ ClusterRoleName: "view", Subjects: []rbacv1.Subject{{Kind: "User", Name: "alice"}}, } @@ -51,11 +51,11 @@ func TestRoleBindingHashFunc_ChangesWhenClusterRoleChanges(t *testing.T) { } func TestRoleBindingHashFunc_ChangesWhenSubjectKindChanges(t *testing.T) { - b1 := api.AdditionalRoleBindingsSpec{ + b1 := rbac.AdditionalRoleBindingsSpec{ ClusterRoleName: "admin", Subjects: []rbacv1.Subject{{Kind: "User", Name: "alice"}}, } - b2 := api.AdditionalRoleBindingsSpec{ + b2 := rbac.AdditionalRoleBindingsSpec{ ClusterRoleName: "admin", Subjects: []rbacv1.Subject{{Kind: "Group", Name: "alice"}}, } @@ -69,11 +69,11 @@ func TestRoleBindingHashFunc_ChangesWhenSubjectKindChanges(t *testing.T) { } func TestRoleBindingHashFunc_ChangesWhenSubjectNameChanges(t *testing.T) { - b1 := api.AdditionalRoleBindingsSpec{ + b1 := rbac.AdditionalRoleBindingsSpec{ ClusterRoleName: "admin", Subjects: []rbacv1.Subject{{Kind: "User", Name: "alice"}}, } - b2 := api.AdditionalRoleBindingsSpec{ + b2 := rbac.AdditionalRoleBindingsSpec{ ClusterRoleName: "admin", Subjects: []rbacv1.Subject{{Kind: "User", Name: "bob"}}, } @@ -87,7 +87,7 @@ func TestRoleBindingHashFunc_ChangesWhenSubjectNameChanges(t *testing.T) { } func TestRoleBindingHashFunc_EmptyInputsStillProduceHash(t *testing.T) { - b := api.AdditionalRoleBindingsSpec{ + b := rbac.AdditionalRoleBindingsSpec{ ClusterRoleName: "", Subjects: nil, } @@ -101,14 +101,14 @@ func TestRoleBindingHashFunc_EmptyInputsStillProduceHash(t *testing.T) { func TestRoleBindingHashFunc_SubjectOrderMatters_CurrentBehavior(t *testing.T) { // This test documents the CURRENT behavior: // the hash is order-dependent because subjects are written in slice order. - b1 := api.AdditionalRoleBindingsSpec{ + b1 := rbac.AdditionalRoleBindingsSpec{ ClusterRoleName: "admin", Subjects: []rbacv1.Subject{ {Kind: "User", Name: "alice"}, {Kind: "Group", Name: "devops"}, }, } - b2 := api.AdditionalRoleBindingsSpec{ + b2 := rbac.AdditionalRoleBindingsSpec{ ClusterRoleName: "admin", Subjects: []rbacv1.Subject{ {Kind: "Group", Name: "devops"}, diff --git a/internal/webhook/utils/kubernetes_version.go b/pkg/utils/kubernetes_version.go similarity index 59% rename from internal/webhook/utils/kubernetes_version.go rename to pkg/utils/kubernetes_version.go index 36ab5c65..ac964ad2 100644 --- a/internal/webhook/utils/kubernetes_version.go +++ b/pkg/utils/kubernetes_version.go @@ -4,13 +4,8 @@ package utils import ( - "path/filepath" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" + "k8s.io/client-go/discovery" ) var versionsWithNodeFix = []string{"v1.18.18", "v1.19.10", "v1.20.6", "v1.21.0"} @@ -42,26 +37,11 @@ func NodeWebhookSupported(currentVersion *version.Version) (bool, error) { return true, nil } -func GetK8sVersion() (*version.Version, error) { - cfg, err := rest.InClusterConfig() - if err != nil { - kubeconfig := filepath.Join(homedir.HomeDir(), ".kube", "config") - cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig) - } - +func GetK8sVersionFromConfig(dc discovery.DiscoveryInterface) (*version.Version, error) { + sv, err := dc.ServerVersion() if err != nil { return nil, err } - client, err := kubernetes.NewForConfig(cfg) - if err != nil { - return nil, err - } - - v, err := client.Discovery().ServerVersion() - if err != nil { - return nil, err - } - - return version.ParseGeneric(v.String()) + return version.ParseGeneric(sv.String()) } diff --git a/pkg/utils/maps.go b/pkg/utils/maps.go index 1f3a3548..df182c09 100644 --- a/pkg/utils/maps.go +++ b/pkg/utils/maps.go @@ -3,6 +3,13 @@ package utils +import ( + "fmt" + "reflect" + + "k8s.io/apimachinery/pkg/runtime" +) + func MapMergeNoOverrite(dst, src map[string]string) { if len(src) == 0 { return @@ -28,3 +35,75 @@ func MapEqual(a, b map[string]string) bool { return true } + +func ToUnstructuredMap(obj any) (map[string]any, error) { + m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return nil, err + } + + return m, nil +} + +func Mapify(data any) map[string]any { + result := make(map[string]any) + v := reflect.ValueOf(data) + + // If the provided data is a pointer, resolve to the underlying value + if v.Kind() == reflect.Pointer { + if v.IsNil() { + return result // Return empty map for nil pointers + } + + v = v.Elem() + } + + // Ensure we're working with a struct + if v.Kind() == reflect.Struct { + for i := range v.NumField() { + field := v.Type().Field(i) + + // Skip unexported fields + if field.PkgPath != "" { + continue + } + + value := v.Field(i) + + // Handle different types with recursive or base handling + //nolint:exhaustive + switch value.Kind() { + case reflect.Pointer: + if !value.IsNil() { + result[field.Name] = Mapify(value.Interface()) + } + case reflect.Struct: + result[field.Name] = Mapify(value.Interface()) + case reflect.Slice: + var slice []any + + for j := range value.Len() { + item := value.Index(j) + if item.Kind() == reflect.Struct { + slice = append(slice, Mapify(item.Interface())) + } else { + slice = append(slice, item.Interface()) + } + } + + result[field.Name] = slice + case reflect.Map: + mapResult := make(map[string]any) + for _, key := range value.MapKeys() { + mapResult[fmt.Sprint(key)] = value.MapIndex(key).Interface() + } + + result[field.Name] = mapResult + default: + result[field.Name] = value.Interface() + } + } + } + + return result +}