diff --git a/api/v1alpha1/conversion_hub.go b/api/v1alpha1/conversion_hub.go index d11060cb..611514ee 100644 --- a/api/v1alpha1/conversion_hub.go +++ b/api/v1alpha1/conversion_hub.go @@ -17,6 +17,8 @@ import ( ) const ( + resourceQuotaScopeAnnotation = "capsule.clastix.io/resource-quota-scope" + podAllowedImagePullPolicyAnnotation = "capsule.clastix.io/allowed-image-pull-policy" podPriorityAllowedAnnotation = "priorityclass.capsule.clastix.io/allowed" @@ -189,6 +191,17 @@ func (t *Tenant) ConvertTo(dstRaw conversion.Hub) error { } if len(t.Spec.ResourceQuota) > 0 { dst.Spec.ResourceQuota = &capsulev1beta1.ResourceQuotaSpec{ + Scope: func() capsulev1beta1.ResourceQuotaScope { + if v, ok := t.GetAnnotations()[resourceQuotaScopeAnnotation]; ok { + switch v { + case string(capsulev1beta1.ResourceQuotaScopeNamespace): + return capsulev1beta1.ResourceQuotaScopeNamespace + case string(capsulev1beta1.ResourceQuotaScopeTenant): + return capsulev1beta1.ResourceQuotaScopeTenant + } + } + return capsulev1beta1.ResourceQuotaScopeTenant + }(), Items: t.Spec.ResourceQuota, } } @@ -292,6 +305,7 @@ func (t *Tenant) ConvertTo(dstRaw conversion.Hub) error { delete(dst.ObjectMeta.Annotations, enablePriorityClassListingAnnotation) delete(dst.ObjectMeta.Annotations, enablePriorityClassUpdateAnnotation) delete(dst.ObjectMeta.Annotations, enablePriorityClassDeletionAnnotation) + delete(dst.ObjectMeta.Annotations, resourceQuotaScopeAnnotation) return nil } @@ -452,6 +466,7 @@ func (t *Tenant) ConvertFrom(srcRaw conversion.Hub) error { t.Spec.LimitRanges = src.Spec.LimitRanges.Items } if src.Spec.ResourceQuota != nil { + t.Annotations[resourceQuotaScopeAnnotation] = string(src.Spec.ResourceQuota.Scope) t.Spec.ResourceQuota = src.Spec.ResourceQuota.Items } if len(src.Spec.AdditionalRoleBindings) > 0 { diff --git a/api/v1beta1/resource_quota.go b/api/v1beta1/resource_quota.go index d6558bd0..4f0a48a6 100644 --- a/api/v1beta1/resource_quota.go +++ b/api/v1beta1/resource_quota.go @@ -5,6 +5,17 @@ package v1beta1 import corev1 "k8s.io/api/core/v1" +// +kubebuilder:validation:Enum=Tenant;Namespace +type ResourceQuotaScope string + +const ( + ResourceQuotaScopeTenant ResourceQuotaScope = "Tenant" + ResourceQuotaScopeNamespace ResourceQuotaScope = "Namespace" +) + type ResourceQuotaSpec struct { + // +kubebuilder:default=Tenant + // Define if the Resource Budget should compute resource across all Namespaces in the Tenant or individually per cluster. Default is Tenant + Scope ResourceQuotaScope `json:"scope,omitempty"` Items []corev1.ResourceQuotaSpec `json:"items,omitempty"` } diff --git a/controllers/tenant_controller.go b/controllers/tenant_controller.go index fd4af997..d4af2b4b 100644 --- a/controllers/tenant_controller.go +++ b/controllers/tenant_controller.go @@ -287,14 +287,18 @@ func (r *TenantReconciler) syncAdditionalRoleBindings(tenant *capsulev1beta1.Ten return nil } -// We're relying on the ResourceQuota resource to represent the resource quota for the single Tenant rather than the -// single Namespace, so abusing of this API although its Namespaced scope. +// When the Resource Budget assigned to a Tenant is Tenant-scoped we have to rely on the ResourceQuota resources to +// represent the resource quota for the single Tenant rather than the single Namespace, +// so abusing of this API although its Namespaced scope. +// // Since a Namespace could take-up all the available resource quota, the Namespace ResourceQuota will be a 1:1 mapping // to the Tenant one: in a second time Capsule is going to sum all the analogous ResourceQuota resources on other Tenant // namespaces to check if the Tenant quota has been exceeded or not, reusing the native Kubernetes policy putting the // .Status.Used value as the .Hard value. // This will trigger a following reconciliation but that's ok: the mutateFn will re-use the same business logic, letting // the mutateFn along with the CreateOrUpdate to don't perform the update since resources are identical. +// +// In case of Namespace-scoped Resource Budget, we're just replicating the resources across all registered Namespaces. func (r *TenantReconciler) syncResourceQuotas(tenant *capsulev1beta1.Tenant) error { // getting requested ResourceQuota keys keys := make([]string, 0, len(tenant.Spec.ResourceQuota.Items)) @@ -323,85 +327,95 @@ func (r *TenantReconciler) syncResourceQuotas(tenant *capsulev1beta1.Tenant) err Namespace: ns, }, } + res, err := controllerutil.CreateOrUpdate(context.TODO(), r.Client, target, func() (err error) { target.SetLabels(map[string]string{ tenantLabel: tenant.Name, typeLabel: strconv.Itoa(i), }) - // Requirement to list ResourceQuota of the current Tenant - tr, err := labels.NewRequirement(tenantLabel, selection.Equals, []string{tenant.Name}) - if err != nil { - r.Log.Error(err, "Cannot build ResourceQuota Tenant requirement") - } - // Requirement to list ResourceQuota for the current index - ir, err := labels.NewRequirement(typeLabel, selection.Equals, []string{strconv.Itoa(i)}) - if err != nil { - r.Log.Error(err, "Cannot build ResourceQuota index requirement") - } - // Listing all the ResourceQuota according to the said requirements. - // These are required since Capsule is going to sum all the used quota to - // sum them and get the Tenant one. - rql := &corev1.ResourceQuotaList{} - err = r.List(context.TODO(), rql, &client.ListOptions{ - LabelSelector: labels.NewSelector().Add(*tr).Add(*ir), - }) - if err != nil { - r.Log.Error(err, "Cannot list ResourceQuota", "tenantFilter", tr.String(), "indexFilter", ir.String()) - return err - } + target.Spec.Scopes = q.Scopes + target.Spec.ScopeSelector = q.ScopeSelector - // Iterating over all the options declared for the ResourceQuota, - // summing all the used quota across different Namespaces to determinate - // if we're hitting a Hard quota at Tenant level. - // For this case, we're going to block the Quota setting the Hard as the - // used one. - for rn, rq := range q.Hard { - r.Log.Info("Desired hard " + rn.String() + " quota is " + rq.String()) - - // Getting the whole usage across all the Tenant Namespaces - var qt resource.Quantity - for _, rq := range rql.Items { - qt.Add(rq.Status.Used[rn]) + switch tenant.Spec.ResourceQuota.Scope { + case capsulev1beta1.ResourceQuotaScopeTenant: + // Calculating the Resource Budget at Tenant scope just if this is put in place. + // Requirement to list ResourceQuota of the current Tenant + tr, err := labels.NewRequirement(tenantLabel, selection.Equals, []string{tenant.Name}) + if err != nil { + r.Log.Error(err, "Cannot build ResourceQuota Tenant requirement") } - r.Log.Info("Computed " + rn.String() + " quota for the whole Tenant is " + qt.String()) - - switch qt.Cmp(q.Hard[rn]) { - case 0: - // The Tenant is matching exactly the Quota: - // falling through next case since we have to block further - // resource allocations. - fallthrough - case 1: - // The Tenant is OverQuota: - // updating all the related ResourceQuota with the current - // used Quota to block further creations. - for i := range rql.Items { - if _, ok := rql.Items[i].Status.Used[rn]; ok { - rql.Items[i].Spec.Hard[rn] = rql.Items[i].Status.Used[rn] - } else { - um := make(map[corev1.ResourceName]resource.Quantity) - um[rn] = resource.Quantity{} - rql.Items[i].Spec.Hard = um - } - } - default: - // The Tenant is respecting the Hard quota: - // restoring the default one for all the elements, - // also for the reconciled one. - for i := range rql.Items { - if rql.Items[i].Spec.Hard == nil { - rql.Items[i].Spec.Hard = map[corev1.ResourceName]resource.Quantity{} - } - rql.Items[i].Spec.Hard[rn] = q.Hard[rn] - } - target.Spec = q + // Requirement to list ResourceQuota for the current index + ir, err := labels.NewRequirement(typeLabel, selection.Equals, []string{strconv.Itoa(i)}) + if err != nil { + r.Log.Error(err, "Cannot build ResourceQuota index requirement") } - if err := r.resourceQuotasUpdate(rn, qt, q.Hard[rn], rql.Items...); err != nil { - r.Log.Error(err, "cannot proceed with outer ResourceQuota") + // Listing all the ResourceQuota according to the said requirements. + // These are required since Capsule is going to sum all the used quota to + // sum them and get the Tenant one. + rql := &corev1.ResourceQuotaList{} + err = r.List(context.TODO(), rql, &client.ListOptions{ + LabelSelector: labels.NewSelector().Add(*tr).Add(*ir), + }) + if err != nil { + r.Log.Error(err, "Cannot list ResourceQuota", "tenantFilter", tr.String(), "indexFilter", ir.String()) return err } + // Iterating over all the options declared for the ResourceQuota, + // summing all the used quota across different Namespaces to determinate + // if we're hitting a Hard quota at Tenant level. + // For this case, we're going to block the Quota setting the Hard as the + // used one. + for rn, rq := range q.Hard { + r.Log.Info("Desired hard " + rn.String() + " quota is " + rq.String()) + + // Getting the whole usage across all the Tenant Namespaces + var qt resource.Quantity + for _, rq := range rql.Items { + qt.Add(rq.Status.Used[rn]) + } + r.Log.Info("Computed " + rn.String() + " quota for the whole Tenant is " + qt.String()) + + switch qt.Cmp(q.Hard[rn]) { + case 0: + // The Tenant is matching exactly the Quota: + // falling through next case since we have to block further + // resource allocations. + fallthrough + case 1: + // The Tenant is OverQuota: + // updating all the related ResourceQuota with the current + // used Quota to block further creations. + for i := range rql.Items { + if _, ok := rql.Items[i].Status.Used[rn]; ok { + rql.Items[i].Spec.Hard[rn] = rql.Items[i].Status.Used[rn] + } else { + um := make(map[corev1.ResourceName]resource.Quantity) + um[rn] = resource.Quantity{} + rql.Items[i].Spec.Hard = um + } + } + default: + // The Tenant is respecting the Hard quota: + // restoring the default one for all the elements, + // also for the reconciled one. + for i := range rql.Items { + if rql.Items[i].Spec.Hard == nil { + rql.Items[i].Spec.Hard = map[corev1.ResourceName]resource.Quantity{} + } + rql.Items[i].Spec.Hard[rn] = q.Hard[rn] + } + target.Spec = q + } + if err := r.resourceQuotasUpdate(rn, qt, q.Hard[rn], rql.Items...); err != nil { + r.Log.Error(err, "cannot proceed with outer ResourceQuota") + return err + } + } + case capsulev1beta1.ResourceQuotaScopeNamespace: + target.Spec.Hard = q.Hard } + return controllerutil.SetControllerReference(tenant, target, r.Scheme) })