github/capsule
1
0
Fork 0
mirror of https://github.com/projectcapsule/capsule.git synced 2026-08-25 16:07:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Tenant or Namespace scope for resource quota budgets

Browse Source
This commit is contained in:
Dario Tranchitella
2021-07-24 14:36:57 +02:00
parent 9244122d42
commit c52f7844db
3 changed files with 109 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/v1alpha1/conversion_hub.go
+15
View File
@@ -17,6 +17,8 @@ import (
)
const (
resourceQuotaScopeAnnotation = "capsule.clastix.io/resource-quota-scope"
podAllowedImagePullPolicyAnnotation = "capsule.clastix.io/allowed-image-pull-policy"
podPriorityAllowedAnnotation = "priorityclass.capsule.clastix.io/allowed"
@@ -189,6 +191,17 @@ func (t *Tenant) ConvertTo(dstRaw conversion.Hub) error {
}
if len(t.Spec.ResourceQuota) > 0 {
dst.Spec.ResourceQuota = &capsulev1beta1.ResourceQuotaSpec{
Scope: func() capsulev1beta1.ResourceQuotaScope {
if v, ok := t.GetAnnotations()[resourceQuotaScopeAnnotation]; ok {
switch v {
case string(capsulev1beta1.ResourceQuotaScopeNamespace):
return capsulev1beta1.ResourceQuotaScopeNamespace
case string(capsulev1beta1.ResourceQuotaScopeTenant):
return capsulev1beta1.ResourceQuotaScopeTenant
}
}
return capsulev1beta1.ResourceQuotaScopeTenant
}(),
Items: t.Spec.ResourceQuota,
}
}
@@ -292,6 +305,7 @@ func (t *Tenant) ConvertTo(dstRaw conversion.Hub) error {
delete(dst.ObjectMeta.Annotations, enablePriorityClassListingAnnotation)
delete(dst.ObjectMeta.Annotations, enablePriorityClassUpdateAnnotation)
delete(dst.ObjectMeta.Annotations, enablePriorityClassDeletionAnnotation)
delete(dst.ObjectMeta.Annotations, resourceQuotaScopeAnnotation)
return nil
}
@@ -452,6 +466,7 @@ func (t *Tenant) ConvertFrom(srcRaw conversion.Hub) error {
t.Spec.LimitRanges = src.Spec.LimitRanges.Items
}
if src.Spec.ResourceQuota != nil {
t.Annotations[resourceQuotaScopeAnnotation] = string(src.Spec.ResourceQuota.Scope)
t.Spec.ResourceQuota = src.Spec.ResourceQuota.Items
}
if len(src.Spec.AdditionalRoleBindings) > 0 {
api/v1beta1/resource_quota.go
+11
View File
@@ -5,6 +5,17 @@ package v1beta1
import corev1 "k8s.io/api/core/v1"
// +kubebuilder:validation:Enum=Tenant;Namespace
type ResourceQuotaScope string
const (
ResourceQuotaScopeTenant ResourceQuotaScope = "Tenant"
ResourceQuotaScopeNamespace ResourceQuotaScope = "Namespace"
)
type ResourceQuotaSpec struct {
// +kubebuilder:default=Tenant
// Define if the Resource Budget should compute resource across all Namespaces in the Tenant or individually per cluster. Default is Tenant
Scope ResourceQuotaScope `json:"scope,omitempty"`
Items []corev1.ResourceQuotaSpec `json:"items,omitempty"`
}
controllers/tenant_controller.go
+83 -69
View File
@@ -287,14 +287,18 @@ func (r *TenantReconciler) syncAdditionalRoleBindings(tenant *capsulev1beta1.Ten
return nil
}
// We're relying on the ResourceQuota resource to represent the resource quota for the single Tenant rather than the
// single Namespace, so abusing of this API although its Namespaced scope.
// When the Resource Budget assigned to a Tenant is Tenant-scoped we have to rely on the ResourceQuota resources to
// represent the resource quota for the single Tenant rather than the single Namespace,
// so abusing of this API although its Namespaced scope.
//
// Since a Namespace could take-up all the available resource quota, the Namespace ResourceQuota will be a 1:1 mapping
// to the Tenant one: in a second time Capsule is going to sum all the analogous ResourceQuota resources on other Tenant
// namespaces to check if the Tenant quota has been exceeded or not, reusing the native Kubernetes policy putting the
// .Status.Used value as the .Hard value.
// This will trigger a following reconciliation but that's ok: the mutateFn will re-use the same business logic, letting
// the mutateFn along with the CreateOrUpdate to don't perform the update since resources are identical.
//
// In case of Namespace-scoped Resource Budget, we're just replicating the resources across all registered Namespaces.
func (r *TenantReconciler) syncResourceQuotas(tenant *capsulev1beta1.Tenant) error {
// getting requested ResourceQuota keys
keys := make([]string, 0, len(tenant.Spec.ResourceQuota.Items))
@@ -323,85 +327,95 @@ func (r *TenantReconciler) syncResourceQuotas(tenant *capsulev1beta1.Tenant) err
Namespace: ns,
},
}
res, err := controllerutil.CreateOrUpdate(context.TODO(), r.Client, target, func() (err error) {
target.SetLabels(map[string]string{
tenantLabel: tenant.Name,
typeLabel: strconv.Itoa(i),
})
// Requirement to list ResourceQuota of the current Tenant
tr, err := labels.NewRequirement(tenantLabel, selection.Equals, []string{tenant.Name})
if err != nil {
r.Log.Error(err, "Cannot build ResourceQuota Tenant requirement")
}
// Requirement to list ResourceQuota for the current index
ir, err := labels.NewRequirement(typeLabel, selection.Equals, []string{strconv.Itoa(i)})
if err != nil {
r.Log.Error(err, "Cannot build ResourceQuota index requirement")
}
// Listing all the ResourceQuota according to the said requirements.
// These are required since Capsule is going to sum all the used quota to
// sum them and get the Tenant one.
rql := &corev1.ResourceQuotaList{}
err = r.List(context.TODO(), rql, &client.ListOptions{
LabelSelector: labels.NewSelector().Add(*tr).Add(*ir),
})
if err != nil {
r.Log.Error(err, "Cannot list ResourceQuota", "tenantFilter", tr.String(), "indexFilter", ir.String())
return err
}
target.Spec.Scopes = q.Scopes
target.Spec.ScopeSelector = q.ScopeSelector
// Iterating over all the options declared for the ResourceQuota,
// summing all the used quota across different Namespaces to determinate
// if we're hitting a Hard quota at Tenant level.
// For this case, we're going to block the Quota setting the Hard as the
// used one.
for rn, rq := range q.Hard {
r.Log.Info("Desired hard " + rn.String() + " quota is " + rq.String())
// Getting the whole usage across all the Tenant Namespaces
var qt resource.Quantity
for _, rq := range rql.Items {
qt.Add(rq.Status.Used[rn])
switch tenant.Spec.ResourceQuota.Scope {
case capsulev1beta1.ResourceQuotaScopeTenant:
// Calculating the Resource Budget at Tenant scope just if this is put in place.
// Requirement to list ResourceQuota of the current Tenant
tr, err := labels.NewRequirement(tenantLabel, selection.Equals, []string{tenant.Name})
if err != nil {
r.Log.Error(err, "Cannot build ResourceQuota Tenant requirement")
}
r.Log.Info("Computed " + rn.String() + " quota for the whole Tenant is " + qt.String())
switch qt.Cmp(q.Hard[rn]) {
case 0:
// The Tenant is matching exactly the Quota:
// falling through next case since we have to block further
// resource allocations.
fallthrough
case 1:
// The Tenant is OverQuota:
// updating all the related ResourceQuota with the current
// used Quota to block further creations.
for i := range rql.Items {
if _, ok := rql.Items[i].Status.Used[rn]; ok {
rql.Items[i].Spec.Hard[rn] = rql.Items[i].Status.Used[rn]
} else {
um := make(map[corev1.ResourceName]resource.Quantity)
um[rn] = resource.Quantity{}
rql.Items[i].Spec.Hard = um
}
}
default:
// The Tenant is respecting the Hard quota:
// restoring the default one for all the elements,
// also for the reconciled one.
for i := range rql.Items {
if rql.Items[i].Spec.Hard == nil {
rql.Items[i].Spec.Hard = map[corev1.ResourceName]resource.Quantity{}
}
rql.Items[i].Spec.Hard[rn] = q.Hard[rn]
}
target.Spec = q
// Requirement to list ResourceQuota for the current index
ir, err := labels.NewRequirement(typeLabel, selection.Equals, []string{strconv.Itoa(i)})
if err != nil {
r.Log.Error(err, "Cannot build ResourceQuota index requirement")
}
if err := r.resourceQuotasUpdate(rn, qt, q.Hard[rn], rql.Items...); err != nil {
r.Log.Error(err, "cannot proceed with outer ResourceQuota")
// Listing all the ResourceQuota according to the said requirements.
// These are required since Capsule is going to sum all the used quota to
// sum them and get the Tenant one.
rql := &corev1.ResourceQuotaList{}
err = r.List(context.TODO(), rql, &client.ListOptions{
LabelSelector: labels.NewSelector().Add(*tr).Add(*ir),
})
if err != nil {
r.Log.Error(err, "Cannot list ResourceQuota", "tenantFilter", tr.String(), "indexFilter", ir.String())
return err
}
// Iterating over all the options declared for the ResourceQuota,
// summing all the used quota across different Namespaces to determinate
// if we're hitting a Hard quota at Tenant level.
// For this case, we're going to block the Quota setting the Hard as the
// used one.
for rn, rq := range q.Hard {
r.Log.Info("Desired hard " + rn.String() + " quota is " + rq.String())
// Getting the whole usage across all the Tenant Namespaces
var qt resource.Quantity
for _, rq := range rql.Items {
qt.Add(rq.Status.Used[rn])
}
r.Log.Info("Computed " + rn.String() + " quota for the whole Tenant is " + qt.String())
switch qt.Cmp(q.Hard[rn]) {
case 0:
// The Tenant is matching exactly the Quota:
// falling through next case since we have to block further
// resource allocations.
fallthrough
case 1:
// The Tenant is OverQuota:
// updating all the related ResourceQuota with the current
// used Quota to block further creations.
for i := range rql.Items {
if _, ok := rql.Items[i].Status.Used[rn]; ok {
rql.Items[i].Spec.Hard[rn] = rql.Items[i].Status.Used[rn]
} else {
um := make(map[corev1.ResourceName]resource.Quantity)
um[rn] = resource.Quantity{}
rql.Items[i].Spec.Hard = um
}
}
default:
// The Tenant is respecting the Hard quota:
// restoring the default one for all the elements,
// also for the reconciled one.
for i := range rql.Items {
if rql.Items[i].Spec.Hard == nil {
rql.Items[i].Spec.Hard = map[corev1.ResourceName]resource.Quantity{}
}
rql.Items[i].Spec.Hard[rn] = q.Hard[rn]
}
target.Spec = q
}
if err := r.resourceQuotasUpdate(rn, qt, q.Hard[rn], rql.Items...); err != nil {
r.Log.Error(err, "cannot proceed with outer ResourceQuota")
return err
}
}
case capsulev1beta1.ResourceQuotaScopeNamespace:
target.Spec.Hard = q.Hard
}
return controllerutil.SetControllerReference(tenant, target, r.Scheme)
})