mirror of
https://github.com/projectcapsule/capsule.git
synced 2026-08-25 16:07:24 +00:00
feat: requests and limit policies (#2095)
* chore: save progress Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> --------- Signed-off-by: Oliver Baehler <oliver@sudo-i.net>
This commit is contained in:
Oliver Bähler
@@ -211,13 +211,16 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config
|
||||
}
|
||||
})
|
||||
|
||||
Expect(ModifyNode(func(node *corev1.Node) error {
|
||||
node.Labels["foo"] = "bar"
|
||||
node.Labels["gatsby-foo"] = "bar"
|
||||
node.Annotations["foo"] = "bar"
|
||||
node.Annotations["gatsby-foo"] = "bar"
|
||||
return k8sClient.Update(context.Background(), node)
|
||||
})).Should(Succeed())
|
||||
EventuallyCreation(func() error {
|
||||
return ModifyNode(func(node *corev1.Node) error {
|
||||
node.Labels["foo"] = "bar"
|
||||
node.Labels["gatsby-foo"] = "bar"
|
||||
node.Annotations["foo"] = "bar"
|
||||
node.Annotations["gatsby-foo"] = "bar"
|
||||
|
||||
return k8sClient.Update(context.Background(), node)
|
||||
})
|
||||
}).Should(Succeed())
|
||||
|
||||
By("adding forbidden labels using exact match", func() {
|
||||
EventuallyCreation(func() error {
|
||||
@@ -228,7 +231,7 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config
|
||||
_, err := cs.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{})
|
||||
return err
|
||||
})
|
||||
}).ShouldNot(Succeed())
|
||||
}).Should(MatchError(ContainSubstring("some labels are marked as forbidden")))
|
||||
})
|
||||
By("adding forbidden labels using regex match", func() {
|
||||
EventuallyCreation(func() error {
|
||||
@@ -239,7 +242,7 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config
|
||||
_, err := cs.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{})
|
||||
return err
|
||||
})
|
||||
}).ShouldNot(Succeed())
|
||||
}).Should(MatchError(ContainSubstring("some labels are marked as forbidden")))
|
||||
})
|
||||
By("modifying forbidden labels", func() {
|
||||
EventuallyCreation(func() error {
|
||||
@@ -250,7 +253,7 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config
|
||||
_, err := cs.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{})
|
||||
return err
|
||||
})
|
||||
}).ShouldNot(Succeed())
|
||||
}).Should(MatchError(ContainSubstring("some labels are marked as forbidden")))
|
||||
})
|
||||
By("adding forbidden annotations using exact match", func() {
|
||||
EventuallyCreation(func() error {
|
||||
@@ -261,7 +264,7 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config
|
||||
_, err := cs.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{})
|
||||
return err
|
||||
})
|
||||
}).ShouldNot(Succeed())
|
||||
}).Should(MatchError(ContainSubstring("some annotations are marked as forbidden")))
|
||||
})
|
||||
By("adding forbidden annotations using regex match", func() {
|
||||
EventuallyCreation(func() error {
|
||||
@@ -272,7 +275,7 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config
|
||||
_, err := cs.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{})
|
||||
return err
|
||||
})
|
||||
}).ShouldNot(Succeed())
|
||||
}).Should(MatchError(ContainSubstring("some annotations are marked as forbidden")))
|
||||
})
|
||||
By("modifying forbidden annotations", func() {
|
||||
EventuallyCreation(func() error {
|
||||
@@ -283,7 +286,7 @@ var _ = Describe("modifying node labels and annotations", Ordered, Label("config
|
||||
_, err := cs.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{})
|
||||
return err
|
||||
})
|
||||
}).ShouldNot(Succeed())
|
||||
}).Should(MatchError(ContainSubstring("some annotations are marked as forbidden")))
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -511,6 +511,53 @@ var _ = Describe("GlobalResourceQuota", Ordered, Label("globalresourcequota", "r
|
||||
"requests.ephemeral-storage (requested=600Mi, current=600Mi, projected=1200Mi, hard=1Gi, exceededBy=176Mi)",
|
||||
))
|
||||
})
|
||||
|
||||
It("rejects direct hard-limit reductions and removals below allocated usage", func() {
|
||||
quotaKey := client.ObjectKey{Name: ephemeralQuotaName}
|
||||
Eventually(func(g Gomega) {
|
||||
current := &capsulev1beta2.GlobalResourceQuota{}
|
||||
g.Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed())
|
||||
used := current.Status.Total.Used[corev1.ResourceRequestsEphemeralStorage]
|
||||
g.Expect(used.Cmp(resource.MustParse("600Mi"))).To(Equal(0))
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
|
||||
By("rejecting a decrease below usage", func() {
|
||||
current := &capsulev1beta2.GlobalResourceQuota{}
|
||||
Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed())
|
||||
current.Spec.Quota.Hard[corev1.ResourceRequestsEphemeralStorage] = resource.MustParse("500Mi")
|
||||
|
||||
err := k8sClient.Update(ctx, current)
|
||||
Expect(err).To(MatchError(ContainSubstring(
|
||||
`spec.quota.hard["requests.ephemeral-storage"] cannot be reduced to 500Mi while 600Mi is allocated`,
|
||||
)))
|
||||
})
|
||||
|
||||
By("rejecting removal of a resource with usage", func() {
|
||||
current := &capsulev1beta2.GlobalResourceQuota{}
|
||||
Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed())
|
||||
delete(current.Spec.Quota.Hard, corev1.ResourceRequestsEphemeralStorage)
|
||||
|
||||
err := k8sClient.Update(ctx, current)
|
||||
Expect(err).To(MatchError(ContainSubstring(
|
||||
`spec.quota.hard["requests.ephemeral-storage"] cannot be removed while 600Mi is allocated`,
|
||||
)))
|
||||
})
|
||||
|
||||
By("allowing a decrease exactly to allocated usage", func() {
|
||||
current := &capsulev1beta2.GlobalResourceQuota{}
|
||||
Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed())
|
||||
current.Spec.Quota.Hard[corev1.ResourceRequestsEphemeralStorage] = resource.MustParse("600Mi")
|
||||
|
||||
Expect(k8sClient.Update(ctx, current)).To(Succeed())
|
||||
Eventually(func(g Gomega) {
|
||||
reconciled := &capsulev1beta2.GlobalResourceQuota{}
|
||||
g.Expect(k8sClient.Get(ctx, quotaKey, reconciled)).To(Succeed())
|
||||
g.Expect(reconciled.Status.ObservedGeneration).To(Equal(reconciled.Generation))
|
||||
hard := reconciled.Status.Total.Hard[corev1.ResourceRequestsEphemeralStorage]
|
||||
g.Expect(hard.Cmp(resource.MustParse("600Mi"))).To(Equal(0))
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func expectResourceListEqual(g Gomega, actual, expected corev1.ResourceList) {
|
||||
|
||||
@@ -0,0 +1,828 @@
|
||||
// Copyright 2020-2026 Project Capsule Authors.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
|
||||
"github.com/projectcapsule/capsule/pkg/api/meta"
|
||||
"github.com/projectcapsule/capsule/pkg/api/rbac"
|
||||
"github.com/projectcapsule/capsule/pkg/api/rules"
|
||||
)
|
||||
|
||||
var _ = Describe("enforcing workload resource namespace rules", Ordered, Label("tenant", "rules", "enforce", "workloads", "resources"), func() {
|
||||
const ownerName = "e2e-rules-resources"
|
||||
|
||||
var tnt *capsulev1beta2.Tenant
|
||||
targetCases := []struct {
|
||||
name string
|
||||
targets []rules.WorkloadValidationTarget
|
||||
}{
|
||||
{name: "pod", targets: []rules.WorkloadValidationTarget{rules.ValidatePod}},
|
||||
{name: "containers", targets: []rules.WorkloadValidationTarget{rules.ValidateContainers}},
|
||||
{name: "initcontainers", targets: []rules.WorkloadValidationTarget{rules.ValidateInitContainers}},
|
||||
{name: "pod-containers", targets: []rules.WorkloadValidationTarget{rules.ValidatePod, rules.ValidateContainers}},
|
||||
{name: "pod-initcontainers", targets: []rules.WorkloadValidationTarget{rules.ValidatePod, rules.ValidateInitContainers}},
|
||||
{name: "containers-initcontainers", targets: []rules.WorkloadValidationTarget{rules.ValidateContainers, rules.ValidateInitContainers}},
|
||||
{
|
||||
name: "pod-containers-initcontainers",
|
||||
targets: []rules.WorkloadValidationTarget{
|
||||
rules.ValidatePod,
|
||||
rules.ValidateContainers,
|
||||
rules.ValidateInitContainers,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
newTenant := func() *capsulev1beta2.Tenant {
|
||||
tenant := &capsulev1beta2.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "e2e-rule-resources",
|
||||
Labels: map[string]string{"env": "e2e"},
|
||||
},
|
||||
Spec: capsulev1beta2.TenantSpec{
|
||||
Owners: rbac.OwnerListSpec{{CoreOwnerSpec: rbac.CoreOwnerSpec{UserSpec: rbac.UserSpec{
|
||||
Name: ownerName,
|
||||
Kind: rbac.UserOwner,
|
||||
}}}},
|
||||
Rules: []*rules.NamespaceRuleBodyTenant{
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "managed"}},
|
||||
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{
|
||||
Action: rules.ActionTypeDeny,
|
||||
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
|
||||
Resources: &rules.WorkloadResourceRules{
|
||||
Requests: map[corev1.ResourceName]rules.WorkloadResourceRequestPolicy{
|
||||
corev1.ResourceCPU: {
|
||||
Policy: rules.WorkloadResourceRequestPolicyDefault,
|
||||
Value: e2eResourceQuantity("100m"),
|
||||
},
|
||||
},
|
||||
Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{
|
||||
corev1.ResourceCPU: {Policy: rules.WorkloadResourceLimitPolicyRemove},
|
||||
corev1.ResourceMemory: {
|
||||
Policy: rules.WorkloadResourceLimitPolicyMatchRequest,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ephemeral-managed"}},
|
||||
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{
|
||||
Action: rules.ActionTypeDeny,
|
||||
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
|
||||
Targets: []rules.WorkloadValidationTarget{
|
||||
rules.ValidateContainers,
|
||||
rules.ValidateInitContainers,
|
||||
},
|
||||
Resources: &rules.WorkloadResourceRules{
|
||||
Requests: map[corev1.ResourceName]rules.WorkloadResourceRequestPolicy{
|
||||
corev1.ResourceEphemeralStorage: {
|
||||
Policy: rules.WorkloadResourceRequestPolicyDefault,
|
||||
Value: e2eResourceQuantity("1Gi"),
|
||||
},
|
||||
},
|
||||
Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{
|
||||
corev1.ResourceEphemeralStorage: {
|
||||
Policy: rules.WorkloadResourceLimitPolicyDefault,
|
||||
Value: e2eResourceQuantity("2Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio"}},
|
||||
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{
|
||||
Action: rules.ActionTypeDeny,
|
||||
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
|
||||
Targets: []rules.WorkloadValidationTarget{rules.ValidateContainers},
|
||||
Resources: &rules.WorkloadResourceRules{
|
||||
Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{
|
||||
corev1.ResourceMemory: {
|
||||
Policy: rules.WorkloadResourceLimitPolicyRatio,
|
||||
Value: e2eResourceQuantity("1.5"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "init-only"}},
|
||||
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{
|
||||
Action: rules.ActionTypeDeny,
|
||||
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
|
||||
Targets: []rules.WorkloadValidationTarget{rules.ValidateInitContainers},
|
||||
Resources: &rules.WorkloadResourceRules{
|
||||
Requests: map[corev1.ResourceName]rules.WorkloadResourceRequestPolicy{
|
||||
corev1.ResourceCPU: {Policy: rules.WorkloadResourceRequestPolicyRemove},
|
||||
corev1.ResourceMemory: {Policy: rules.WorkloadResourceRequestPolicyPreserve},
|
||||
},
|
||||
Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{
|
||||
corev1.ResourceCPU: {Policy: rules.WorkloadResourceLimitPolicyRemove},
|
||||
corev1.ResourceMemory: {
|
||||
Policy: rules.WorkloadResourceLimitPolicyDefault,
|
||||
Value: e2eResourceQuantity("256Mi"),
|
||||
},
|
||||
corev1.ResourceEphemeralStorage: {Policy: rules.WorkloadResourceLimitPolicyPreserve},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "pod-ratio"}},
|
||||
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{
|
||||
Action: rules.ActionTypeDeny,
|
||||
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
|
||||
Targets: []rules.WorkloadValidationTarget{rules.ValidatePod},
|
||||
Resources: &rules.WorkloadResourceRules{
|
||||
Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{
|
||||
corev1.ResourceCPU: {
|
||||
Policy: rules.WorkloadResourceLimitPolicyRatio,
|
||||
Value: e2eResourceQuantity("1.5"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio-allow"}},
|
||||
NamespaceRuleBodyNamespace: resourceRatioRule(rules.ActionTypeAllow, rules.ValidateContainers, "1.5"),
|
||||
},
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio-audit"}},
|
||||
NamespaceRuleBodyNamespace: resourceRatioRule(rules.ActionTypeAudit, rules.ValidateContainers, "1.5"),
|
||||
},
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ephemeral-ratio"}},
|
||||
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{
|
||||
Action: rules.ActionTypeDeny,
|
||||
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
|
||||
Resources: &rules.WorkloadResourceRules{
|
||||
Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{
|
||||
corev1.ResourceEphemeralStorage: {
|
||||
Policy: rules.WorkloadResourceLimitPolicyRatio,
|
||||
Value: e2eResourceQuantity("1.5"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio-override"}},
|
||||
NamespaceRuleBodyNamespace: resourceRatioRule(rules.ActionTypeDeny, rules.ValidateContainers, "1.5"),
|
||||
},
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio-override"}},
|
||||
NamespaceRuleBodyNamespace: resourceRatioRule(rules.ActionTypeAllow, rules.ValidateContainers, "2"),
|
||||
},
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio-preserve"}},
|
||||
NamespaceRuleBodyNamespace: resourceRatioRule(rules.ActionTypeDeny, rules.ValidateContainers, "1.5"),
|
||||
},
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"resource-policy": "ratio-preserve"}},
|
||||
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{
|
||||
Action: rules.ActionTypeAllow,
|
||||
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
|
||||
Targets: []rules.WorkloadValidationTarget{rules.ValidateContainers},
|
||||
Resources: &rules.WorkloadResourceRules{
|
||||
Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{
|
||||
corev1.ResourceMemory: {Policy: rules.WorkloadResourceLimitPolicyPreserve},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, targetCase := range targetCases {
|
||||
tenant.Spec.Rules = append(tenant.Spec.Rules, &rules.NamespaceRuleBodyTenant{
|
||||
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{
|
||||
"resource-policy": "target-matrix-" + targetCase.name,
|
||||
}},
|
||||
NamespaceRuleBodyNamespace: resourceRatioTargetsRule(
|
||||
rules.ActionTypeDeny,
|
||||
targetCase.targets,
|
||||
"1.5",
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
return tenant
|
||||
}
|
||||
|
||||
newPod := func(name string, memoryLimit string) *corev1.Pod {
|
||||
resources := corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")},
|
||||
}
|
||||
if memoryLimit != "" {
|
||||
resources.Limits = corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse(memoryLimit),
|
||||
}
|
||||
}
|
||||
|
||||
return &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name},
|
||||
Spec: corev1.PodSpec{
|
||||
SecurityContext: nobodyPodSecurityContext(),
|
||||
Containers: []corev1.Container{{
|
||||
Name: "app",
|
||||
Image: "registry.k8s.io/pause:3.9",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
SecurityContext: restrictedContainerSecurityContext(),
|
||||
Resources: resources,
|
||||
}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
createNamespace := func(policy string) *corev1.Namespace {
|
||||
ns := NewNamespace("", map[string]string{
|
||||
meta.TenantLabel: tnt.Name,
|
||||
"resource-policy": policy,
|
||||
})
|
||||
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
|
||||
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
|
||||
|
||||
return ns
|
||||
}
|
||||
|
||||
createPodAndExpectDenied := func(cs kubernetes.Interface, namespace string, pod *corev1.Pod, substrings ...string) {
|
||||
Eventually(func() error {
|
||||
candidate := pod.DeepCopy()
|
||||
candidate.Name = fmt.Sprintf("%s-%d", pod.Name, time.Now().UnixNano()%1e6)
|
||||
|
||||
_, err := cs.CoreV1().Pods(namespace).Create(context.Background(), candidate, metav1.CreateOptions{})
|
||||
if err == nil {
|
||||
_ = cs.CoreV1().Pods(namespace).Delete(context.Background(), candidate.Name, metav1.DeleteOptions{})
|
||||
|
||||
return fmt.Errorf("expected Pod creation to be denied")
|
||||
}
|
||||
if apierrors.IsAlreadyExists(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, substring := range substrings {
|
||||
if !strings.Contains(err.Error(), substring) {
|
||||
return fmt.Errorf("expected error to contain %q, got %v", substring, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
}
|
||||
|
||||
createPodAndExpectAllowed := func(cs kubernetes.Interface, namespace string, pod *corev1.Pod) *corev1.Pod {
|
||||
var created *corev1.Pod
|
||||
EventuallyCreation(func() error {
|
||||
var err error
|
||||
created, err = cs.CoreV1().Pods(namespace).Create(context.Background(), pod, metav1.CreateOptions{})
|
||||
|
||||
return err
|
||||
}).Should(Succeed())
|
||||
|
||||
return created
|
||||
}
|
||||
|
||||
expectResourceAuditEvent := func(namespace string, podName string, substrings ...string) {
|
||||
Eventually(func() error {
|
||||
events, err := clusterAdminClient().EventsV1().Events(namespace).List(context.Background(), metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, event := range events.Items {
|
||||
if event.Regarding.Name != podName || event.Reason != "NamespaceRuleAudit" {
|
||||
continue
|
||||
}
|
||||
|
||||
matched := true
|
||||
for _, substring := range substrings {
|
||||
if !strings.Contains(event.Note, substring) {
|
||||
matched = false
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
if matched {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("expected resource audit event for Pod %q containing %q", podName, substrings)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
}
|
||||
|
||||
JustBeforeEach(func() {
|
||||
tnt = newTenant()
|
||||
EventuallyCreation(func() error {
|
||||
tnt.ResourceVersion = ""
|
||||
|
||||
return k8sClient.Create(context.Background(), tnt)
|
||||
}).Should(Succeed())
|
||||
TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval)
|
||||
})
|
||||
|
||||
JustAfterEach(func() {
|
||||
EventuallyDeletion(tnt)
|
||||
})
|
||||
|
||||
It("projects ordered resource policies into the namespace RuleStatus", func() {
|
||||
ns := createNamespace("ratio-override")
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
status := &capsulev1beta2.RuleStatus{}
|
||||
g.Expect(k8sClient.Get(context.Background(), client.ObjectKey{
|
||||
Name: meta.NameForManagedRuleStatus(),
|
||||
Namespace: ns.Name,
|
||||
}, status)).To(Succeed())
|
||||
g.Expect(status.Status.Rules).To(HaveLen(2))
|
||||
|
||||
for index, expected := range []struct {
|
||||
action rules.ActionType
|
||||
ratio string
|
||||
}{
|
||||
{action: rules.ActionTypeDeny, ratio: "1.5"},
|
||||
{action: rules.ActionTypeAllow, ratio: "2"},
|
||||
} {
|
||||
projected := status.Status.Rules[index]
|
||||
g.Expect(projected).NotTo(BeNil())
|
||||
g.Expect(projected.Enforce).NotTo(BeNil())
|
||||
g.Expect(projected.Enforce.Action).To(Equal(expected.action))
|
||||
g.Expect(projected.Enforce.Workloads.Targets).To(Equal(
|
||||
[]rules.WorkloadValidationTarget{rules.ValidateContainers},
|
||||
))
|
||||
|
||||
policy := projected.Enforce.Workloads.Resources.Limits[corev1.ResourceMemory]
|
||||
g.Expect(policy.Policy).To(Equal(rules.WorkloadResourceLimitPolicyRatio))
|
||||
g.Expect(policy.Value).NotTo(BeNil())
|
||||
g.Expect(policy.Value.Cmp(resource.MustParse(expected.ratio))).To(Equal(0))
|
||||
}
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
})
|
||||
|
||||
It("defaults, removes, and matches resources at all default targets", func() {
|
||||
ns := createNamespace("managed")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
pod := newPod("managed-resources", "2Gi")
|
||||
pod.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("250m")
|
||||
pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{
|
||||
Name: "sidecar",
|
||||
Image: "registry.k8s.io/pause:3.9",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
SecurityContext: restrictedContainerSecurityContext(),
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("128Mi")},
|
||||
Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")},
|
||||
},
|
||||
})
|
||||
pod.Spec.InitContainers = []corev1.Container{{
|
||||
Name: "init-with-limit",
|
||||
Image: "registry.k8s.io/pause:3.9",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
SecurityContext: restrictedContainerSecurityContext(),
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("50m"),
|
||||
corev1.ResourceMemory: resource.MustParse("512Mi"),
|
||||
},
|
||||
Limits: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("2"),
|
||||
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: "init-defaulted",
|
||||
Image: "registry.k8s.io/pause:3.9",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
SecurityContext: restrictedContainerSecurityContext(),
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("256Mi")},
|
||||
},
|
||||
}}
|
||||
pod.Spec.Resources = &corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("2"),
|
||||
corev1.ResourceMemory: resource.MustParse("2Gi"),
|
||||
},
|
||||
Limits: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("512Mi"),
|
||||
},
|
||||
}
|
||||
|
||||
created := createPodAndExpectAllowed(cs, ns.Name, pod)
|
||||
|
||||
containerResources := created.Spec.Containers[0].Resources
|
||||
Expect(containerResources.Requests.Cpu().Cmp(resource.MustParse("250m"))).To(Equal(0))
|
||||
Expect(containerResources.Limits).NotTo(HaveKey(corev1.ResourceCPU))
|
||||
Expect(containerResources.Limits.Memory().Cmp(resource.MustParse("1Gi"))).To(Equal(0))
|
||||
|
||||
sidecarResources := created.Spec.Containers[1].Resources
|
||||
Expect(sidecarResources.Requests.Cpu().Cmp(resource.MustParse("100m"))).To(Equal(0))
|
||||
Expect(sidecarResources.Limits).NotTo(HaveKey(corev1.ResourceCPU))
|
||||
Expect(sidecarResources.Limits.Memory().Cmp(resource.MustParse("128Mi"))).To(Equal(0))
|
||||
|
||||
initResources := created.Spec.InitContainers[0].Resources
|
||||
Expect(initResources.Requests.Cpu().Cmp(resource.MustParse("50m"))).To(Equal(0))
|
||||
Expect(initResources.Limits).NotTo(HaveKey(corev1.ResourceCPU))
|
||||
Expect(initResources.Limits.Memory().Cmp(resource.MustParse("512Mi"))).To(Equal(0))
|
||||
|
||||
defaultedInitResources := created.Spec.InitContainers[1].Resources
|
||||
Expect(defaultedInitResources.Requests.Cpu().Cmp(resource.MustParse("100m"))).To(Equal(0))
|
||||
Expect(defaultedInitResources.Limits).NotTo(HaveKey(corev1.ResourceCPU))
|
||||
Expect(defaultedInitResources.Limits.Memory().Cmp(resource.MustParse("256Mi"))).To(Equal(0))
|
||||
|
||||
Expect(created.Spec.Resources).NotTo(BeNil())
|
||||
Expect(created.Spec.Resources.Requests.Cpu().Cmp(resource.MustParse("2"))).To(Equal(0))
|
||||
Expect(created.Spec.Resources.Limits).NotTo(HaveKey(corev1.ResourceCPU))
|
||||
Expect(created.Spec.Resources.Limits.Memory().Cmp(resource.MustParse("2Gi"))).To(Equal(0))
|
||||
})
|
||||
|
||||
It("applies Ratio only to every explicit target combination", func() {
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
|
||||
for _, targetCase := range targetCases {
|
||||
By(targetCase.name)
|
||||
|
||||
ns := createNamespace("target-matrix-" + targetCase.name)
|
||||
pod := &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "target-matrix-" + targetCase.name},
|
||||
Spec: corev1.PodSpec{
|
||||
SecurityContext: nobodyPodSecurityContext(),
|
||||
Containers: []corev1.Container{{
|
||||
Name: "app",
|
||||
Image: "registry.k8s.io/pause:3.9",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
SecurityContext: restrictedContainerSecurityContext(),
|
||||
Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{
|
||||
corev1.ResourceMemory: resource.MustParse("200Mi"),
|
||||
}},
|
||||
}},
|
||||
InitContainers: []corev1.Container{{
|
||||
Name: "init",
|
||||
Image: "registry.k8s.io/pause:3.9",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
SecurityContext: restrictedContainerSecurityContext(),
|
||||
Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{
|
||||
corev1.ResourceMemory: resource.MustParse("400Mi"),
|
||||
}},
|
||||
}},
|
||||
Resources: &corev1.ResourceRequirements{Requests: corev1.ResourceList{
|
||||
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
created := createPodAndExpectAllowed(cs, ns.Name, pod)
|
||||
targeted := make(map[rules.WorkloadValidationTarget]bool, len(targetCase.targets))
|
||||
for _, target := range targetCase.targets {
|
||||
targeted[target] = true
|
||||
}
|
||||
|
||||
assertLimit := func(
|
||||
target rules.WorkloadValidationTarget,
|
||||
resources corev1.ResourceRequirements,
|
||||
expected string,
|
||||
) {
|
||||
limit, found := resources.Limits[corev1.ResourceMemory]
|
||||
Expect(found).To(Equal(targeted[target]), "target %q in case %q", target, targetCase.name)
|
||||
if found {
|
||||
Expect(limit.Cmp(resource.MustParse(expected))).To(Equal(0), "target %q in case %q", target, targetCase.name)
|
||||
}
|
||||
}
|
||||
|
||||
assertLimit(rules.ValidateContainers, created.Spec.Containers[0].Resources, "300Mi")
|
||||
assertLimit(rules.ValidateInitContainers, created.Spec.InitContainers[0].Resources, "600Mi")
|
||||
assertLimit(rules.ValidatePod, *created.Spec.Resources, "1536Mi")
|
||||
}
|
||||
})
|
||||
|
||||
It("defaults ephemeral storage only on explicitly targeted containers", func() {
|
||||
ns := createNamespace("ephemeral-managed")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
pod := newPod("managed-ephemeral-storage", "")
|
||||
pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{
|
||||
Name: "sidecar",
|
||||
Image: "registry.k8s.io/pause:3.9",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
SecurityContext: restrictedContainerSecurityContext(),
|
||||
Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{
|
||||
corev1.ResourceEphemeralStorage: resource.MustParse("1536Mi"),
|
||||
}},
|
||||
})
|
||||
pod.Spec.InitContainers = []corev1.Container{{
|
||||
Name: "init-with-limit",
|
||||
Image: "registry.k8s.io/pause:3.9",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
SecurityContext: restrictedContainerSecurityContext(),
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("1Gi")},
|
||||
Limits: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("3Gi")},
|
||||
},
|
||||
}, {
|
||||
Name: "init-defaulted",
|
||||
Image: "registry.k8s.io/pause:3.9",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
SecurityContext: restrictedContainerSecurityContext(),
|
||||
}}
|
||||
|
||||
created := createPodAndExpectAllowed(cs, ns.Name, pod)
|
||||
|
||||
Expect(created.Spec.Containers[0].Resources.Requests.StorageEphemeral().Cmp(resource.MustParse("1Gi"))).To(Equal(0))
|
||||
Expect(created.Spec.Containers[0].Resources.Limits.StorageEphemeral().Cmp(resource.MustParse("2Gi"))).To(Equal(0))
|
||||
Expect(created.Spec.Containers[1].Resources.Requests.StorageEphemeral().Cmp(resource.MustParse("1536Mi"))).To(Equal(0))
|
||||
Expect(created.Spec.Containers[1].Resources.Limits.StorageEphemeral().Cmp(resource.MustParse("2Gi"))).To(Equal(0))
|
||||
Expect(created.Spec.InitContainers[0].Resources.Requests.StorageEphemeral().Cmp(resource.MustParse("1Gi"))).To(Equal(0))
|
||||
Expect(created.Spec.InitContainers[0].Resources.Limits.StorageEphemeral().Cmp(resource.MustParse("3Gi"))).To(Equal(0))
|
||||
Expect(created.Spec.InitContainers[1].Resources.Requests.StorageEphemeral().Cmp(resource.MustParse("1Gi"))).To(Equal(0))
|
||||
Expect(created.Spec.InitContainers[1].Resources.Limits.StorageEphemeral().Cmp(resource.MustParse("2Gi"))).To(Equal(0))
|
||||
Expect(created.Spec.Resources).To(BeNil())
|
||||
})
|
||||
|
||||
It("applies Preserve, Remove, and Default only to an explicitly targeted init container", func() {
|
||||
ns := createNamespace("init-only")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
pod := newPod("init-only-resources", "2Gi")
|
||||
pod.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("250m")
|
||||
pod.Spec.InitContainers = []corev1.Container{{
|
||||
Name: "init",
|
||||
Image: "registry.k8s.io/pause:3.9",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
SecurityContext: restrictedContainerSecurityContext(),
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("100m"),
|
||||
corev1.ResourceMemory: resource.MustParse("64Mi"),
|
||||
corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"),
|
||||
},
|
||||
Limits: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("3Gi")},
|
||||
},
|
||||
}}
|
||||
|
||||
created := createPodAndExpectAllowed(cs, ns.Name, pod)
|
||||
|
||||
containerResources := created.Spec.Containers[0].Resources
|
||||
Expect(containerResources.Requests.Cpu().Cmp(resource.MustParse("250m"))).To(Equal(0))
|
||||
Expect(containerResources.Limits.Cpu().Cmp(resource.MustParse("1"))).To(Equal(0))
|
||||
Expect(containerResources.Limits.Memory().Cmp(resource.MustParse("2Gi"))).To(Equal(0))
|
||||
|
||||
initResources := created.Spec.InitContainers[0].Resources
|
||||
Expect(initResources.Requests).NotTo(HaveKey(corev1.ResourceCPU))
|
||||
Expect(initResources.Requests.Memory().Cmp(resource.MustParse("64Mi"))).To(Equal(0))
|
||||
Expect(initResources.Limits).NotTo(HaveKey(corev1.ResourceCPU))
|
||||
Expect(initResources.Limits.Memory().Cmp(resource.MustParse("256Mi"))).To(Equal(0))
|
||||
Expect(initResources.Requests.StorageEphemeral().Cmp(resource.MustParse("1Gi"))).To(Equal(0))
|
||||
Expect(initResources.Limits.StorageEphemeral().Cmp(resource.MustParse("3Gi"))).To(Equal(0))
|
||||
})
|
||||
|
||||
It("accepts MatchRequest after Kubernetes defaults a missing request from its limit", func() {
|
||||
ns := createNamespace("managed")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
pod := newPod("match-request-missing", "1Gi")
|
||||
delete(pod.Spec.Containers[0].Resources.Requests, corev1.ResourceMemory)
|
||||
delete(pod.Spec.Containers[0].Resources.Limits, corev1.ResourceCPU)
|
||||
|
||||
created := createPodAndExpectAllowed(cs, ns.Name, pod)
|
||||
Expect(created.Spec.Containers[0].Resources.Requests.Memory().Cmp(resource.MustParse("1Gi"))).To(Equal(0))
|
||||
Expect(created.Spec.Containers[0].Resources.Limits.Memory().Cmp(resource.MustParse("1Gi"))).To(Equal(0))
|
||||
})
|
||||
|
||||
It("defaults a missing limit from Ratio", func() {
|
||||
ns := createNamespace("ratio")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
pod := newPod("ratio-default", "")
|
||||
|
||||
var created *corev1.Pod
|
||||
EventuallyCreation(func() error {
|
||||
var err error
|
||||
created, err = cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{})
|
||||
|
||||
return err
|
||||
}).Should(Succeed())
|
||||
|
||||
Expect(created.Spec.Containers[0].Resources.Limits.Memory().Cmp(resource.MustParse("1536Mi"))).To(Equal(0))
|
||||
})
|
||||
|
||||
It("preserves a compliant explicit Ratio limit and ignores untargeted locations", func() {
|
||||
ns := createNamespace("ratio")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
pod := newPod("ratio-target-isolation", "1280Mi")
|
||||
pod.Spec.InitContainers = []corev1.Container{{
|
||||
Name: "init",
|
||||
Image: "registry.k8s.io/pause:3.9",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
SecurityContext: restrictedContainerSecurityContext(),
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")},
|
||||
Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")},
|
||||
},
|
||||
}}
|
||||
pod.Spec.Resources = &corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")},
|
||||
Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")},
|
||||
}
|
||||
|
||||
created := createPodAndExpectAllowed(cs, ns.Name, pod)
|
||||
Expect(created.Spec.Containers[0].Resources.Limits.Memory().Cmp(resource.MustParse("1280Mi"))).To(Equal(0))
|
||||
Expect(created.Spec.InitContainers[0].Resources.Limits.Memory().Cmp(resource.MustParse("2Gi"))).To(Equal(0))
|
||||
Expect(created.Spec.Resources.Limits.Memory().Cmp(resource.MustParse("2Gi"))).To(Equal(0))
|
||||
})
|
||||
|
||||
It("denies an explicitly excessive Ratio limit", func() {
|
||||
ns := createNamespace("ratio")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
|
||||
createPodAndExpectDenied(
|
||||
cs,
|
||||
ns.Name,
|
||||
newPod("ratio-denied", "2Gi"),
|
||||
"violates policy Ratio",
|
||||
"must not exceed 1536Mi",
|
||||
)
|
||||
})
|
||||
|
||||
It("denies Ratio when the targeted resource request is missing", func() {
|
||||
ns := createNamespace("ratio")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
pod := newPod("ratio-missing-request", "")
|
||||
delete(pod.Spec.Containers[0].Resources.Requests, corev1.ResourceMemory)
|
||||
|
||||
createPodAndExpectDenied(
|
||||
cs,
|
||||
ns.Name,
|
||||
pod,
|
||||
"violates policy Ratio",
|
||||
"requires a request greater than zero",
|
||||
)
|
||||
})
|
||||
|
||||
It("defaults and enforces Ratio at the explicit Pod target without affecting containers", func() {
|
||||
ns := createNamespace("pod-ratio")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
pod := newPod("pod-ratio-default", "")
|
||||
pod.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("100m")
|
||||
pod.Spec.Containers[0].Resources.Limits = corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("200m")}
|
||||
pod.Spec.Resources = &corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")},
|
||||
}
|
||||
|
||||
created := createPodAndExpectAllowed(cs, ns.Name, pod)
|
||||
Expect(created.Spec.Resources.Limits.Cpu().Cmp(resource.MustParse("750m"))).To(Equal(0))
|
||||
Expect(created.Spec.Containers[0].Resources.Limits.Cpu().Cmp(resource.MustParse("200m"))).To(Equal(0))
|
||||
|
||||
excessive := newPod("pod-ratio-denied", "")
|
||||
excessive.Spec.Resources = &corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")},
|
||||
Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")},
|
||||
}
|
||||
createPodAndExpectDenied(
|
||||
cs,
|
||||
ns.Name,
|
||||
excessive,
|
||||
`spec.resources.limits["cpu"]`,
|
||||
"must not exceed 750m",
|
||||
)
|
||||
})
|
||||
|
||||
It("uses Ratio as an allow-list policy", func() {
|
||||
ns := createNamespace("ratio-allow")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
|
||||
createPodAndExpectAllowed(cs, ns.Name, newPod("ratio-allow-compliant", "1280Mi"))
|
||||
createPodAndExpectDenied(
|
||||
cs,
|
||||
ns.Name,
|
||||
newPod("ratio-allow-denied", "2Gi"),
|
||||
"does not satisfy any allowed resource policy",
|
||||
`limits["memory"]`,
|
||||
)
|
||||
})
|
||||
|
||||
It("audits an excessive Ratio without blocking admission", func() {
|
||||
ns := createNamespace("ratio-audit")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
pod := newPod("ratio-audited", "2Gi")
|
||||
|
||||
created := createPodAndExpectAllowed(cs, ns.Name, pod)
|
||||
Expect(created.Spec.Containers[0].Resources.Limits.Memory().Cmp(resource.MustParse("2Gi"))).To(Equal(0))
|
||||
expectResourceAuditEvent(
|
||||
ns.Name,
|
||||
pod.Name,
|
||||
"workload resource limit",
|
||||
"violates policy Ratio",
|
||||
"must not exceed 1536Mi",
|
||||
)
|
||||
})
|
||||
|
||||
It("uses a later Allow policy to override an earlier Ratio denial", func() {
|
||||
ns := createNamespace("ratio-override")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
pod := newPod("ratio-allow-override", "1792Mi")
|
||||
|
||||
created := createPodAndExpectAllowed(cs, ns.Name, pod)
|
||||
Expect(created.Spec.Containers[0].Resources.Limits.Memory().Cmp(resource.MustParse("1792Mi"))).To(Equal(0))
|
||||
})
|
||||
|
||||
It("uses a later Preserve policy to clear an earlier Ratio constraint", func() {
|
||||
ns := createNamespace("ratio-preserve")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
pod := newPod("ratio-preserve-override", "2Gi")
|
||||
|
||||
created := createPodAndExpectAllowed(cs, ns.Name, pod)
|
||||
Expect(created.Spec.Containers[0].Resources.Limits.Memory().Cmp(resource.MustParse("2Gi"))).To(Equal(0))
|
||||
})
|
||||
|
||||
It("applies ephemeral-storage Ratio to default container targets but not Pod-level resources", func() {
|
||||
ns := createNamespace("ephemeral-ratio")
|
||||
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
|
||||
pod := newPod("ephemeral-ratio-default", "")
|
||||
pod.Spec.Containers[0].Resources.Requests[corev1.ResourceEphemeralStorage] = resource.MustParse("2Gi")
|
||||
pod.Spec.InitContainers = []corev1.Container{{
|
||||
Name: "init",
|
||||
Image: "registry.k8s.io/pause:3.9",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
SecurityContext: restrictedContainerSecurityContext(),
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("1Gi")},
|
||||
},
|
||||
}}
|
||||
created := createPodAndExpectAllowed(cs, ns.Name, pod)
|
||||
Expect(created.Spec.Containers[0].Resources.Limits.StorageEphemeral().Cmp(resource.MustParse("3Gi"))).To(Equal(0))
|
||||
Expect(created.Spec.InitContainers[0].Resources.Limits.StorageEphemeral().Cmp(resource.MustParse("1536Mi"))).To(Equal(0))
|
||||
Expect(created.Spec.Resources).To(BeNil())
|
||||
|
||||
excessive := newPod("ephemeral-ratio-denied", "")
|
||||
excessive.Spec.Containers[0].Resources.Requests[corev1.ResourceEphemeralStorage] = resource.MustParse("2Gi")
|
||||
excessive.Spec.Containers[0].Resources.Limits = corev1.ResourceList{
|
||||
corev1.ResourceEphemeralStorage: resource.MustParse("4Gi"),
|
||||
}
|
||||
createPodAndExpectDenied(
|
||||
cs,
|
||||
ns.Name,
|
||||
excessive,
|
||||
`limits["ephemeral-storage"]`,
|
||||
"must not exceed 3Gi",
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
func e2eResourceQuantity(value string) *resource.Quantity {
|
||||
quantity := resource.MustParse(value)
|
||||
|
||||
return &quantity
|
||||
}
|
||||
|
||||
func resourceRatioRule(
|
||||
action rules.ActionType,
|
||||
target rules.WorkloadValidationTarget,
|
||||
ratio string,
|
||||
) *rules.NamespaceRuleBodyNamespace {
|
||||
return resourceRatioTargetsRule(action, []rules.WorkloadValidationTarget{target}, ratio)
|
||||
}
|
||||
|
||||
func resourceRatioTargetsRule(
|
||||
action rules.ActionType,
|
||||
targets []rules.WorkloadValidationTarget,
|
||||
ratio string,
|
||||
) *rules.NamespaceRuleBodyNamespace {
|
||||
return &rules.NamespaceRuleBodyNamespace{Enforce: &rules.NamespaceRuleEnforceBody{
|
||||
Action: action,
|
||||
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
|
||||
Targets: targets,
|
||||
Resources: &rules.WorkloadResourceRules{
|
||||
Limits: map[corev1.ResourceName]rules.WorkloadResourceLimitPolicy{
|
||||
corev1.ResourceMemory: {
|
||||
Policy: rules.WorkloadResourceLimitPolicyRatio,
|
||||
Value: e2eResourceQuantity(ratio),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}}
|
||||
}
|
||||
@@ -0,0 +1,542 @@
|
||||
// Copyright 2020-2026 Project Capsule Authors
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
|
||||
"github.com/projectcapsule/capsule/pkg/api/meta"
|
||||
"github.com/projectcapsule/capsule/pkg/api/rbac"
|
||||
"github.com/projectcapsule/capsule/pkg/api/rules"
|
||||
tenantutils "github.com/projectcapsule/capsule/pkg/tenant"
|
||||
)
|
||||
|
||||
var _ = Describe("rule-generated GlobalResourceQuota admission", Ordered,
|
||||
Label("resourcequota", "rules", "admission", "managed", "skip-on-openshift"), func() {
|
||||
const (
|
||||
tenantName = "e2e-rule-quota-admission"
|
||||
quotaName = "compute"
|
||||
selectorKey = "e2e.projectcapsule.dev/quota-scope"
|
||||
rbacName = "e2e-rule-quota-admission-tamper"
|
||||
)
|
||||
|
||||
ctx := context.Background()
|
||||
owner := rbac.UserSpec{Name: tenantName, Kind: rbac.UserOwner}
|
||||
tnt := &capsulev1beta2.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: tenantName,
|
||||
Labels: map[string]string{"env": "e2e"},
|
||||
},
|
||||
Spec: capsulev1beta2.TenantSpec{
|
||||
Owners: rbac.OwnerListSpec{{CoreOwnerSpec: rbac.CoreOwnerSpec{UserSpec: owner}}},
|
||||
Rules: []*rules.NamespaceRuleBodyTenant{{
|
||||
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
|
||||
Quota: []rules.ResourceQuotaRule{{
|
||||
Name: quotaName,
|
||||
ResourceQuotaSpec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{
|
||||
corev1.ResourceLimitsCPU: resource.MustParse("8"),
|
||||
corev1.ResourcePods: resource.MustParse("100"),
|
||||
}},
|
||||
}},
|
||||
},
|
||||
NamespaceSelector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{selectorKey: "application"},
|
||||
},
|
||||
}},
|
||||
},
|
||||
}
|
||||
quotaKey := client.ObjectKey{Name: tenantutils.RuleGlobalResourceQuotaName(tnt, quotaName)}
|
||||
tamperRole := &rbacv1.ClusterRole{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: rbacName},
|
||||
Rules: []rbacv1.PolicyRule{{
|
||||
APIGroups: []string{capsulev1beta2.GroupVersion.Group},
|
||||
Resources: []string{"globalresourcequotas"},
|
||||
Verbs: []string{"get", "create", "update", "patch", "delete"},
|
||||
}},
|
||||
}
|
||||
setTenantQuota := func(selector *metav1.LabelSelector, cpu string) error {
|
||||
current := &capsulev1beta2.Tenant{}
|
||||
if err := k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, current); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
current.Spec.Rules[0].NamespaceSelector = selector
|
||||
current.Spec.Rules[0].Quota[0].Hard[corev1.ResourceLimitsCPU] = resource.MustParse(cpu)
|
||||
|
||||
return k8sClient.Update(ctx, current)
|
||||
}
|
||||
expectGeneratedQuota := func(scopeValue, hard, used string, namespaces ...string) {
|
||||
Eventually(func(g Gomega) {
|
||||
current := &capsulev1beta2.GlobalResourceQuota{}
|
||||
g.Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed())
|
||||
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation))
|
||||
g.Expect(current.Status.Namespaces).To(ConsistOf(namespaces))
|
||||
|
||||
selector := current.Spec.NamespaceSelectors[0].LabelSelector
|
||||
g.Expect(selector).NotTo(BeNil())
|
||||
g.Expect(selector.MatchLabels).To(HaveKeyWithValue(meta.TenantLabel, tenantName))
|
||||
if scopeValue == "" {
|
||||
g.Expect(selector.MatchLabels).NotTo(HaveKey(selectorKey))
|
||||
} else {
|
||||
g.Expect(selector.MatchLabels).To(HaveKeyWithValue(selectorKey, scopeValue))
|
||||
}
|
||||
|
||||
hardCPU := current.Spec.Quota.Hard[corev1.ResourceLimitsCPU]
|
||||
g.Expect(hardCPU.Cmp(resource.MustParse(hard))).To(Equal(0))
|
||||
usedCPU := current.Status.Total.Used[corev1.ResourceLimitsCPU]
|
||||
g.Expect(usedCPU.Cmp(resource.MustParse(used))).To(Equal(0))
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
}
|
||||
tamperBinding := &rbacv1.ClusterRoleBinding{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: rbacName},
|
||||
RoleRef: rbacv1.RoleRef{
|
||||
APIGroup: rbacv1.GroupName,
|
||||
Kind: "ClusterRole",
|
||||
Name: rbacName,
|
||||
},
|
||||
Subjects: []rbacv1.Subject{{
|
||||
APIGroup: rbacv1.GroupName,
|
||||
Kind: rbacv1.UserKind,
|
||||
Name: owner.Name,
|
||||
}},
|
||||
}
|
||||
|
||||
BeforeAll(func() {
|
||||
EventuallyCreation(func() error {
|
||||
tamperRole.ResourceVersion = ""
|
||||
|
||||
return k8sClient.Create(ctx, tamperRole)
|
||||
}).Should(Succeed())
|
||||
EventuallyCreation(func() error {
|
||||
tamperBinding.ResourceVersion = ""
|
||||
|
||||
return k8sClient.Create(ctx, tamperBinding)
|
||||
}).Should(Succeed())
|
||||
|
||||
EventuallyCreation(func() error {
|
||||
tnt.ResourceVersion = ""
|
||||
|
||||
return k8sClient.Create(ctx, tnt)
|
||||
}).Should(Succeed())
|
||||
TenantReadyTrue(tnt)
|
||||
|
||||
currentTenant := &capsulev1beta2.Tenant{}
|
||||
Expect(k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, currentTenant)).To(Succeed())
|
||||
tnt.UID = currentTenant.UID
|
||||
|
||||
By("waiting for the managed quota and its status to reconcile", func() {
|
||||
Eventually(func(g Gomega) {
|
||||
quota := &capsulev1beta2.GlobalResourceQuota{}
|
||||
g.Expect(k8sClient.Get(ctx, quotaKey, quota)).To(Succeed())
|
||||
g.Expect(quota.Labels).To(HaveKeyWithValue(meta.NewManagedByCapsuleLabel, meta.ValueController))
|
||||
g.Expect(metav1.IsControlledBy(quota, currentTenant)).To(BeTrue())
|
||||
g.Expect(quota.Status.ObservedGeneration).To(Equal(quota.Generation))
|
||||
|
||||
condition := quota.Status.Conditions.GetConditionByType(meta.ReadyCondition)
|
||||
g.Expect(condition).NotTo(BeNil())
|
||||
g.Expect(condition.Status).To(Equal(metav1.ConditionTrue))
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
})
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
controllerClient := impersonationClient(ControllerServiceAccountFull, nil)
|
||||
forged := &capsulev1beta2.GlobalResourceQuota{ObjectMeta: metav1.ObjectMeta{Name: rbacName}}
|
||||
Expect(ignoreNotFound(controllerClient.Delete(ctx, forged))).To(Succeed())
|
||||
|
||||
EventuallyDeletion(tnt)
|
||||
EventuallyDeletion(tamperBinding)
|
||||
EventuallyDeletion(tamperRole)
|
||||
})
|
||||
|
||||
It("rejects a Tenant rule update that decreases or removes a hard limit while changing scope", func() {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mutate func(corev1.ResourceList)
|
||||
message string
|
||||
}{
|
||||
{
|
||||
name: "decrease",
|
||||
mutate: func(hard corev1.ResourceList) {
|
||||
hard[corev1.ResourceLimitsCPU] = resource.MustParse("0")
|
||||
},
|
||||
message: `rules[0].quota[0].hard["limits.cpu"] cannot be reduced from 8 to 0 while namespace selectors are changing`,
|
||||
},
|
||||
{
|
||||
name: "removal",
|
||||
mutate: func(hard corev1.ResourceList) {
|
||||
delete(hard, corev1.ResourceLimitsCPU)
|
||||
},
|
||||
message: `rules[0].quota[0].hard["limits.cpu"] cannot be removed while namespace selectors are changing`,
|
||||
},
|
||||
} {
|
||||
By(test.name, func() {
|
||||
Eventually(func() error {
|
||||
current := &capsulev1beta2.Tenant{}
|
||||
if err := k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, current); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
updated := current.DeepCopy()
|
||||
updated.Spec.Rules[0].NamespaceSelector = nil
|
||||
test.mutate(updated.Spec.Rules[0].Quota[0].Hard)
|
||||
|
||||
return k8sClient.Update(ctx, updated)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring(test.message)))
|
||||
})
|
||||
}
|
||||
|
||||
persisted := &capsulev1beta2.Tenant{}
|
||||
Expect(k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, persisted)).To(Succeed())
|
||||
Expect(persisted.Spec.Rules[0].NamespaceSelector).NotTo(BeNil())
|
||||
Expect(persisted.Spec.Rules[0].NamespaceSelector.MatchLabels).To(HaveKeyWithValue(selectorKey, "application"))
|
||||
persistedLimit := persisted.Spec.Rules[0].Quota[0].Hard[corev1.ResourceLimitsCPU]
|
||||
Expect(persistedLimit.Cmp(resource.MustParse("8"))).To(Equal(0))
|
||||
})
|
||||
|
||||
It("rejects the same unsafe scope and hard-limit changes on the generated quota", func() {
|
||||
controllerClient := impersonationClient(ControllerServiceAccountFull, nil)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mutate func(corev1.ResourceList)
|
||||
message string
|
||||
}{
|
||||
{
|
||||
name: "decrease",
|
||||
mutate: func(hard corev1.ResourceList) {
|
||||
hard[corev1.ResourceLimitsCPU] = resource.MustParse("0")
|
||||
},
|
||||
message: `spec.quota.hard["limits.cpu"] cannot be reduced from 8 to 0 while namespace selectors are changing`,
|
||||
},
|
||||
{
|
||||
name: "removal",
|
||||
mutate: func(hard corev1.ResourceList) {
|
||||
delete(hard, corev1.ResourceLimitsCPU)
|
||||
},
|
||||
message: `spec.quota.hard["limits.cpu"] cannot be removed while namespace selectors are changing`,
|
||||
},
|
||||
} {
|
||||
By(test.name, func() {
|
||||
Eventually(func() error {
|
||||
current := &capsulev1beta2.GlobalResourceQuota{}
|
||||
if err := controllerClient.Get(ctx, quotaKey, current); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
updated := current.DeepCopy()
|
||||
delete(updated.Spec.NamespaceSelectors[0].LabelSelector.MatchLabels, selectorKey)
|
||||
test.mutate(updated.Spec.Quota.Hard)
|
||||
|
||||
return controllerClient.Update(ctx, updated)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring(test.message)))
|
||||
})
|
||||
}
|
||||
|
||||
persisted := &capsulev1beta2.GlobalResourceQuota{}
|
||||
Expect(k8sClient.Get(ctx, quotaKey, persisted)).To(Succeed())
|
||||
Expect(persisted.Spec.NamespaceSelectors[0].LabelSelector.MatchLabels).To(HaveKeyWithValue(selectorKey, "application"))
|
||||
persistedLimit := persisted.Spec.Quota.Hard[corev1.ResourceLimitsCPU]
|
||||
Expect(persistedLimit.Cmp(resource.MustParse("8"))).To(Equal(0))
|
||||
})
|
||||
|
||||
It("allows equal or increased limits with a scope change and a later same-scope decrease", func() {
|
||||
Eventually(func() error {
|
||||
return setTenantQuota(nil, "8")
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
expectGeneratedQuota("", "8", "0")
|
||||
|
||||
applicationSelector := &metav1.LabelSelector{MatchLabels: map[string]string{selectorKey: "application"}}
|
||||
Eventually(func() error {
|
||||
return setTenantQuota(applicationSelector, "9")
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
expectGeneratedQuota("application", "9", "0")
|
||||
|
||||
Eventually(func() error {
|
||||
return setTenantQuota(applicationSelector, "8")
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
expectGeneratedQuota("application", "8", "0")
|
||||
})
|
||||
|
||||
It("allows a scope-first transition, observes newly selected usage, and then rejects an unsafe decrease", func() {
|
||||
ns := NewNamespace("", map[string]string{meta.TenantLabel: tenantName})
|
||||
NamespaceCreation(ns, owner, defaultTimeoutInterval).Should(Succeed())
|
||||
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
|
||||
|
||||
cs := ownerClient(owner)
|
||||
pod := MakePod(ns.Name, "scope-first-usage", nil, nil, "registry.k8s.io/pause:3.10", "", "")
|
||||
pod.Spec.Containers[0].Resources.Limits = corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("300m"),
|
||||
}
|
||||
EventuallyCreation(func() error {
|
||||
_, err := cs.CoreV1().Pods(ns.Name).Create(ctx, pod, metav1.CreateOptions{})
|
||||
|
||||
return err
|
||||
}).Should(Succeed())
|
||||
|
||||
Eventually(func() error {
|
||||
return setTenantQuota(nil, "8")
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
expectGeneratedQuota("", "8", "300m", ns.Name)
|
||||
|
||||
Eventually(func() error {
|
||||
return setTenantQuota(nil, "0")
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring(
|
||||
`rules[0].quota[0].hard["limits.cpu"] cannot be reduced to 0 while 300m is allocated`,
|
||||
)))
|
||||
|
||||
Eventually(func() error {
|
||||
current := &capsulev1beta2.Tenant{}
|
||||
if err := k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, current); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
delete(current.Spec.Rules[0].Quota[0].Hard, corev1.ResourceLimitsCPU)
|
||||
|
||||
return k8sClient.Update(ctx, current)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring(
|
||||
`rules[0].quota[0].hard["limits.cpu"] cannot be removed while 300m is allocated`,
|
||||
)))
|
||||
|
||||
applicationSelector := &metav1.LabelSelector{MatchLabels: map[string]string{selectorKey: "application"}}
|
||||
Eventually(func() error {
|
||||
return setTenantQuota(applicationSelector, "8")
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
expectGeneratedQuota("application", "8", "0")
|
||||
})
|
||||
|
||||
It("denies authorized non-admin creation, updates, and deletes of managed quotas", func() {
|
||||
tamperClient := impersonationClient(owner.Name, withDefaultGroups(nil))
|
||||
current := &capsulev1beta2.GlobalResourceQuota{}
|
||||
Eventually(func() error {
|
||||
return tamperClient.Get(ctx, quotaKey, current)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
|
||||
forged := &capsulev1beta2.GlobalResourceQuota{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: rbacName,
|
||||
Labels: map[string]string{meta.NewManagedByCapsuleLabel: meta.ValueController},
|
||||
},
|
||||
Spec: capsulev1beta2.GlobalResourceQuotaSpec{
|
||||
Quota: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{
|
||||
corev1.ResourceLimitsCPU: resource.MustParse("1"),
|
||||
}},
|
||||
},
|
||||
}
|
||||
err := tamperClient.Create(ctx, forged)
|
||||
Expect(err).To(MatchError(ContainSubstring(
|
||||
"Labeling resources as controller managed can only be done by the controller or administrators",
|
||||
)))
|
||||
|
||||
updated := current.DeepCopy()
|
||||
updated.Annotations = map[string]string{"e2e.projectcapsule.dev/tampered": "true"}
|
||||
err = tamperClient.Update(ctx, updated)
|
||||
Expect(err).To(MatchError(ContainSubstring(
|
||||
"Labeling resources as controller managed can only be done by the controller or administrators",
|
||||
)))
|
||||
|
||||
labelRemoval := current.DeepCopy()
|
||||
delete(labelRemoval.Labels, meta.NewManagedByCapsuleLabel)
|
||||
err = tamperClient.Update(ctx, labelRemoval)
|
||||
Expect(err).To(MatchError(ContainSubstring(
|
||||
"Labeling resources as controller managed can only be done by the controller or administrators",
|
||||
)))
|
||||
|
||||
err = tamperClient.Delete(ctx, current)
|
||||
Expect(err).To(MatchError(ContainSubstring(
|
||||
"Labeling resources as controller managed can only be done by the controller or administrators",
|
||||
)))
|
||||
|
||||
persisted := &capsulev1beta2.GlobalResourceQuota{}
|
||||
Expect(k8sClient.Get(ctx, quotaKey, persisted)).To(Succeed())
|
||||
Expect(persisted.Annotations).NotTo(HaveKey("e2e.projectcapsule.dev/tampered"))
|
||||
Expect(persisted.Labels).To(HaveKeyWithValue(meta.NewManagedByCapsuleLabel, meta.ValueController))
|
||||
})
|
||||
|
||||
It("does not apply managed protection to an unlabeled GlobalResourceQuota", func() {
|
||||
tamperClient := impersonationClient(owner.Name, withDefaultGroups(nil))
|
||||
unmanaged := &capsulev1beta2.GlobalResourceQuota{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: rbacName},
|
||||
Spec: capsulev1beta2.GlobalResourceQuotaSpec{
|
||||
Quota: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{
|
||||
corev1.ResourcePods: resource.MustParse("10"),
|
||||
}},
|
||||
},
|
||||
}
|
||||
Expect(tamperClient.Create(ctx, unmanaged)).To(Succeed())
|
||||
|
||||
current := &capsulev1beta2.GlobalResourceQuota{}
|
||||
Expect(tamperClient.Get(ctx, client.ObjectKey{Name: unmanaged.Name}, current)).To(Succeed())
|
||||
current.Annotations = map[string]string{"e2e.projectcapsule.dev/updated": "true"}
|
||||
Expect(tamperClient.Update(ctx, current)).To(Succeed())
|
||||
Expect(tamperClient.Delete(ctx, current)).To(Succeed())
|
||||
Eventually(func() bool {
|
||||
err := k8sClient.Get(ctx, client.ObjectKey{Name: unmanaged.Name}, &capsulev1beta2.GlobalResourceQuota{})
|
||||
|
||||
return apierrors.IsNotFound(err)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue())
|
||||
})
|
||||
|
||||
It("uses the owner reference to restore controller drift and keeps status reconciliation working", func() {
|
||||
controllerClient := impersonationClient(ControllerServiceAccountFull, nil)
|
||||
current := &capsulev1beta2.GlobalResourceQuota{}
|
||||
Expect(controllerClient.Get(ctx, quotaKey, current)).To(Succeed())
|
||||
|
||||
scopeDrift := current.DeepCopy()
|
||||
delete(scopeDrift.Spec.NamespaceSelectors[0].LabelSelector.MatchLabels, selectorKey)
|
||||
Expect(controllerClient.Update(ctx, scopeDrift)).To(Succeed())
|
||||
Eventually(func(g Gomega) {
|
||||
reconciled := &capsulev1beta2.GlobalResourceQuota{}
|
||||
g.Expect(k8sClient.Get(ctx, quotaKey, reconciled)).To(Succeed())
|
||||
g.Expect(reconciled.Spec.NamespaceSelectors[0].LabelSelector.MatchLabels).
|
||||
To(HaveKeyWithValue(selectorKey, "application"))
|
||||
g.Expect(reconciled.Status.ObservedGeneration).To(Equal(reconciled.Generation))
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
|
||||
Expect(controllerClient.Get(ctx, quotaKey, current)).To(Succeed())
|
||||
originalGeneration := current.Generation
|
||||
|
||||
drifted := current.DeepCopy()
|
||||
drifted.Spec.Quota.Hard[corev1.ResourceLimitsCPU] = resource.MustParse("9")
|
||||
Expect(controllerClient.Update(ctx, drifted)).To(Succeed())
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
reconciled := &capsulev1beta2.GlobalResourceQuota{}
|
||||
g.Expect(k8sClient.Get(ctx, quotaKey, reconciled)).To(Succeed())
|
||||
g.Expect(reconciled.Generation).To(BeNumerically(">", originalGeneration))
|
||||
reconciledLimit := reconciled.Spec.Quota.Hard[corev1.ResourceLimitsCPU]
|
||||
g.Expect(reconciledLimit.Cmp(resource.MustParse("8"))).To(Equal(0))
|
||||
g.Expect(reconciled.Status.ObservedGeneration).To(Equal(reconciled.Generation))
|
||||
|
||||
condition := reconciled.Status.Conditions.GetConditionByType(meta.ReadyCondition)
|
||||
g.Expect(condition).NotTo(BeNil())
|
||||
g.Expect(condition.Status).To(Equal(metav1.ConditionTrue))
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
})
|
||||
|
||||
It("deletes the managed quota through the controller before the Tenant finalizes", func() {
|
||||
current := &capsulev1beta2.Tenant{}
|
||||
Expect(k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, current)).To(Succeed())
|
||||
Expect(current.Finalizers).To(ContainElement(meta.ControllerFinalizer))
|
||||
|
||||
EventuallyDeletion(tnt)
|
||||
Eventually(func() bool {
|
||||
err := k8sClient.Get(ctx, quotaKey, &capsulev1beta2.GlobalResourceQuota{})
|
||||
|
||||
return apierrors.IsNotFound(err)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue())
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("managed GlobalResourceQuota administrator admission", Ordered,
|
||||
Label("config", "resourcequota", "admission", "managed", "skip-on-openshift"), func() {
|
||||
const (
|
||||
adminName = "e2e-managed-global-quota-admin"
|
||||
quotaName = "e2e-managed-global-quota-admin"
|
||||
)
|
||||
|
||||
ctx := context.Background()
|
||||
administrator := rbac.UserSpec{Name: adminName, Kind: rbac.UserOwner}
|
||||
originConfig := &capsulev1beta2.CapsuleConfiguration{}
|
||||
adminRole := &rbacv1.ClusterRole{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: adminName},
|
||||
Rules: []rbacv1.PolicyRule{{
|
||||
APIGroups: []string{capsulev1beta2.GroupVersion.Group},
|
||||
Resources: []string{"globalresourcequotas"},
|
||||
Verbs: []string{"get", "create", "update", "patch", "delete"},
|
||||
}},
|
||||
}
|
||||
adminBinding := &rbacv1.ClusterRoleBinding{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: adminName},
|
||||
RoleRef: rbacv1.RoleRef{
|
||||
APIGroup: rbacv1.GroupName,
|
||||
Kind: "ClusterRole",
|
||||
Name: adminName,
|
||||
},
|
||||
Subjects: []rbacv1.Subject{{
|
||||
APIGroup: rbacv1.GroupName,
|
||||
Kind: rbacv1.UserKind,
|
||||
Name: adminName,
|
||||
}},
|
||||
}
|
||||
|
||||
BeforeAll(func() {
|
||||
Expect(k8sClient.Get(ctx, client.ObjectKey{Name: defaultConfigurationName}, originConfig)).To(Succeed())
|
||||
|
||||
EventuallyCreation(func() error {
|
||||
adminRole.ResourceVersion = ""
|
||||
|
||||
return k8sClient.Create(ctx, adminRole)
|
||||
}).Should(Succeed())
|
||||
EventuallyCreation(func() error {
|
||||
adminBinding.ResourceVersion = ""
|
||||
|
||||
return k8sClient.Create(ctx, adminBinding)
|
||||
}).Should(Succeed())
|
||||
|
||||
ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) {
|
||||
configuration.Spec.Administrators = append(configuration.Spec.Administrators, administrator)
|
||||
})
|
||||
})
|
||||
|
||||
AfterAll(func() {
|
||||
controllerClient := impersonationClient(ControllerServiceAccountFull, nil)
|
||||
quota := &capsulev1beta2.GlobalResourceQuota{ObjectMeta: metav1.ObjectMeta{Name: quotaName}}
|
||||
Expect(ignoreNotFound(controllerClient.Delete(ctx, quota))).To(Succeed())
|
||||
|
||||
Eventually(func() error {
|
||||
current := &capsulev1beta2.CapsuleConfiguration{}
|
||||
if err := k8sClient.Get(ctx, client.ObjectKey{Name: originConfig.Name}, current); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
current.Spec = originConfig.Spec
|
||||
|
||||
return k8sClient.Update(ctx, current)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
|
||||
EventuallyDeletion(adminBinding)
|
||||
EventuallyDeletion(adminRole)
|
||||
})
|
||||
|
||||
It("allows a configured administrator to create, update, and delete a managed quota", func() {
|
||||
adminClient := impersonationClient(administrator.Name, nil)
|
||||
quota := &capsulev1beta2.GlobalResourceQuota{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: quotaName,
|
||||
Labels: map[string]string{meta.NewManagedByCapsuleLabel: meta.ValueController},
|
||||
},
|
||||
Spec: capsulev1beta2.GlobalResourceQuotaSpec{
|
||||
Quota: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{
|
||||
corev1.ResourcePods: resource.MustParse("10"),
|
||||
}},
|
||||
},
|
||||
}
|
||||
EventuallyCreation(func() error {
|
||||
quota.ResourceVersion = ""
|
||||
|
||||
return adminClient.Create(ctx, quota)
|
||||
}).Should(Succeed())
|
||||
|
||||
current := &capsulev1beta2.GlobalResourceQuota{}
|
||||
Expect(adminClient.Get(ctx, client.ObjectKey{Name: quotaName}, current)).To(Succeed())
|
||||
current.Annotations = map[string]string{"e2e.projectcapsule.dev/admin-updated": "true"}
|
||||
Expect(adminClient.Update(ctx, current)).To(Succeed())
|
||||
Expect(adminClient.Delete(ctx, current)).To(Succeed())
|
||||
Eventually(func() bool {
|
||||
err := k8sClient.Get(ctx, client.ObjectKey{Name: quotaName}, &capsulev1beta2.GlobalResourceQuota{})
|
||||
|
||||
return apierrors.IsNotFound(err)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue())
|
||||
})
|
||||
})
|
||||
+7
-15
@@ -391,8 +391,13 @@ var _ = Describe("rule-generated GlobalResourceQuota", Ordered, Label("resourceq
|
||||
|
||||
It("rejects invalid rule limits and projects valid changes to every namespace", func() {
|
||||
quotaKey := clientKey("", tenantutils.RuleGlobalResourceQuotaName(tnt, "service-count"))
|
||||
marker := "e2e.projectcapsule.dev/stable-identity"
|
||||
resourceName := corev1.ResourceServices
|
||||
initialQuota := &capsulev1beta2.GlobalResourceQuota{}
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(ctx, quotaKey, initialQuota)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
stableUID := initialQuota.UID
|
||||
|
||||
setServiceLimit := func(limit string) error {
|
||||
current := &capsulev1beta2.Tenant{}
|
||||
if err := k8sClient.Get(ctx, clientKey("", tenantName), current); err != nil {
|
||||
@@ -424,7 +429,7 @@ var _ = Describe("rule-generated GlobalResourceQuota", Ordered, Label("resourceq
|
||||
Eventually(func(g Gomega) {
|
||||
current := &capsulev1beta2.GlobalResourceQuota{}
|
||||
g.Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed())
|
||||
g.Expect(current.Annotations).To(HaveKeyWithValue(marker, "preserved"))
|
||||
g.Expect(current.UID).To(Equal(stableUID))
|
||||
g.Expect(current.Labels).To(HaveKeyWithValue(meta.RuleQuotaLabel, "service-count"))
|
||||
g.Expect(current.Status.ObservedGeneration).To(Equal(current.Generation))
|
||||
|
||||
@@ -457,19 +462,6 @@ var _ = Describe("rule-generated GlobalResourceQuota", Ordered, Label("resourceq
|
||||
}
|
||||
}
|
||||
|
||||
Eventually(func() error {
|
||||
current := &capsulev1beta2.GlobalResourceQuota{}
|
||||
if err := k8sClient.Get(ctx, quotaKey, current); err != nil {
|
||||
return err
|
||||
}
|
||||
if current.Annotations == nil {
|
||||
current.Annotations = map[string]string{}
|
||||
}
|
||||
current.Annotations[marker] = "preserved"
|
||||
|
||||
return k8sClient.Update(ctx, current)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
|
||||
// The concurrent admission scenario immediately before this test creates
|
||||
// exactly five Services. Wait for native quota accounting before changing
|
||||
// the shared limit.
|
||||
|
||||
Reference in New Issue
Block a user