fix: improve rules api (#1961)

* fix(controller): decode old object for delete requests

Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com>

* chore: modernize golang

Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com>

* chore: modernize golang

Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com>

* chore: modernize golang

Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com>

* fix: preserve ca-bundles injected from external providers

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* fix: improve rules api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* fix: improve rules api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* fix: improve rules api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* fix: improve rules api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* fix: improve rules api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* fix: improve rules api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* fix: improve rules api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* fix: improve rules api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

---------

Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com>
Signed-off-by: Oliver Baehler <oliver@sudo-i.net>
This commit is contained in:
Oliver Bähler
2026-06-10 00:18:40 +02:00
committed by GitHub
parent f4af5c8043
commit 34262c5536
47 changed files with 4271 additions and 980 deletions
@@ -173,6 +173,7 @@ type ServiceAccountClient struct {
// +kubebuilder:resource:scope=Cluster
// +kubebuilder:storageversion
// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="Reconcile status"
// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="Reconcile Message"
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
// CapsuleConfiguration is the Schema for the Capsule configuration API.
+1
View File
@@ -64,6 +64,7 @@ type CustomQuotaSpecSourceConfig struct {
// +kubebuilder:printcolumn:name="Available",type="string",JSONPath=".status.usage.available",description="The total amount available"
// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="Reconcile Status"
// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="Reconcile Message"
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
type CustomQuota struct {
metav1.TypeMeta `json:",inline"`
+1
View File
@@ -25,6 +25,7 @@ type GlobalCustomQuotaSpec struct {
// +kubebuilder:printcolumn:name="Available",type="string",JSONPath=".status.usage.available",description="The total amount available"
// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="Reconcile Status"
// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="Reconcile Message"
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
type GlobalCustomQuota struct {
metav1.TypeMeta `json:",inline"`
-1
View File
@@ -75,7 +75,6 @@ type QuantityLedgerSpec struct {
// +kubebuilder:printcolumn:name="TargetNamespace",type=string,JSONPath=`.spec.targetRef.namespace`
// +kubebuilder:printcolumn:name="TargetName",type=string,JSONPath=`.spec.targetRef.name`
// +kubebuilder:printcolumn:name="Reserved",type=string,JSONPath=`.status.reserved`
// +kubebuilder:printcolumn:name="Reservations",type=integer,JSONPath=`.status.reservations.size()`
// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
type QuantityLedger struct {
metav1.TypeMeta `json:",inline"`
+1
View File
@@ -33,6 +33,7 @@ type RuleStatusStatus struct {
// +kubebuilder:subresource:status
// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="Ready Status"
// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="Ready Message"
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age"
type RuleStatus struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitzero"`
@@ -19,6 +19,10 @@ spec:
jsonPath: .status.conditions[?(@.type=="Ready")].status
name: Ready
type: string
- description: Reconcile Message
jsonPath: .status.conditions[?(@.type=="Ready")].message
name: Status
type: string
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
@@ -37,6 +37,10 @@ spec:
jsonPath: .status.conditions[?(@.type=="Ready")].message
name: Status
type: string
- description: Age
jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1beta2
schema:
openAPIV3Schema:
@@ -37,6 +37,10 @@ spec:
jsonPath: .status.conditions[?(@.type=="Ready")].message
name: Status
type: string
- description: Age
jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1beta2
schema:
openAPIV3Schema:
@@ -29,9 +29,6 @@ spec:
- jsonPath: .status.reserved
name: Reserved
type: string
- jsonPath: .status.reservations.size()
name: Reservations
type: integer
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
@@ -23,6 +23,10 @@ spec:
jsonPath: .status.conditions[?(@.type=="Ready")].message
name: Message
type: string
- description: Age
jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1beta2
schema:
openAPIV3Schema:
@@ -64,46 +68,54 @@ spec:
- deny
- audit
type: string
registries:
description: |-
Define registries which are allowed to be used within this tenant
The rules are aggregated, since you can use Regular Expressions the match registry endpoints
items:
properties:
exp:
description: Expression used to evaluate regex
workloads:
description: Enforcement for Workloads (Pods)
properties:
qosClasses:
description: |-
Define Pod QoS classes matched by this enforcement rule.
Supported values are Guaranteed, Burstable and BestEffort.
items:
description: PodQOSClass defines the supported qos classes
of Pods.
type: string
negate:
default: false
description: Negate regular Expression
type: boolean
policy:
description: Allowed PullPolicy for the given registry.
Supplying no value allows all policies.
items:
description: PullPolicy describes a policy for if/when
to pull a container image
type: string
type: array
url:
description: |-
Deprecated: Use exp field
OCI Registry endpoint, is treated as regular expression.
type: string
validation:
default:
- pod/images
type: array
registries:
description: |-
Define registries which are allowed to be used within this tenant
The rules are aggregated, since you can use Regular Expressions the match registry endpoints
items:
properties:
exp:
description: Expression used to evaluate regex
type: string
negate:
default: false
description: Negate regular Expression
type: boolean
policy:
description: Allowed PullPolicy for the given registry.
Supplying no value allows all policies.
items:
description: PullPolicy describes a policy for if/when
to pull a container image
type: string
type: array
type: object
type: array
targets:
description: |-
Define the enforcement targets this rule applies to.
If empty, each webhook applies its own backwards-compatible default.
items:
enum:
- pod/initcontainers
- pod/ephemeralcontainers
- pod/containers
- pod/volumes
description: Requesting Resources
items:
enum:
- pod/images
- pod/volumes
type: string
type: array
type: object
type: array
type: string
type: array
type: object
type: object
type: object
type: array
@@ -193,46 +205,54 @@ spec:
- deny
- audit
type: string
registries:
description: |-
Define registries which are allowed to be used within this tenant
The rules are aggregated, since you can use Regular Expressions the match registry endpoints
items:
properties:
exp:
description: Expression used to evaluate regex
workloads:
description: Enforcement for Workloads (Pods)
properties:
qosClasses:
description: |-
Define Pod QoS classes matched by this enforcement rule.
Supported values are Guaranteed, Burstable and BestEffort.
items:
description: PodQOSClass defines the supported qos classes
of Pods.
type: string
negate:
default: false
description: Negate regular Expression
type: boolean
policy:
description: Allowed PullPolicy for the given registry.
Supplying no value allows all policies.
items:
description: PullPolicy describes a policy for if/when
to pull a container image
type: string
type: array
url:
description: |-
Deprecated: Use exp field
OCI Registry endpoint, is treated as regular expression.
type: string
validation:
default:
- pod/images
type: array
registries:
description: |-
Define registries which are allowed to be used within this tenant
The rules are aggregated, since you can use Regular Expressions the match registry endpoints
items:
properties:
exp:
description: Expression used to evaluate regex
type: string
negate:
default: false
description: Negate regular Expression
type: boolean
policy:
description: Allowed PullPolicy for the given registry.
Supplying no value allows all policies.
items:
description: PullPolicy describes a policy for
if/when to pull a container image
type: string
type: array
type: object
type: array
targets:
description: |-
Define the enforcement targets this rule applies to.
If empty, each webhook applies its own backwards-compatible default.
items:
enum:
- pod/initcontainers
- pod/ephemeralcontainers
- pod/containers
- pod/volumes
description: Requesting Resources
items:
enum:
- pod/images
- pod/volumes
type: string
type: array
type: object
type: array
type: string
type: array
type: object
type: object
type: object
rules:
@@ -258,46 +278,54 @@ spec:
- deny
- audit
type: string
registries:
description: |-
Define registries which are allowed to be used within this tenant
The rules are aggregated, since you can use Regular Expressions the match registry endpoints
items:
properties:
exp:
description: Expression used to evaluate regex
workloads:
description: Enforcement for Workloads (Pods)
properties:
qosClasses:
description: |-
Define Pod QoS classes matched by this enforcement rule.
Supported values are Guaranteed, Burstable and BestEffort.
items:
description: PodQOSClass defines the supported qos
classes of Pods.
type: string
negate:
default: false
description: Negate regular Expression
type: boolean
policy:
description: Allowed PullPolicy for the given registry.
Supplying no value allows all policies.
items:
description: PullPolicy describes a policy for if/when
to pull a container image
type: string
type: array
url:
description: |-
Deprecated: Use exp field
OCI Registry endpoint, is treated as regular expression.
type: string
validation:
default:
- pod/images
type: array
registries:
description: |-
Define registries which are allowed to be used within this tenant
The rules are aggregated, since you can use Regular Expressions the match registry endpoints
items:
properties:
exp:
description: Expression used to evaluate regex
type: string
negate:
default: false
description: Negate regular Expression
type: boolean
policy:
description: Allowed PullPolicy for the given
registry. Supplying no value allows all policies.
items:
description: PullPolicy describes a policy for
if/when to pull a container image
type: string
type: array
type: object
type: array
targets:
description: |-
Define the enforcement targets this rule applies to.
If empty, each webhook applies its own backwards-compatible default.
items:
enum:
- pod/initcontainers
- pod/ephemeralcontainers
- pod/containers
- pod/volumes
description: Requesting Resources
items:
enum:
- pod/images
- pod/volumes
type: string
type: array
type: object
type: array
type: string
type: array
type: object
type: object
type: object
type: array
@@ -2515,46 +2515,54 @@ spec:
- deny
- audit
type: string
registries:
description: |-
Define registries which are allowed to be used within this tenant
The rules are aggregated, since you can use Regular Expressions the match registry endpoints
items:
properties:
exp:
description: Expression used to evaluate regex
workloads:
description: Enforcement for Workloads (Pods)
properties:
qosClasses:
description: |-
Define Pod QoS classes matched by this enforcement rule.
Supported values are Guaranteed, Burstable and BestEffort.
items:
description: PodQOSClass defines the supported qos
classes of Pods.
type: string
negate:
default: false
description: Negate regular Expression
type: boolean
policy:
description: Allowed PullPolicy for the given registry.
Supplying no value allows all policies.
items:
description: PullPolicy describes a policy for if/when
to pull a container image
type: string
type: array
url:
description: |-
Deprecated: Use exp field
OCI Registry endpoint, is treated as regular expression.
type: string
validation:
default:
- pod/images
type: array
registries:
description: |-
Define registries which are allowed to be used within this tenant
The rules are aggregated, since you can use Regular Expressions the match registry endpoints
items:
properties:
exp:
description: Expression used to evaluate regex
type: string
negate:
default: false
description: Negate regular Expression
type: boolean
policy:
description: Allowed PullPolicy for the given
registry. Supplying no value allows all policies.
items:
description: PullPolicy describes a policy for
if/when to pull a container image
type: string
type: array
type: object
type: array
targets:
description: |-
Define the enforcement targets this rule applies to.
If empty, each webhook applies its own backwards-compatible default.
items:
enum:
- pod/initcontainers
- pod/ephemeralcontainers
- pod/containers
- pod/volumes
description: Requesting Resources
items:
enum:
- pod/images
- pod/volumes
type: string
type: array
type: object
type: array
type: string
type: array
type: object
type: object
namespaceSelector:
description: Select namespaces which are going to be targeted
@@ -2606,7 +2614,7 @@ spec:
permissions:
description: Permissions for given rule
properties:
rules:
promotions:
description: |-
Define Promotion Rules which distributed additional ClusterRoles across the Tenant
for promoted ServiceAccounts.
@@ -3125,23 +3133,6 @@ spec:
to pull a container image
type: string
type: array
url:
description: |-
Deprecated: Use exp field
OCI Registry endpoint, is treated as regular expression.
type: string
validation:
default:
- pod/images
- pod/volumes
description: Requesting Resources
items:
enum:
- pod/images
- pod/volumes
type: string
type: array
type: object
type: array
type: object
+1
View File
@@ -564,6 +564,7 @@ func main() {
pod.ContainerRegistry(cfg, registryCache),
pod.PriorityClass(),
pod.RuntimeClass(),
pod.QoSClass(cfg),
),
),
route.Ingress(ingress.Class(cfg, kubeVersion), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()),
@@ -165,6 +165,62 @@ var _ = Describe("GlobalTenantResource", Ordered, Label("replications", "global"
})
It("skips applying resources to terminating namespaces and removes them from processedItems", func() {
terminatingNamespace := tenantANamespaces[2]
releaseNamespace := holdNamespaceTerminating(ctx, terminatingNamespace)
defer releaseNamespace()
gtr := newRawConfigMapGlobalTenantResource("gtr-skip-terminating-namespace", map[string]string{
"mode": "active",
})
gtr.Spec.TenantSelector = metav1.LabelSelector{
MatchLabels: map[string]string{"energy": "solar"},
}
renameFirstRawConfigMap(gtr, "gtr-skip-terminating")
EventuallyCreation(func() error {
return k8sClient.Create(ctx, gtr)
}).Should(Succeed())
By("verifying non-terminating selected namespaces still receive the resource")
for _, ns := range tenantANamespaces[:2] {
expectConfigMapData(ns, "gtr-skip-terminating", map[string]string{
"mode": "active",
})
}
By("verifying the terminating selected namespace is skipped")
Consistently(func() error {
return k8sClient.Get(ctx, types.NamespacedName{
Name: "gtr-skip-terminating",
Namespace: terminatingNamespace,
}, &corev1.ConfigMap{})
}, 2*resyncPeriod.Duration, defaultPollInterval).Should(HaveOccurred())
By("verifying non-selected tenants do not receive the resource")
for _, ns := range tenantBNamespaces {
expectConfigMapAbsent(ns, "gtr-skip-terminating")
}
By("verifying the terminating namespace item is not kept in processedItems")
Eventually(func(g Gomega) {
current := &capsulev1beta2.GlobalTenantResource{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{
Name: gtr.Name,
}, current)).To(Succeed())
for _, item := range current.Status.ProcessedItems {
g.Expect(item.Name).To(Equal("gtr-skip-terminating"))
g.Expect(item.Namespace).ToNot(Equal(terminatingNamespace))
g.Expect(item.Status).To(Equal(metav1.ConditionTrue))
}
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
It("fails to replicate namespacedItems when the impersonated service account cannot read source resources", func() {
saName := "gtr-no-namespaceditem-read"
ensureServiceAccount("capsule-system", saName)
+73
View File
@@ -134,6 +134,79 @@ var _ = Describe("TenantResource SSA", Ordered, Label("replications", "namespace
})
})
It("skips applying resources to terminating namespaces and removes them from processedItems", func() {
terminatingNamespace := targetNamespaces[2]
releaseNamespace := holdNamespaceTerminating(ctx, terminatingNamespace)
defer releaseNamespace()
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{
Name: "skip-terminating-namespace",
Namespace: baseNamespace,
},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: metav1.Duration{Duration: 5 * time.Second},
Resources: []capsulev1beta2.ResourceSpec{{
RawItems: []capsulev1beta2.RawExtension{{
RawExtension: runtime.RawExtension{
Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "tr-skip-terminating",
},
Data: map[string]string{
"mode": "active",
},
},
},
}},
}},
},
},
}
EventuallyCreation(func() error {
return k8sClient.Create(ctx, tr)
}).Should(Succeed())
By("verifying non-terminating namespaces still receive the resource")
for _, ns := range targetNamespaces[:2] {
expectConfigMapData(ns, "tr-skip-terminating", map[string]string{
"mode": "active",
})
}
By("verifying the terminating namespace is skipped")
Consistently(func() error {
return k8sClient.Get(ctx, types.NamespacedName{
Name: "tr-skip-terminating",
Namespace: terminatingNamespace,
}, &corev1.ConfigMap{})
}, 2*resyncPeriod.Duration, defaultPollInterval).Should(HaveOccurred())
By("verifying the terminating namespace item is not kept in processedItems")
Eventually(func(g Gomega) {
current := &capsulev1beta2.TenantResource{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{
Name: tr.Name,
Namespace: tr.Namespace,
}, current)).To(Succeed())
for _, item := range current.Status.ProcessedItems {
g.Expect(item.Name).To(Equal("tr-skip-terminating"))
g.Expect(item.Namespace).ToNot(Equal(terminatingNamespace))
g.Expect(item.Status).To(Equal(metav1.ConditionTrue))
}
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
Context("generators and template context", func() {
It("fails when a templated namespace resolves to a forbidden namespace", func() {
+590
View File
@@ -0,0 +1,590 @@
// Copyright 2020-2026 Project Capsule Authors.
// SPDX-License-Identifier: Apache-2.0
package e2e
import (
"context"
"fmt"
"strings"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/controller-runtime/pkg/client"
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
"github.com/projectcapsule/capsule/pkg/api/meta"
"github.com/projectcapsule/capsule/pkg/api/rbac"
"github.com/projectcapsule/capsule/pkg/api/rules"
)
var _ = Describe("enforcing pod QoS namespace rules", Ordered, Label("tenant", "rules", "enforce", "workloads", "qos"), func() {
const ownerName = "e2e-rules-qos"
var tnt *capsulev1beta2.Tenant
newTenant := func() *capsulev1beta2.Tenant {
return &capsulev1beta2.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "e2e-rule-qos",
Labels: map[string]string{
"env": "e2e",
},
},
Spec: capsulev1beta2.TenantSpec{
Owners: rbac.OwnerListSpec{
{
CoreOwnerSpec: rbac.CoreOwnerSpec{
UserSpec: rbac.UserSpec{
Name: ownerName,
Kind: "User",
},
},
},
},
Rules: []*rules.NamespaceRuleBodyTenant{
{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeDeny,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
QoSClasses: []corev1.PodQOSClass{
corev1.PodQOSBestEffort,
},
},
},
},
},
{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeAudit,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
QoSClasses: []corev1.PodQOSClass{
corev1.PodQOSBurstable,
},
},
},
},
},
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"allow-best-effort": "true",
},
},
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeAllow,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
QoSClasses: []corev1.PodQOSClass{
corev1.PodQOSBestEffort,
},
},
},
},
},
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"deny-burstable": "true",
},
},
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeDeny,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
QoSClasses: []corev1.PodQOSClass{
corev1.PodQOSBurstable,
},
},
},
},
},
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"containers-target": "true",
},
},
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeDeny,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
Targets: []rules.WorkloadValidationTarget{
rules.ValidateContainers,
},
QoSClasses: []corev1.PodQOSClass{
corev1.PodQOSBestEffort,
},
},
},
},
},
},
},
}
}
type expectedQoSStatusRule struct {
action rules.ActionType
targets []rules.WorkloadValidationTarget
qosClasses []corev1.PodQOSClass
}
expectNamespaceStatusRules := func(nsName string, want []expectedQoSStatusRule) {
Eventually(func(g Gomega) {
nsStatus := &capsulev1beta2.RuleStatus{}
g.Expect(k8sClient.Get(
context.Background(),
client.ObjectKey{Name: meta.NameForManagedRuleStatus(), Namespace: nsName},
nsStatus,
)).To(Succeed())
g.Expect(nsStatus.Status.Rules).To(HaveLen(len(want)))
for i, expected := range want {
gotRule := nsStatus.Status.Rules[i]
g.Expect(gotRule).NotTo(BeNil())
g.Expect(gotRule.Enforce.Action).To(Equal(expected.action))
g.Expect(gotRule.Enforce.Workloads.Targets).To(Equal(expected.targets))
g.Expect(gotRule.Enforce.Workloads.QoSClasses).To(Equal(expected.qosClasses))
}
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
createPodAndExpectDenied := func(cs kubernetes.Interface, nsName string, pod *corev1.Pod, substrings ...string) {
base := pod.DeepCopy()
baseName := base.Name
if baseName == "" {
baseName = "pod"
}
Eventually(func() error {
p := base.DeepCopy()
p.Name = fmt.Sprintf("%s-%d", baseName, time.Now().UnixNano()%1e6)
_, err := cs.CoreV1().Pods(nsName).Create(context.Background(), p, metav1.CreateOptions{})
if err == nil {
_ = cs.CoreV1().Pods(nsName).Delete(context.Background(), p.Name, metav1.DeleteOptions{})
return fmt.Errorf("expected create to be denied, but it succeeded")
}
if apierrors.IsAlreadyExists(err) {
return fmt.Errorf("unexpected AlreadyExists: %v", err)
}
msg := err.Error()
for _, substring := range substrings {
if !strings.Contains(msg, substring) {
return fmt.Errorf("expected error to contain %q, got: %s", substring, msg)
}
}
return nil
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
createPodAndExpectAllowed := func(cs kubernetes.Interface, nsName string, pod *corev1.Pod) {
EventuallyCreation(func() error {
_, err := cs.CoreV1().Pods(nsName).Create(context.Background(), pod, metav1.CreateOptions{})
return err
}).Should(Succeed())
}
expectAuditEvent := func(cs kubernetes.Interface, nsName string, podName string, substrings ...string) {
Eventually(func() error {
events, err := cs.CoreV1().Events(nsName).List(context.Background(), metav1.ListOptions{})
if err != nil {
return err
}
for _, event := range events.Items {
if event.InvolvedObject.Name != podName {
continue
}
msg := event.Message
matched := true
for _, substring := range substrings {
if !strings.Contains(msg, substring) {
matched = false
break
}
}
if matched {
return nil
}
}
return fmt.Errorf("expected audit event for pod %q containing %q", podName, substrings)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
bestEffortPod := func(name string) *corev1.Pod {
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: corev1.PodSpec{
SecurityContext: nobodyPodSecurityContext(),
Containers: []corev1.Container{
{
Name: "c",
Image: "registry.k8s.io/pause:3.9",
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: restrictedContainerSecurityContext(),
},
},
},
}
}
burstablePod := func(name string) *corev1.Pod {
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: corev1.PodSpec{
SecurityContext: nobodyPodSecurityContext(),
Containers: []corev1.Container{
{
Name: "c",
Image: "registry.k8s.io/pause:3.9",
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: restrictedContainerSecurityContext(),
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("10m"),
corev1.ResourceMemory: resource.MustParse("16Mi"),
},
},
},
},
},
}
}
guaranteedPod := func(name string) *corev1.Pod {
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: corev1.PodSpec{
SecurityContext: nobodyPodSecurityContext(),
Containers: []corev1.Container{
{
Name: "c",
Image: "registry.k8s.io/pause:3.9",
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: restrictedContainerSecurityContext(),
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("10m"),
corev1.ResourceMemory: resource.MustParse("16Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("10m"),
corev1.ResourceMemory: resource.MustParse("16Mi"),
},
},
},
},
},
}
}
podWithInitContainerQoS := func(name string, initResources corev1.ResourceRequirements, containerResources corev1.ResourceRequirements) *corev1.Pod {
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: corev1.PodSpec{
SecurityContext: nobodyPodSecurityContext(),
InitContainers: []corev1.Container{
{
Name: "init",
Image: "registry.k8s.io/pause:3.9",
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: restrictedContainerSecurityContext(),
Resources: initResources,
},
},
Containers: []corev1.Container{
{
Name: "c",
Image: "registry.k8s.io/pause:3.9",
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: restrictedContainerSecurityContext(),
Resources: containerResources,
},
},
},
}
}
burstableResources := func() corev1.ResourceRequirements {
return corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("10m"),
corev1.ResourceMemory: resource.MustParse("16Mi"),
},
}
}
guaranteedResources := func() corev1.ResourceRequirements {
return corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("10m"),
corev1.ResourceMemory: resource.MustParse("16Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("10m"),
corev1.ResourceMemory: resource.MustParse("16Mi"),
},
}
}
JustBeforeEach(func() {
tnt = newTenant()
EventuallyCreation(func() error {
tnt.ResourceVersion = ""
return k8sClient.Create(context.TODO(), tnt)
}).Should(Succeed())
TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval)
})
JustAfterEach(func() {
EventuallyDeletion(tnt)
})
It("stores QoS workload rules as independent status rule blocks", func() {
ns := NewNamespace("", map[string]string{
meta.TenantLabel: tnt.GetName(),
})
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
expectNamespaceStatusRules(ns.GetName(), []expectedQoSStatusRule{
{
action: rules.ActionTypeDeny,
qosClasses: []corev1.PodQOSClass{
corev1.PodQOSBestEffort,
},
},
{
action: rules.ActionTypeAudit,
qosClasses: []corev1.PodQOSClass{
corev1.PodQOSBurstable,
},
},
})
})
It("stores namespace-selector matched QoS rules as additional status rule blocks", func() {
ns := NewNamespace("", map[string]string{
"allow-best-effort": "true",
meta.TenantLabel: tnt.GetName(),
})
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
expectNamespaceStatusRules(ns.GetName(), []expectedQoSStatusRule{
{
action: rules.ActionTypeDeny,
qosClasses: []corev1.PodQOSClass{
corev1.PodQOSBestEffort,
},
},
{
action: rules.ActionTypeAudit,
qosClasses: []corev1.PodQOSClass{
corev1.PodQOSBurstable,
},
},
{
action: rules.ActionTypeAllow,
qosClasses: []corev1.PodQOSClass{
corev1.PodQOSBestEffort,
},
},
})
})
It("denies BestEffort pods by default", func() {
ns := NewNamespace("", map[string]string{
meta.TenantLabel: tnt.GetName(),
})
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
createPodAndExpectDenied(cs, ns.Name, bestEffortPod("besteffort-denied"),
"BestEffort",
"denied",
)
})
It("allows Guaranteed pods when no QoS rule matches them", func() {
ns := NewNamespace("", map[string]string{
meta.TenantLabel: tnt.GetName(),
})
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
createPodAndExpectAllowed(cs, ns.Name, guaranteedPod("guaranteed-allowed"))
})
It("audits Burstable pods by allowing admission and emitting an event", func() {
ns := NewNamespace("", map[string]string{
meta.TenantLabel: tnt.GetName(),
})
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
pod := burstablePod("burstable-audited")
createPodAndExpectAllowed(cs, ns.Name, pod)
expectAuditEvent(cs, ns.Name, pod.Name,
"Burstable",
"audit QoS rule",
)
})
It("allows BestEffort pods when a later namespace-selected allow rule matches", func() {
ns := NewNamespace("", map[string]string{
"allow-best-effort": "true",
meta.TenantLabel: tnt.GetName(),
})
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
createPodAndExpectAllowed(cs, ns.Name, bestEffortPod("besteffort-allowed"))
})
It("denies Burstable pods when a later namespace-selected deny rule overrides an earlier audit rule", func() {
ns := NewNamespace("", map[string]string{
"deny-burstable": "true",
meta.TenantLabel: tnt.GetName(),
})
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
createPodAndExpectDenied(cs, ns.Name, burstablePod("burstable-denied"),
"Burstable",
"denied",
)
})
It("computes QoS across init containers and regular containers", func() {
ns := NewNamespace("", map[string]string{
meta.TenantLabel: tnt.GetName(),
})
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
pod := podWithInitContainerQoS(
"init-causes-burstable",
burstableResources(),
guaranteedResources(),
)
createPodAndExpectAllowed(cs, ns.Name, pod)
expectAuditEvent(cs, ns.Name, pod.Name,
"Burstable",
"audit QoS rule",
)
})
It("uses empty targets as all targets for QoS rules", func() {
ns := NewNamespace("", map[string]string{
meta.TenantLabel: tnt.GetName(),
})
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
createPodAndExpectDenied(cs, ns.Name, bestEffortPod("empty-targets-denied"),
"BestEffort",
"denied",
)
})
It("applies QoS rules when explicit workload targets are configured", func() {
ns := NewNamespace("", map[string]string{
"containers-target": "true",
meta.TenantLabel: tnt.GetName(),
})
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
expectNamespaceStatusRules(ns.GetName(), []expectedQoSStatusRule{
{
action: rules.ActionTypeDeny,
qosClasses: []corev1.PodQOSClass{
corev1.PodQOSBestEffort,
},
},
{
action: rules.ActionTypeAudit,
qosClasses: []corev1.PodQOSClass{
corev1.PodQOSBurstable,
},
},
{
action: rules.ActionTypeDeny,
targets: []rules.WorkloadValidationTarget{
rules.ValidateContainers,
},
qosClasses: []corev1.PodQOSClass{
corev1.PodQOSBestEffort,
},
},
})
createPodAndExpectDenied(cs, ns.Name, bestEffortPod("explicit-target-denied"),
"BestEffort",
"denied",
)
})
})
@@ -24,6 +24,21 @@ import (
"github.com/projectcapsule/capsule/pkg/api/rules"
)
var (
targetContainers = []rules.WorkloadValidationTarget{
"pod/containers",
}
targetEphemeralContainers = []rules.WorkloadValidationTarget{
"pod/ephemeralcontainers",
}
targetInitContainers = []rules.WorkloadValidationTarget{
"pod/initcontainers",
}
targetVolumes = []rules.WorkloadValidationTarget{
"pod/volumes",
}
)
var _ = Describe("enforcing container registry namespace rules", Ordered, Label("tenant", "rules", "images", "registry"), func() {
const ownerName = "e2e-rules-registry"
@@ -50,15 +65,15 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
},
Rules: []*rules.NamespaceRuleBodyTenant{
{
NamespaceRuleBodyNamespace: rules.NamespaceRuleBodyNamespace{
Enforce: rules.NamespaceRuleEnforceBody{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeAllow,
Registries: []rules.OCIRegistry{
{
Registry: "harbor/.*",
Validation: []rules.RegistryValidationTarget{
rules.ValidateImages,
rules.ValidateVolumes,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
Registries: []rules.OCIRegistry{
{
RegExpression: api.RegExpression{
Expression: "harbor/.*",
},
},
},
},
@@ -66,18 +81,121 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
},
},
{
NamespaceRuleBodyNamespace: rules.NamespaceRuleBodyNamespace{
Enforce: rules.NamespaceRuleEnforceBody{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeDeny,
Registries: []rules.OCIRegistry{
{
Registry: "harbor/customer/.*",
Policy: []corev1.PullPolicy{
corev1.PullNever,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
Targets: targetContainers,
Registries: []rules.OCIRegistry{
{
RegExpression: api.RegExpression{
Expression: "harbor/customer/containers/.*",
},
},
Validation: []rules.RegistryValidationTarget{
rules.ValidateImages,
rules.ValidateVolumes,
},
},
},
},
},
{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeDeny,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
Targets: targetInitContainers,
Registries: []rules.OCIRegistry{
{
RegExpression: api.RegExpression{
Expression: "harbor/customer/init/.*",
},
},
},
},
},
},
},
{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeDeny,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
Targets: targetEphemeralContainers,
Registries: []rules.OCIRegistry{
{
RegExpression: api.RegExpression{
Expression: "harbor/customer/debug/.*",
},
},
},
},
},
},
},
{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeDeny,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
Targets: targetVolumes,
Registries: []rules.OCIRegistry{
{
RegExpression: api.RegExpression{
Expression: "harbor/customer/volume/.*",
},
},
},
},
},
},
},
{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeAudit,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
Targets: targetContainers,
Registries: []rules.OCIRegistry{
{
RegExpression: api.RegExpression{
Expression: "audit/containers/.*",
},
},
},
},
},
},
},
{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeAudit,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
Targets: targetVolumes,
Registries: []rules.OCIRegistry{
{
RegExpression: api.RegExpression{
Expression: "audit/volumes/.*",
},
},
},
},
},
},
},
{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeAllow,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
Targets: targetContainers,
Registries: []rules.OCIRegistry{
{
RegExpression: api.RegExpression{
Expression: "policy/.*",
},
Policy: []corev1.PullPolicy{
corev1.PullNever,
},
},
},
},
@@ -90,31 +208,16 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
"environment": "prod",
},
},
NamespaceRuleBodyNamespace: rules.NamespaceRuleBodyNamespace{
Enforce: rules.NamespaceRuleEnforceBody{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeAllow,
Registries: []rules.OCIRegistry{
{
Registry: "harbor/customer/prod-image/.*",
Validation: []rules.RegistryValidationTarget{
rules.ValidateImages,
rules.ValidateVolumes,
},
},
},
},
},
},
{
NamespaceRuleBodyNamespace: rules.NamespaceRuleBodyNamespace{
Enforce: rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeAudit,
Registries: []rules.OCIRegistry{
{
Registry: "audit/.*",
Validation: []rules.RegistryValidationTarget{
rules.ValidateImages,
rules.ValidateVolumes,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
Targets: targetContainers,
Registries: []rules.OCIRegistry{
{
RegExpression: api.RegExpression{
Expression: "harbor/customer/containers/prod/.*",
},
},
},
},
@@ -127,17 +230,17 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
"negate": "true",
},
},
NamespaceRuleBodyNamespace: rules.NamespaceRuleBodyNamespace{
Enforce: rules.NamespaceRuleEnforceBody{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rules.ActionTypeDeny,
Registries: []rules.OCIRegistry{
{
RegExpression: api.RegExpression{
Expression: "trusted/.*",
Negate: true,
},
Validation: []rules.RegistryValidationTarget{
rules.ValidateImages,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
Targets: targetContainers,
Registries: []rules.OCIRegistry{
{
RegExpression: api.RegExpression{
Expression: "trusted/.*",
Negate: true,
},
},
},
},
@@ -151,6 +254,7 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
type expectedStatusRule struct {
action rules.ActionType
targets []rules.WorkloadValidationTarget
expressions []string
negated []bool
}
@@ -158,26 +262,42 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
expectNamespaceStatusRules := func(nsName string, want []expectedStatusRule) {
Eventually(func(g Gomega) {
nsStatus := &capsulev1beta2.RuleStatus{}
g.Expect(k8sClient.Get(
context.Background(),
client.ObjectKey{Name: meta.NameForManagedRuleStatus(), Namespace: nsName},
client.ObjectKey{
Name: meta.NameForManagedRuleStatus(),
Namespace: nsName,
},
nsStatus,
)).To(Succeed())
g.Expect(nsStatus.Status.Rules).To(HaveLen(len(want)))
for i, expected := range want {
gotRule := nsStatus.Status.Rules[i]
g.Expect(gotRule).NotTo(BeNil())
g.Expect(gotRule.Enforce.Action).To(Equal(expected.action))
g.Expect(gotRule.Enforce.Registries).To(HaveLen(len(expected.expressions)))
got := nsStatus.Status.Rules[i]
g.Expect(got).NotTo(BeNil())
g.Expect(got.Enforce).NotTo(BeNil())
g.Expect(got.Enforce.Action).To(Equal(expected.action))
if len(expected.targets) == 0 {
g.Expect(got.Enforce.Workloads.Targets).To(BeEmpty())
} else {
g.Expect(got.Enforce.Workloads.Targets).To(Equal(expected.targets))
}
g.Expect(got.Enforce.Workloads.Registries).To(HaveLen(len(expected.expressions)))
for j, expectedExpression := range expected.expressions {
expr := gotRule.Enforce.Registries[j].Expression()
expr := got.Enforce.Workloads.Registries[j].Expression()
g.Expect(expr.Expression).To(Equal(expectedExpression))
if len(expected.negated) > j {
g.Expect(expr.Negate).To(Equal(expected.negated[j]))
} else {
g.Expect(expr.Negate).To(BeFalse())
}
}
}
@@ -317,6 +437,26 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
EventuallyDeletion(tnt)
})
It("denies an allowed registry reference when its pull policy is not permitted", func() {
ns := NewNamespace("", map[string]string{
meta.TenantLabel: tnt.GetName(),
})
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
pod := restrictedPod("policy-denied", "policy/team/app:1", corev1.PullIfNotPresent)
createPodAndExpectDenied(cs, ns.Name, pod,
"containers[0]",
"policy/team/app:1",
"pullPolicy=IfNotPresent",
"allowed: Never",
)
})
It("stores matching tenant rules as independent status rule blocks", func() {
ns := NewNamespace("", map[string]string{
meta.TenantLabel: tnt.GetName(),
@@ -332,11 +472,38 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
},
{
action: rules.ActionTypeDeny,
expressions: []string{"harbor/customer/.*"},
targets: targetContainers,
expressions: []string{"harbor/customer/containers/.*"},
},
{
action: rules.ActionTypeDeny,
targets: targetInitContainers,
expressions: []string{"harbor/customer/init/.*"},
},
{
action: rules.ActionTypeDeny,
targets: targetEphemeralContainers,
expressions: []string{"harbor/customer/debug/.*"},
},
{
action: rules.ActionTypeDeny,
targets: targetVolumes,
expressions: []string{"harbor/customer/volume/.*"},
},
{
action: rules.ActionTypeAudit,
expressions: []string{"audit/.*"},
targets: targetContainers,
expressions: []string{"audit/containers/.*"},
},
{
action: rules.ActionTypeAudit,
targets: targetVolumes,
expressions: []string{"audit/volumes/.*"},
},
{
action: rules.ActionTypeAllow,
targets: targetContainers,
expressions: []string{"policy/.*"},
},
})
})
@@ -357,15 +524,43 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
},
{
action: rules.ActionTypeDeny,
expressions: []string{"harbor/customer/.*"},
targets: targetContainers,
expressions: []string{"harbor/customer/containers/.*"},
},
{
action: rules.ActionTypeAllow,
expressions: []string{"harbor/customer/prod-image/.*"},
action: rules.ActionTypeDeny,
targets: targetInitContainers,
expressions: []string{"harbor/customer/init/.*"},
},
{
action: rules.ActionTypeDeny,
targets: targetEphemeralContainers,
expressions: []string{"harbor/customer/debug/.*"},
},
{
action: rules.ActionTypeDeny,
targets: targetVolumes,
expressions: []string{"harbor/customer/volume/.*"},
},
{
action: rules.ActionTypeAudit,
expressions: []string{"audit/.*"},
targets: targetContainers,
expressions: []string{"audit/containers/.*"},
},
{
action: rules.ActionTypeAudit,
targets: targetVolumes,
expressions: []string{"audit/volumes/.*"},
},
{
action: rules.ActionTypeAllow,
targets: targetContainers,
expressions: []string{"policy/.*"},
},
{
action: rules.ActionTypeAllow,
targets: targetContainers,
expressions: []string{"harbor/customer/containers/prod/.*"},
},
})
})
@@ -386,14 +581,42 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
},
{
action: rules.ActionTypeDeny,
expressions: []string{"harbor/customer/.*"},
},
{
action: rules.ActionTypeAudit,
expressions: []string{"audit/.*"},
targets: targetContainers,
expressions: []string{"harbor/customer/containers/.*"},
},
{
action: rules.ActionTypeDeny,
targets: targetInitContainers,
expressions: []string{"harbor/customer/init/.*"},
},
{
action: rules.ActionTypeDeny,
targets: targetEphemeralContainers,
expressions: []string{"harbor/customer/debug/.*"},
},
{
action: rules.ActionTypeDeny,
targets: targetVolumes,
expressions: []string{"harbor/customer/volume/.*"},
},
{
action: rules.ActionTypeAudit,
targets: targetContainers,
expressions: []string{"audit/containers/.*"},
},
{
action: rules.ActionTypeAudit,
targets: targetVolumes,
expressions: []string{"audit/volumes/.*"},
},
{
action: rules.ActionTypeAllow,
targets: targetContainers,
expressions: []string{"policy/.*"},
},
{
action: rules.ActionTypeDeny,
targets: targetContainers,
expressions: []string{"trusted/.*"},
negated: []bool{true},
},
@@ -425,13 +648,13 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
pod := restrictedPod("customer-denied", "harbor/customer/app:1", corev1.PullIfNotPresent)
pod := restrictedPod("customer-denied", "harbor/customer/containers/app:1", corev1.PullIfNotPresent)
createPodAndExpectDenied(cs, ns.Name, pod,
"containers[0]",
"harbor/customer/app:1",
"harbor/customer/containers/app:1",
"denied",
"harbor/customer/.*",
"harbor/customer/containers/.*",
)
})
@@ -450,12 +673,12 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
createPodAndExpectAllowed(cs, ns.Name, pod)
updatePodAndExpectDenied(cs, ns.Name, pod.Name, func(pod *corev1.Pod) {
pod.Spec.Containers[0].Image = "harbor/customer/adad:1"
pod.Spec.Containers[0].Image = "harbor/customer/containers/app:1"
},
"containers[0]",
"harbor/customer/adad:1",
"harbor/customer/containers/app:1",
"denied",
"harbor/customer/.*",
"harbor/customer/containers/.*",
)
})
@@ -470,15 +693,15 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
denied := restrictedPod("prod-customer-denied", "harbor/customer/other-image/app:1", corev1.PullIfNotPresent)
denied := restrictedPod("prod-customer-denied", "harbor/customer/containers/other/app:1", corev1.PullIfNotPresent)
createPodAndExpectDenied(cs, ns.Name, denied,
"containers[0]",
"harbor/customer/other-image/app:1",
"harbor/customer/containers/other/app:1",
"denied",
"harbor/customer/.*",
"harbor/customer/containers/.*",
)
allowed := restrictedPod("prod-customer-allowed", "harbor/customer/prod-image/app:1", corev1.PullIfNotPresent)
allowed := restrictedPod("prod-customer-allowed", "harbor/customer/containers/prod/app:1", corev1.PullIfNotPresent)
createPodAndExpectAllowed(cs, ns.Name, allowed)
})
@@ -492,13 +715,13 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
pod := restrictedPod("audit-allowed", "audit/team/app:1", corev1.PullIfNotPresent)
pod := restrictedPod("audit-allowed", "audit/containers/team/app:1", corev1.PullIfNotPresent)
createPodAndExpectAllowed(cs, ns.Name, pod)
expectAuditEvent(cs, ns.Name, pod.Name,
"matched audit registry rule",
"audit/.*",
"audit/containers/.*",
)
})
@@ -521,7 +744,7 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
InitContainers: []corev1.Container{
{
Name: "init",
Image: "harbor/customer/init:1",
Image: "harbor/customer/init/app:1",
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: restrictedContainerSecurityContext(),
},
@@ -539,9 +762,9 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
createPodAndExpectDenied(cs, ns.Name, pod,
"initContainers[0]",
"harbor/customer/init:1",
"harbor/customer/init/app:1",
"denied",
"harbor/customer/.*",
"harbor/customer/init/.*",
)
})
@@ -574,7 +797,7 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
Name: "imgvol",
VolumeSource: corev1.VolumeSource{
Image: &corev1.ImageVolumeSource{
Reference: "harbor/customer/volume:1",
Reference: "harbor/customer/volume/app:1",
PullPolicy: corev1.PullIfNotPresent,
},
},
@@ -585,9 +808,55 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
createPodAndExpectDenied(cs, ns.Name, pod,
"volumes[0](imgvol)",
"harbor/customer/volume:1",
"harbor/customer/volume/app:1",
"denied",
"harbor/customer/.*",
"harbor/customer/volume/.*",
)
})
It("audits image volumes independently from container decisions", Label("skip-on-openshift"), func() {
ns := NewNamespace("", map[string]string{
meta.TenantLabel: tnt.GetName(),
})
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "volume-audit-allowed",
},
Spec: corev1.PodSpec{
SecurityContext: nobodyPodSecurityContext(),
Containers: []corev1.Container{
{
Name: "c",
Image: "harbor/platform/app:1",
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: restrictedContainerSecurityContext(),
},
},
Volumes: []corev1.Volume{
{
Name: "imgvol",
VolumeSource: corev1.VolumeSource{
Image: &corev1.ImageVolumeSource{
Reference: "audit/volumes/team/app:1",
PullPolicy: corev1.PullIfNotPresent,
},
},
},
},
},
}
createPodAndExpectAllowed(cs, ns.Name, pod)
expectAuditEvent(cs, ns.Name, pod.Name,
"matched audit registry rule",
"audit/volumes/.*",
)
})
@@ -610,7 +879,7 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
ephemeral := corev1.EphemeralContainer{
EphemeralContainerCommon: corev1.EphemeralContainerCommon{
Name: "debug",
Image: "harbor/customer/debug:1",
Image: "harbor/customer/debug/app:1",
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: restrictedContainerSecurityContext(),
},
@@ -622,7 +891,7 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
return err
}
current.Spec.EphemeralContainers = append(current.Spec.EphemeralContainers, ephemeral)
current.Spec.EphemeralContainers = []corev1.EphemeralContainer{ephemeral}
_, err = cs.CoreV1().Pods(ns.Name).UpdateEphemeralContainers(
context.Background(),
@@ -631,15 +900,15 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
metav1.UpdateOptions{},
)
if err == nil {
return fmt.Errorf("expected UpdateEphemeralContainers to be denied, but it succeeded")
return fmt.Errorf("expected ephemeral container update to be denied, but it succeeded")
}
msg := err.Error()
for _, substring := range []string{
"ephemeralContainers[0]",
"harbor/customer/debug:1",
"harbor/customer/debug/app:1",
"denied",
"harbor/customer/.*",
"harbor/customer/debug/.*",
} {
if !strings.Contains(msg, substring) {
return fmt.Errorf("expected error to contain %q, got: %s", substring, msg)
@@ -649,4 +918,42 @@ var _ = Describe("enforcing container registry namespace rules", Ordered, Label(
return nil
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
It("allows an allowed registry reference when its pull policy is permitted", func() {
ns := NewNamespace("", map[string]string{
meta.TenantLabel: tnt.GetName(),
})
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
pod := restrictedPod("policy-allowed", "policy/team/app:1", corev1.PullNever)
createPodAndExpectAllowed(cs, ns.Name, pod)
})
It("applies namespace-selector matched negated regex rules after the base rules", func() {
ns := NewNamespace("", map[string]string{
"negate": "true",
meta.TenantLabel: tnt.GetName(),
})
cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())
denied := restrictedPod("negated-denied", "harbor/platform/app:1", corev1.PullIfNotPresent)
createPodAndExpectDenied(cs, ns.Name, denied,
"containers[0]",
"harbor/platform/app:1",
"denied",
"trusted/.*",
)
allowed := restrictedPod("negated-allowed", "trusted/platform/app:1", corev1.PullIfNotPresent)
createPodAndExpectAllowed(cs, ns.Name, allowed)
})
})
+59
View File
@@ -1143,3 +1143,62 @@ func EnsureRuntimeClass(ctx context.Context, rtc *nodev1.RuntimeClass) {
return err
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
const namespaceTerminationHoldFinalizer = "e2e.projectcapsule.dev/hold-termination"
func holdNamespaceTerminating(ctx context.Context, name string) func() {
Eventually(func() error {
ns := &corev1.Namespace{}
if err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, ns); err != nil {
return err
}
if controllerutil.AddFinalizer(ns, namespaceTerminationHoldFinalizer) {
return k8sClient.Update(ctx, ns)
}
return nil
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
Eventually(func() error {
ns := &corev1.Namespace{}
if err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, ns); err != nil {
return err
}
if ns.DeletionTimestamp != nil {
return nil
}
return k8sClient.Delete(ctx, ns)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
Eventually(func(g Gomega) {
ns := &corev1.Namespace{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, ns)).To(Succeed())
g.Expect(ns.DeletionTimestamp).ToNot(BeNil())
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
return func() {
Eventually(func() error {
ns := &corev1.Namespace{}
if err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, ns); err != nil {
if apierrors.IsNotFound(err) {
return nil
}
return err
}
if controllerutil.RemoveFinalizer(ns, namespaceTerminationHoldFinalizer) {
return k8sClient.Update(ctx, ns)
}
return nil
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
}
+27 -15
View File
@@ -11,7 +11,7 @@ spec:
kind: User
rules:
- permissions:
rules:
promotions:
- clusterRoles:
- "configmap-replicator"
- namespaceSelector:
@@ -21,7 +21,7 @@ spec:
values:
- "test"
permissions:
rules:
promotions:
- clusterRoles:
- "secret-replicator"
- namespaceSelector:
@@ -31,7 +31,7 @@ spec:
values:
- "prod"
permissions:
rules:
promotions:
- clusterRoles:
- "sade-boi"
permissions:
@@ -89,16 +89,29 @@ spec:
kind: User
name: alice
rules:
- enforce:
action: "deny"
registries:
- url: "harbor/.*"
- permissions:
promotions:
- clusterRoles:
- "admin"
selector:
matchLabels:
owner.projectcapsule.dev/role: "replication"
- enforce:
action: "allow"
registries:
- url: "harbor/customer/.*"
policy:
- "Never"
workloads:
registries:
- exp: ".*"
- enforce:
action: "deny"
workloads:
registries:
- exp: "harbor/.*"
- enforce:
action: "deny"
workloads:
registries:
- exp: "harbor/customer/.*"
- namespaceSelector:
matchExpressions:
- key: env
@@ -107,10 +120,9 @@ spec:
- "prod"
enforce:
action: "allow"
registries:
- url: "harbor/v2/customer-registry/prod-image/.*"
policy:
- "Always"
workloads:
registries:
- exp: "harbor/customer/prod-image/.*"
---
apiVersion: capsule.clastix.io/v1beta2
kind: Tenant
+12
View File
@@ -95,6 +95,10 @@ func (c *RegexCache) GetOrCompile(expr api.RegExpression) (*CompiledRegex, bool,
}
func (c *RegexCache) Has(id string) bool {
if c == nil {
return false
}
c.mu.RLock()
defer c.mu.RUnlock()
@@ -104,6 +108,10 @@ func (c *RegexCache) Has(id string) bool {
}
func (c *RegexCache) Stats() int {
if c == nil {
return 0
}
c.mu.RLock()
defer c.mu.RUnlock()
@@ -111,6 +119,10 @@ func (c *RegexCache) Stats() int {
}
func (c *RegexCache) Reset() {
if c == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
+65
View File
@@ -9,6 +9,71 @@ import (
"github.com/projectcapsule/capsule/pkg/api"
)
func TestCompiledRegexMatchString(t *testing.T) {
t.Parallel()
tests := []struct {
name string
expression api.RegExpression
value string
want bool
}{
{
name: "normal expression matches matching value",
expression: api.RegExpression{
Expression: "trusted/.*",
},
value: "trusted/team/app:1",
want: true,
},
{
name: "normal expression does not match non matching value",
expression: api.RegExpression{
Expression: "trusted/.*",
},
value: "docker.io/team/app:1",
want: false,
},
{
name: "negated expression does not match matching value",
expression: api.RegExpression{
Expression: "trusted/.*",
Negate: true,
},
value: "trusted/team/app:1",
want: false,
},
{
name: "negated expression matches non matching value",
expression: api.RegExpression{
Expression: "trusted/.*",
Negate: true,
},
value: "docker.io/team/app:1",
want: true,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
cache := NewRegexCache()
compiled, _, err := cache.GetOrCompile(tt.expression)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if got := compiled.MatchString(tt.value); got != tt.want {
t.Fatalf("MatchString() = %v, want %v", got, tt.want)
}
})
}
}
func TestRegexCache_GetOrCompile(t *testing.T) {
t.Parallel()
+7 -76
View File
@@ -18,19 +18,15 @@ import (
)
type RuleSet struct {
ID string
Compiled []CompiledRule
HasImages bool
HasVolumes bool
ID string
Compiled []CompiledRule
}
type CompiledRule struct {
Expression api.RegExpression
RegexID string
AllowedPolicy map[corev1.PullPolicy]struct{} // nil/empty => allow any
ValidateImages bool
ValidateVolumes bool
AllowedPolicy map[corev1.PullPolicy]struct{} // nil/empty => allow any
}
func (r *CompiledRule) AllowsPullPolicy(pullPolicy corev1.PullPolicy) bool {
@@ -43,17 +39,6 @@ func (r *CompiledRule) AllowsPullPolicy(pullPolicy corev1.PullPolicy) bool {
return ok
}
func (r *CompiledRule) MatchesTarget(target rules.RegistryValidationTarget) bool {
switch target {
case rules.ValidateImages:
return r.ValidateImages
case rules.ValidateVolumes:
return r.ValidateVolumes
default:
return false
}
}
type RegistryRuleSetCache struct {
regexCache *RegexCache
@@ -91,7 +76,6 @@ func (c *RegistryRuleSetCache) GetOrBuild(specRules []rules.OCIRegistry) (rs *Ru
return rs, true, nil
}
// Build outside locks. Regex compilation is delegated to RegexCache.
built, err := c.buildRuleSet(id, specRules)
if err != nil {
return nil, false, err
@@ -113,15 +97,12 @@ func (c *RegistryRuleSetCache) GetOrBuild(specRules []rules.OCIRegistry) (rs *Ru
return built, false, nil
}
// Match matches a reference against target, regex and pullPolicy.
// Admission deny/allow/audit evaluation should usually use MatchReference instead,
// because it needs to distinguish "regex matched but pullPolicy is forbidden" from
// "regex did not match".
// Match matches reference, regex and pullPolicy.
// Admission evaluation should usually use MatchReference instead.
func (c *RegistryRuleSetCache) Match(
specRules []rules.OCIRegistry,
reference string,
pullPolicy corev1.PullPolicy,
target rules.RegistryValidationTarget,
) (*CompiledRule, error) {
rs, _, err := c.GetOrBuild(specRules)
if err != nil {
@@ -132,15 +113,13 @@ func (c *RegistryRuleSetCache) Match(
return nil, nil
}
return c.MatchRuleSet(rs, reference, pullPolicy, target)
return c.MatchRuleSet(rs, reference, pullPolicy)
}
// MatchRuleSet matches a reference against target, regex and pullPolicy.
func (c *RegistryRuleSetCache) MatchRuleSet(
rs *RuleSet,
reference string,
pullPolicy corev1.PullPolicy,
target rules.RegistryValidationTarget,
) (*CompiledRule, error) {
if c == nil {
return nil, fmt.Errorf("registry rule set cache is nil")
@@ -157,10 +136,6 @@ func (c *RegistryRuleSetCache) MatchRuleSet(
for i := range rs.Compiled {
rule := &rs.Compiled[i]
if !rule.MatchesTarget(target) {
continue
}
if !rule.AllowsPullPolicy(pullPolicy) {
continue
}
@@ -178,12 +153,11 @@ func (c *RegistryRuleSetCache) MatchRuleSet(
return nil, nil
}
// MatchReference matches a reference against target and regex only.
// MatchReference matches reference and regex only.
// It intentionally does not check pullPolicy.
func (c *RegistryRuleSetCache) MatchReference(
rs *RuleSet,
reference string,
target rules.RegistryValidationTarget,
) (*CompiledRule, error) {
if c == nil {
return nil, fmt.Errorf("registry rule set cache is nil")
@@ -200,10 +174,6 @@ func (c *RegistryRuleSetCache) MatchReference(
for i := range rs.Compiled {
rule := &rs.Compiled[i]
if !rule.MatchesTarget(target) {
continue
}
compiled, _, err := c.regexCache.GetOrCompile(rule.Expression)
if err != nil {
return nil, err
@@ -228,7 +198,6 @@ func (c *RegistryRuleSetCache) Stats() int {
return len(c.rs)
}
// activeIDs: set of ids currently referenced by RuleStatus in cluster.
func (c *RegistryRuleSetCache) PruneActive(activeIDs map[string]struct{}) int {
if c == nil {
return 0
@@ -273,13 +242,6 @@ func (c *RegistryRuleSetCache) HashRules(specRules []rules.OCIRegistry) string {
sort.Strings(policies)
validations := make([]string, 0, len(r.Validation))
for _, v := range r.Validation {
validations = append(validations, strings.TrimSpace(string(v)))
}
sort.Strings(validations)
b.WriteString(strings.TrimSpace(expr.Expression))
b.WriteString(sepField)
@@ -299,16 +261,6 @@ func (c *RegistryRuleSetCache) HashRules(specRules []rules.OCIRegistry) string {
b.WriteString(p)
}
b.WriteString(sepField)
for i, v := range validations {
if i > 0 {
b.WriteString(sepList)
}
b.WriteString(v)
}
b.WriteString(sepRule)
}
@@ -317,7 +269,6 @@ func (c *RegistryRuleSetCache) HashRules(specRules []rules.OCIRegistry) string {
return hex.EncodeToString(sum[:])
}
// Has is useful in tests and debugging.
func (c *RegistryRuleSetCache) Has(id string) bool {
if c == nil {
return false
@@ -342,8 +293,6 @@ func (c *RegistryRuleSetCache) Reset() {
c.rs = make(map[string]*RuleSet)
}
// InsertForTest can be behind a build tag if you prefer, but it is fine to keep simple.
//
//nolint:unused
func (c *RegistryRuleSetCache) insertForTest(id string) {
c.mu.Lock()
@@ -387,24 +336,6 @@ func (c *RegistryRuleSetCache) buildRuleSet(id string, specRules []rules.OCIRegi
}
}
if len(r.Validation) == 0 {
cr.ValidateImages = true
cr.ValidateVolumes = true
rs.HasImages = true
rs.HasVolumes = true
} else {
for _, v := range r.Validation {
switch v {
case rules.ValidateImages:
cr.ValidateImages = true
rs.HasImages = true
case rules.ValidateVolumes:
cr.ValidateVolumes = true
rs.HasVolumes = true
}
}
}
rs.Compiled = append(rs.Compiled, cr)
}
+781 -479
View File
File diff suppressed because it is too large Load Diff
@@ -68,7 +68,11 @@ func collectRegexExpressionsFromNamespaceRule(
return
}
for _, registry := range rule.Enforce.Registries {
if rule.Enforce == nil {
return
}
for _, registry := range rule.Enforce.Workloads.Registries {
expr := registry.RegExpression
if expr.Expression == "" {
continue
@@ -36,11 +36,19 @@ func (r *CacheInvalidator) rebuildRuleStatusRegistryCache(ctx context.Context, l
item := &rsList.Items[i]
for _, rule := range item.Status.Rules {
if rule == nil || len(rule.Enforce.Registries) == 0 {
if rule == nil {
continue
}
if _, _, err := r.RegistryCache.GetOrBuild(rule.Enforce.Registries); err != nil {
if rule.Enforce == nil {
continue
}
if len(rule.Enforce.Workloads.Registries) == 0 {
continue
}
if _, _, err := r.RegistryCache.GetOrBuild(rule.Enforce.Workloads.Registries); err != nil {
return fmt.Errorf(
"build registry cache for RuleStatus %s/%s: %w",
item.Namespace,
+2
View File
@@ -130,6 +130,8 @@ func (r *globalResourceController) Reconcile(ctx context.Context, request reconc
//nolint:dupl
defer func() {
meta.RemoveReconcileTriggerAnnotation(tntResource)
reconcileErr := err
if statusErr != nil {
reconcileErr = statusErr
@@ -167,6 +167,8 @@ func (r *namespacedResourceController) Reconcile(ctx context.Context, request re
//nolint:dupl
defer func() {
meta.RemoveReconcileTriggerAnnotation(tntResource)
reconcileErr := err
if statusErr != nil {
reconcileErr = statusErr
+22 -11
View File
@@ -148,20 +148,31 @@ func (r Manager) reconcile(ctx context.Context, instance *capsulev1beta2.RuleSta
continue
}
normalized := *rule
normalized.Enforce = rule.Enforce
normalized.Enforce.Registries = append(
[]rules.OCIRegistry(nil),
rule.Enforce.Registries...,
)
// Keep status compact: skip empty enforce blocks.
if len(normalized.Enforce.Registries) == 0 {
if rule.Enforce == nil {
continue
}
ruleStatus = append(ruleStatus, &normalized)
normalized := &rules.NamespaceRuleBodyNamespace{
Enforce: &rules.NamespaceRuleEnforceBody{
Action: rule.Enforce.Action,
Workloads: rules.NamespaceRuleEnforceWorkloadsBody{
Targets: append(
[]rules.WorkloadValidationTarget(nil),
rule.Enforce.Workloads.Targets...,
),
Registries: append(
[]rules.OCIRegistry(nil),
rule.Enforce.Workloads.Registries...,
),
QoSClasses: append(
[]corev1.PodQOSClass(nil),
rule.Enforce.Workloads.QoSClasses...,
),
},
},
}
ruleStatus = append(ruleStatus, normalized)
}
instance.Status.Rules = ruleStatus
+285
View File
@@ -0,0 +1,285 @@
// Copyright 2020-2026 Project Capsule Authors
// SPDX-License-Identifier: Apache-2.0
package pod
import (
"context"
"fmt"
"net/http"
"slices"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/tools/events"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
"github.com/projectcapsule/capsule/pkg/api/rules"
ad "github.com/projectcapsule/capsule/pkg/runtime/admission"
"github.com/projectcapsule/capsule/pkg/runtime/configuration"
evt "github.com/projectcapsule/capsule/pkg/runtime/events"
"github.com/projectcapsule/capsule/pkg/runtime/handlers"
"github.com/projectcapsule/capsule/pkg/runtime/workloads"
)
type qosHandler struct {
configuration configuration.Configuration
}
func QoSClass(configuration configuration.Configuration) handlers.TypedHandlerWithTenantWithRuleset[*corev1.Pod] {
return &qosHandler{
configuration: configuration,
}
}
func (h *qosHandler) OnCreate(
_ client.Client,
_ client.Reader,
pod *corev1.Pod,
_ admission.Decoder,
recorder events.EventRecorder,
tnt *capsulev1beta2.Tenant,
ruleBlocks []*rules.NamespaceRuleBodyNamespace,
) handlers.Func {
return func(ctx context.Context, req admission.Request) *admission.Response {
return h.validate(req, pod, tnt, recorder, ruleBlocks)
}
}
func (h *qosHandler) OnUpdate(
_ client.Client,
_ client.Reader,
_ *corev1.Pod,
pod *corev1.Pod,
_ admission.Decoder,
recorder events.EventRecorder,
tnt *capsulev1beta2.Tenant,
ruleBlocks []*rules.NamespaceRuleBodyNamespace,
) handlers.Func {
return func(ctx context.Context, req admission.Request) *admission.Response {
return h.validate(req, pod, tnt, recorder, ruleBlocks)
}
}
func (h *qosHandler) OnDelete(
client.Client,
client.Reader,
*corev1.Pod,
admission.Decoder,
events.EventRecorder,
*capsulev1beta2.Tenant,
[]*rules.NamespaceRuleBodyNamespace,
) handlers.Func {
return func(context.Context, admission.Request) *admission.Response {
return nil
}
}
func (h *qosHandler) validate(
req admission.Request,
pod *corev1.Pod,
tnt *capsulev1beta2.Tenant,
recorder events.EventRecorder,
ruleBlocks []*rules.NamespaceRuleBodyNamespace,
) *admission.Response {
if pod == nil {
resp := admission.Errored(http.StatusInternalServerError, fmt.Errorf("pod is nil"))
return &resp
}
if len(ruleBlocks) == 0 {
return nil
}
qosClass := workloads.GetPodQoSClass(pod)
evaluation, err := evaluateQoSClass(ruleBlocks, qosClass)
if err != nil {
resp := admission.Errored(http.StatusInternalServerError, err)
return &resp
}
if evaluation == nil {
return nil
}
warnings := make([]string, 0, len(evaluation.Audits))
for _, audit := range evaluation.Audits {
msg := fmt.Sprintf(
"pod %q uses QoS class %q and matched audit QoS rule",
pod.Name,
qosClass,
)
h.auditWithEvent(recorder, tnt, pod, msg)
warnings = append(warnings, msg)
_ = audit
}
if evaluation.Decision == nil {
if len(warnings) > 0 {
resp := admission.Allowed("QoS class audited")
resp.Warnings = append(resp.Warnings, warnings...)
return &resp
}
return nil
}
switch evaluation.Decision.Action {
case rules.ActionTypeAllow:
if len(warnings) > 0 {
resp := admission.Allowed("QoS class allowed with warnings")
resp.Warnings = append(resp.Warnings, warnings...)
return &resp
}
return nil
case rules.ActionTypeDeny:
msg := fmt.Sprintf(
"pod %q uses QoS class %q which is denied by namespace rule",
pod.Name,
qosClass,
)
return h.denyWithEvent(
recorder,
tnt,
pod,
evt.ReasonForbiddenPodQoSClass,
msg,
)
case rules.ActionTypeAudit:
msg := fmt.Sprintf(
"pod %q uses QoS class %q and matched audit QoS rule",
pod.Name,
qosClass,
)
h.auditWithEvent(recorder, tnt, pod, msg)
resp := admission.Allowed("QoS class audited")
resp.Warnings = append(resp.Warnings, append(warnings, msg)...)
return &resp
default:
resp := admission.Errored(
http.StatusInternalServerError,
fmt.Errorf("unsupported namespace rule action %q", evaluation.Decision.Action),
)
return &resp
}
}
type qosDecision struct {
Action rules.ActionType
Rule *rules.NamespaceRuleBodyNamespace
Class corev1.PodQOSClass
}
type qosEvaluation struct {
Decision *qosDecision
Audits []*qosDecision
}
func evaluateQoSClass(
ruleBlocks []*rules.NamespaceRuleBodyNamespace,
qosClass corev1.PodQOSClass,
) (*qosEvaluation, error) {
evaluation := &qosEvaluation{}
for _, rule := range ruleBlocks {
if rule == nil || rule.Enforce == nil {
continue
}
if len(rule.Enforce.Workloads.QoSClasses) == 0 {
continue
}
if !rule.Enforce.WorkloadTargetsAny(
rules.ValidateInitContainers,
rules.ValidateEphemeralContainers,
rules.ValidateContainers,
rules.ValidateVolumes,
) {
continue
}
if !qosClassMatches(rule.Enforce.Workloads.QoSClasses, qosClass) {
continue
}
action := rule.Enforce.Action.OrDefault()
decision := &qosDecision{
Action: action,
Rule: rule,
Class: qosClass,
}
switch action {
case rules.ActionTypeAllow, rules.ActionTypeDeny:
// Last matching allow/deny wins.
evaluation.Decision = decision
case rules.ActionTypeAudit:
evaluation.Audits = append(evaluation.Audits, decision)
default:
return nil, fmt.Errorf("unsupported namespace rule action %q", action)
}
}
return evaluation, nil
}
func qosClassMatches(classes []corev1.PodQOSClass, got corev1.PodQOSClass) bool {
return slices.Contains(classes, got)
}
func (h *qosHandler) auditWithEvent(
recorder events.EventRecorder,
tnt *capsulev1beta2.Tenant,
pod *corev1.Pod,
msg string,
) {
recorder.Eventf(
pod,
tnt,
corev1.EventTypeWarning,
evt.ReasonForbiddenPodQoSClass,
evt.ActionValidationDenied,
msg,
)
}
func (h *qosHandler) denyWithEvent(
recorder events.EventRecorder,
tnt *capsulev1beta2.Tenant,
pod *corev1.Pod,
reason string,
msg string,
) *admission.Response {
recorder.Eventf(
pod,
tnt,
corev1.EventTypeWarning,
reason,
evt.ActionValidationDenied,
msg,
)
return ad.Deny(msg)
}
+73 -56
View File
@@ -30,7 +30,10 @@ type registryHandler struct {
cache *cache.RegistryRuleSetCache
}
func ContainerRegistry(configuration configuration.Configuration, cache *cache.RegistryRuleSetCache) handlers.TypedHandlerWithTenantWithRuleset[*corev1.Pod] {
func ContainerRegistry(
configuration configuration.Configuration,
cache *cache.RegistryRuleSetCache,
) handlers.TypedHandlerWithTenantWithRuleset[*corev1.Pod] {
return &registryHandler{
configuration: configuration,
cache: cache,
@@ -89,7 +92,19 @@ func (h *registryHandler) validate(
ruleBlocks []*rules.NamespaceRuleBodyNamespace,
) *admission.Response {
if h.cache == nil {
resp := admission.Errored(http.StatusInternalServerError, fmt.Errorf("registry rule set cache is nil"))
resp := admission.Errored(
http.StatusInternalServerError,
fmt.Errorf("registry rule set cache is nil"),
)
return &resp
}
if pod == nil {
resp := admission.Errored(
http.StatusInternalServerError,
fmt.Errorf("pod is nil"),
)
return &resp
}
@@ -107,11 +122,23 @@ func (h *registryHandler) validate(
warnings := make([]string, 0)
if resp := h.validateContainers(req, pod, tnt, recorder, ruleBlocks, &warnings); resp != nil {
if resp := h.validateContainers(
recorder,
tnt,
pod,
ruleBlocks,
&warnings,
); resp != nil {
return resp
}
if resp := h.validateVolumes(req, pod, tnt, recorder, ruleBlocks, &warnings); resp != nil {
if resp := h.validateVolumes(
recorder,
tnt,
pod,
ruleBlocks,
&warnings,
); resp != nil {
return resp
}
@@ -126,10 +153,9 @@ func (h *registryHandler) validate(
}
func (h *registryHandler) validateContainers(
req admission.Request,
pod *corev1.Pod,
tnt *capsulev1beta2.Tenant,
recorder events.EventRecorder,
tnt *capsulev1beta2.Tenant,
pod *corev1.Pod,
ruleBlocks []*rules.NamespaceRuleBodyNamespace,
warnings *[]string,
) *admission.Response {
@@ -138,11 +164,10 @@ func (h *registryHandler) validateContainers(
if resp := h.verifyOCIReference(
recorder,
req,
tnt,
pod,
ruleBlocks,
rules.ValidateImages,
rules.ValidateInitContainers,
c.Image,
c.ImagePullPolicy,
fmt.Sprintf("initContainers[%d]", i),
@@ -152,35 +177,15 @@ func (h *registryHandler) validateContainers(
}
}
for i := range pod.Spec.EphemeralContainers {
c := pod.Spec.EphemeralContainers[i]
if resp := h.verifyOCIReference(
recorder,
req,
tnt,
pod,
ruleBlocks,
rules.ValidateImages,
c.Image,
c.ImagePullPolicy,
fmt.Sprintf("ephemeralContainers[%d]", i),
warnings,
); resp != nil {
return resp
}
}
for i := range pod.Spec.Containers {
c := pod.Spec.Containers[i]
if resp := h.verifyOCIReference(
recorder,
req,
tnt,
pod,
ruleBlocks,
rules.ValidateImages,
rules.ValidateContainers,
c.Image,
c.ImagePullPolicy,
fmt.Sprintf("containers[%d]", i),
@@ -190,14 +195,31 @@ func (h *registryHandler) validateContainers(
}
}
for i := range pod.Spec.EphemeralContainers {
c := pod.Spec.EphemeralContainers[i]
if resp := h.verifyOCIReference(
recorder,
tnt,
pod,
ruleBlocks,
rules.ValidateEphemeralContainers,
c.Image,
c.ImagePullPolicy,
fmt.Sprintf("ephemeralContainers[%d]", i),
warnings,
); resp != nil {
return resp
}
}
return nil
}
func (h *registryHandler) validateVolumes(
req admission.Request,
pod *corev1.Pod,
tnt *capsulev1beta2.Tenant,
recorder events.EventRecorder,
tnt *capsulev1beta2.Tenant,
pod *corev1.Pod,
ruleBlocks []*rules.NamespaceRuleBodyNamespace,
warnings *[]string,
) *admission.Response {
@@ -207,25 +229,13 @@ func (h *registryHandler) validateVolumes(
continue
}
ref := strings.TrimSpace(v.Image.Reference)
if ref == "" {
return h.denyWithEvent(
recorder,
tnt,
pod,
evt.ReasonForbiddenContainerRegistry,
fmt.Sprintf("volume %q has empty image.reference", v.Name),
)
}
if resp := h.verifyOCIReference(
recorder,
req,
tnt,
pod,
ruleBlocks,
rules.ValidateVolumes,
ref,
v.Image.Reference,
v.Image.PullPolicy,
fmt.Sprintf("volumes[%d](%s)", i, v.Name),
warnings,
@@ -239,11 +249,10 @@ func (h *registryHandler) validateVolumes(
func (h *registryHandler) verifyOCIReference(
recorder events.EventRecorder,
req admission.Request,
tnt *capsulev1beta2.Tenant,
pod *corev1.Pod,
ruleBlocks []*rules.NamespaceRuleBodyNamespace,
target rules.RegistryValidationTarget,
target rules.WorkloadValidationTarget,
reference string,
pullPolicy corev1.PullPolicy,
where string,
@@ -361,17 +370,25 @@ type registryEvaluation struct {
func (h *registryHandler) evaluateOCIReference(
ruleBlocks []*rules.NamespaceRuleBodyNamespace,
target rules.RegistryValidationTarget,
target rules.WorkloadValidationTarget,
ref string,
) (*registryEvaluation, error) {
evaluation := &registryEvaluation{}
for _, rule := range ruleBlocks {
if rule == nil || len(rule.Enforce.Registries) == 0 {
if rule == nil || rule.Enforce == nil {
continue
}
rs, _, err := h.cache.GetOrBuild(rule.Enforce.Registries)
if len(rule.Enforce.Workloads.Registries) == 0 {
continue
}
if !rule.Enforce.WorkloadTargetsAny(target) {
continue
}
rs, _, err := h.cache.GetOrBuild(rule.Enforce.Workloads.Registries)
if err != nil {
return nil, err
}
@@ -380,7 +397,10 @@ func (h *registryHandler) evaluateOCIReference(
continue
}
matched, err := h.cache.MatchReference(rs, ref, target)
// Important:
// Match by image reference only. Pull policy is validated after the
// final allow decision has been selected.
matched, err := h.cache.MatchReference(rs, ref)
if err != nil {
return nil, err
}
@@ -389,10 +409,7 @@ func (h *registryHandler) evaluateOCIReference(
continue
}
action := rule.Enforce.Action
if action == "" {
action = rules.ActionTypeDeny
}
action := rule.Enforce.Action.OrDefault()
decision := &registryDecision{
RuleDecision: rules.RuleDecision{
@@ -7,6 +7,7 @@ import (
"context"
"fmt"
"regexp"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/events"
@@ -70,6 +71,10 @@ func (h *RuleValidationHandler) OnUpdate(
}
func ValidateRule(tnt *capsulev1beta2.Tenant, req admission.Request) *admission.Response {
if tnt == nil {
return nil
}
if len(tnt.Spec.Rules) == 0 {
return nil
}
@@ -79,6 +84,15 @@ func ValidateRule(tnt *capsulev1beta2.Tenant, req admission.Request) *admission.
continue
}
body := rule.NamespaceRuleBodyNamespace
if body == nil {
continue
}
if rule.Enforce == nil {
continue
}
if rule.NamespaceSelector != nil {
if _, err := metav1.LabelSelectorAsSelector(rule.NamespaceSelector); err != nil {
return ad.Deny(
@@ -87,19 +101,19 @@ func ValidateRule(tnt *capsulev1beta2.Tenant, req admission.Request) *admission.
}
}
for j, registry := range rule.Enforce.Registries {
for j, registry := range rule.Enforce.Workloads.Registries {
expr := registry.Expression()
if expr.Expression == "" {
if strings.TrimSpace(expr.Expression) == "" {
return ad.Deny(
fmt.Sprintf("rules[%d].enforce.registries[%d].exp must not be empty", i, j),
fmt.Sprintf("rules[%d].enforce.workloads.registries[%d].exp must not be empty", i, j),
)
}
if _, err := regexp.Compile(expr.Expression); err != nil {
return ad.Deny(
fmt.Sprintf(
"rules[%d].enforce.registries[%d].exp %q is invalid: %v",
"rules[%d].enforce.workloads.registries[%d].exp %q is invalid: %v",
i,
j,
expr.Expression,
+19
View File
@@ -5,6 +5,7 @@ package meta
import (
"context"
"maps"
"strings"
"time"
@@ -75,6 +76,24 @@ func TriggerRequestReconcileAnnotation(
})
}
func RemoveReconcileTriggerAnnotation(
obj client.Object,
) {
annotations := obj.GetAnnotations()
if _, ok := annotations[ReconcileAnnotation]; !ok {
return
}
annotations = maps.Clone(annotations)
delete(annotations, ReconcileAnnotation)
if len(annotations) == 0 {
obj.SetAnnotations(nil)
} else {
obj.SetAnnotations(annotations)
}
}
func annotationRemove(obj client.Object, anno string) {
annotations := obj.GetAnnotations()
+105
View File
@@ -48,6 +48,8 @@ func (p *Processor) Reconcile(
return true
}
terminatingNamespaces := map[string]bool{}
for _, i := range *processed {
if _, exists := acc[i.GetKey("")]; exists {
continue
@@ -137,6 +139,31 @@ func (p *Processor) Reconcile(
for _, obj := range *item.Objects {
fieldOwner := opts.FieldOwnerPrefix + "/" + item.Resource.FieldOwner("")
terminating, namespace, err := p.isNamespaceTerminatingForObject(ctx, obj.Object, terminatingNamespaces)
if err != nil {
hadError = true
or.Status = metav1.ConditionFalse
or.Message = "checking namespace termination failed for item " + obj.Origin.Origin + ": " + err.Error()
processed.UpdateItem(or)
continue
}
if terminating {
log.V(4).Info(
"skipping apply because namespace is terminating",
"item", obj.Origin.Origin,
"namespace", namespace,
"Kind", obj.Object.GetKind(),
"Name", obj.Object.GetName(),
)
processed.RemoveItem(or)
continue
}
ver, created, err := p.Apply(
ctx,
c,
@@ -458,3 +485,81 @@ func (r *Processor) handleCreatedMetadata(
existingObject.GetName(),
)
}
func (r *Processor) isNamespaceTerminatingForObject(
ctx context.Context,
obj *unstructured.Unstructured,
cache map[string]bool,
) (terminating bool, namespace string, err error) {
// The Namespace object itself is cluster-scoped, but if Capsule is applying
// a Namespace which is already terminating, we should skip it as well.
if obj.GroupVersionKind().Group == "" && obj.GetKind() == "Namespace" {
namespace = obj.GetName()
ns := &corev1.Namespace{}
if err := r.GatherClient.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil {
if apierrors.IsNotFound(err) {
cache[namespace] = false
return false, namespace, nil
}
return false, namespace, err
}
terminating = ns.DeletionTimestamp != nil || ns.Status.Phase == corev1.NamespaceTerminating
cache[namespace] = terminating
return terminating, namespace, nil
}
mapping, err := r.Mapper.RESTMapping(
obj.GroupVersionKind().GroupKind(),
obj.GroupVersionKind().Version,
)
if err != nil {
return false, "", err
}
if mapping.Scope.Name() != k8smeta.RESTScopeNameNamespace {
return false, "", nil
}
namespace = obj.GetNamespace()
if namespace == "" {
return false, "", nil
}
return r.isNamespaceTerminating(ctx, namespace, cache)
}
func (r *Processor) isNamespaceTerminating(
ctx context.Context,
namespace string,
cache map[string]bool,
) (bool, string, error) {
if namespace == "" {
return false, namespace, nil
}
if terminating, ok := cache[namespace]; ok {
return terminating, namespace, nil
}
ns := &corev1.Namespace{}
if err := r.GatherClient.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil {
if apierrors.IsNotFound(err) {
cache[namespace] = true
return true, namespace, nil
}
return false, namespace, err
}
terminating := ns.DeletionTimestamp != nil || ns.Status.Phase == corev1.NamespaceTerminating
cache[namespace] = terminating
return terminating, namespace, nil
}
+30
View File
@@ -0,0 +1,30 @@
// Copyright 2020-2026 Project Capsule Authors
// SPDX-License-Identifier: Apache-2.0
package rules
import "slices"
func (a ActionType) OrDefault() ActionType {
if a == "" {
return ActionTypeDeny
}
return a
}
func (e NamespaceRuleEnforceBody) GetWorkloadTargets(target WorkloadValidationTarget) bool {
if len(e.Workloads.Targets) == 0 {
return true
}
return slices.Contains(e.Workloads.Targets, target)
}
func (e NamespaceRuleEnforceBody) WorkloadTargetsAny(targets ...WorkloadValidationTarget) bool {
if len(e.Workloads.Targets) == 0 {
return true
}
return slices.ContainsFunc(targets, e.GetWorkloadTargets)
}
+2 -3
View File
@@ -12,7 +12,6 @@ type NamespaceRuleEnforceBody struct {
//+kubebuilder:default:=deny
Action ActionType `json:"action,omitempty"`
// Define registries which are allowed to be used within this tenant
// The rules are aggregated, since you can use Regular Expressions the match registry endpoints
Registries []OCIRegistry `json:"registries,omitempty"`
// Enforcement for Workloads (Pods)
Workloads NamespaceRuleEnforceWorkloadsBody `json:"workloads,omitempty"`
}
@@ -16,40 +16,16 @@ func (i ImagePullPolicySpec) String() string {
return string(i)
}
// +kubebuilder:validation:Enum=pod/images;pod/volumes
type RegistryValidationTarget string
const (
ValidateImages RegistryValidationTarget = "pod/images"
ValidateVolumes RegistryValidationTarget = "pod/volumes"
)
// +kubebuilder:object:generate=true
type OCIRegistry struct {
api.RegExpression `json:",inline"`
// Deprecated: Use exp field
//
// OCI Registry endpoint, is treated as regular expression.
Registry string `json:"url,omitempty"`
// Allowed PullPolicy for the given registry. Supplying no value allows all policies.
// +optional
// +kubebuilder:validation:Items:Enum=Always;Never;IfNotPresent
Policy []corev1.PullPolicy `json:"policy,omitempty"`
// Requesting Resources
//+kubebuilder:default:={pod/images,pod/volumes}
Validation []RegistryValidationTarget `json:"validation,omitempty"`
}
func (r OCIRegistry) Expression() api.RegExpression {
if r.RegExpression.Expression != "" {
return r.RegExpression
}
return api.RegExpression{
Expression: r.Registry,
Negate: false,
}
return r.RegExpression
}
+35
View File
@@ -0,0 +1,35 @@
// Copyright 2020-2026 Project Capsule Authors
// SPDX-License-Identifier: Apache-2.0
package rules
import corev1 "k8s.io/api/core/v1"
// +kubebuilder:validation:Enum=pod/initcontainers;pod/ephemeralcontainers;pod/containers;pod/volumes
type WorkloadValidationTarget string
const (
DeprecatedValidateImages WorkloadValidationTarget = "pod/images"
ValidateInitContainers WorkloadValidationTarget = "pod/initcontainers"
ValidateEphemeralContainers WorkloadValidationTarget = "pod/ephemeralcontainers"
ValidateContainers WorkloadValidationTarget = "pod/containers"
ValidateVolumes WorkloadValidationTarget = "pod/volumes"
)
// +kubebuilder:object:generate=true
type NamespaceRuleEnforceWorkloadsBody struct {
// Define the enforcement targets this rule applies to.
// If empty, each webhook applies its own backwards-compatible default.
// +optional
Targets []WorkloadValidationTarget `json:"targets,omitempty"`
// Define Pod QoS classes matched by this enforcement rule.
// Supported values are Guaranteed, Burstable and BestEffort.
// +optional
QoSClasses []corev1.PodQOSClass `json:"qosClasses,omitempty"`
// Define registries which are allowed to be used within this tenant
// The rules are aggregated, since you can use Regular Expressions the match registry endpoints
Registries []OCIRegistry `json:"registries,omitempty"`
}
+1 -1
View File
@@ -11,7 +11,7 @@ import (
type NamespaceRulePermissionBody struct {
// Define Promotion Rules which distributed additional ClusterRoles across the Tenant
// for promoted ServiceAccounts.
Promotions []*NamespaceRulePromotionRule `json:"rules,omitempty"`
Promotions []*NamespaceRulePromotionRule `json:"promotions,omitempty"`
}
// +kubebuilder:object:generate=true
+3 -3
View File
@@ -12,18 +12,18 @@ import (
type NamespaceRuleBodyNamespace struct {
// Enforcement for given rule
//+optional
Enforce NamespaceRuleEnforceBody `json:"enforce,omitzero"`
Enforce *NamespaceRuleEnforceBody `json:"enforce,omitzero"`
}
// Rules Distributed via Tenants
// +kubebuilder:object:generate=true
type NamespaceRuleBodyTenant struct {
NamespaceRuleBodyNamespace `json:",inline"`
*NamespaceRuleBodyNamespace `json:",inline"`
// Select namespaces which are going to be targeted with this rule
NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"`
// Permissions for given rule
//+optional
Permissions NamespaceRulePermissionBody `json:"permissions,omitzero"`
Permissions NamespaceRulePermissionBody `json:"permissions,omitempty"`
}
+39 -10
View File
@@ -15,7 +15,11 @@ import (
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NamespaceRuleBodyNamespace) DeepCopyInto(out *NamespaceRuleBodyNamespace) {
*out = *in
in.Enforce.DeepCopyInto(&out.Enforce)
if in.Enforce != nil {
in, out := &in.Enforce, &out.Enforce
*out = new(NamespaceRuleEnforceBody)
(*in).DeepCopyInto(*out)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceRuleBodyNamespace.
@@ -31,7 +35,11 @@ func (in *NamespaceRuleBodyNamespace) DeepCopy() *NamespaceRuleBodyNamespace {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NamespaceRuleBodyTenant) DeepCopyInto(out *NamespaceRuleBodyTenant) {
*out = *in
in.NamespaceRuleBodyNamespace.DeepCopyInto(&out.NamespaceRuleBodyNamespace)
if in.NamespaceRuleBodyNamespace != nil {
in, out := &in.NamespaceRuleBodyNamespace, &out.NamespaceRuleBodyNamespace
*out = new(NamespaceRuleBodyNamespace)
(*in).DeepCopyInto(*out)
}
if in.NamespaceSelector != nil {
in, out := &in.NamespaceSelector, &out.NamespaceSelector
*out = new(metav1.LabelSelector)
@@ -53,6 +61,32 @@ func (in *NamespaceRuleBodyTenant) DeepCopy() *NamespaceRuleBodyTenant {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NamespaceRuleEnforceBody) DeepCopyInto(out *NamespaceRuleEnforceBody) {
*out = *in
in.Workloads.DeepCopyInto(&out.Workloads)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceRuleEnforceBody.
func (in *NamespaceRuleEnforceBody) DeepCopy() *NamespaceRuleEnforceBody {
if in == nil {
return nil
}
out := new(NamespaceRuleEnforceBody)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NamespaceRuleEnforceWorkloadsBody) DeepCopyInto(out *NamespaceRuleEnforceWorkloadsBody) {
*out = *in
if in.Targets != nil {
in, out := &in.Targets, &out.Targets
*out = make([]WorkloadValidationTarget, len(*in))
copy(*out, *in)
}
if in.QoSClasses != nil {
in, out := &in.QoSClasses, &out.QoSClasses
*out = make([]v1.PodQOSClass, len(*in))
copy(*out, *in)
}
if in.Registries != nil {
in, out := &in.Registries, &out.Registries
*out = make([]OCIRegistry, len(*in))
@@ -62,12 +96,12 @@ func (in *NamespaceRuleEnforceBody) DeepCopyInto(out *NamespaceRuleEnforceBody)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceRuleEnforceBody.
func (in *NamespaceRuleEnforceBody) DeepCopy() *NamespaceRuleEnforceBody {
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceRuleEnforceWorkloadsBody.
func (in *NamespaceRuleEnforceWorkloadsBody) DeepCopy() *NamespaceRuleEnforceWorkloadsBody {
if in == nil {
return nil
}
out := new(NamespaceRuleEnforceBody)
out := new(NamespaceRuleEnforceWorkloadsBody)
in.DeepCopyInto(out)
return out
}
@@ -132,11 +166,6 @@ func (in *OCIRegistry) DeepCopyInto(out *OCIRegistry) {
*out = make([]v1.PullPolicy, len(*in))
copy(*out, *in)
}
if in.Validation != nil {
in, out := &in.Validation, &out.Validation
*out = make([]RegistryValidationTarget, len(*in))
copy(*out, *in)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OCIRegistry.
+1
View File
@@ -38,6 +38,7 @@ const (
ReasonMissingFQCI string = "MissingFQCI"
ReasonForbiddenContainerRegistry string = "ForbiddenContainerRegistry"
ReasonForbiddenPullPolicy string = "ForbiddenPullPolicy"
ReasonForbiddenPodQoSClass string = "ForbiddenPodQoSClass"
// Ingress.
ReasonWildcardDenied string = "WildcardDenied"
+10 -9
View File
@@ -21,18 +21,19 @@ func (ReconcileRequestedPredicate) Update(e event.UpdateEvent) bool {
return false
}
oldA := e.ObjectOld.GetAnnotations()
newA := e.ObjectNew.GetAnnotations()
oldValue, oldPresent := e.ObjectOld.GetAnnotations()[meta.ReconcileAnnotation]
newValue, newPresent := e.ObjectNew.GetAnnotations()[meta.ReconcileAnnotation]
oldV := ""
if oldA != nil {
oldV = oldA[meta.ReconcileAnnotation]
oldPresent = oldPresent && oldValue != ""
newPresent = newPresent && newValue != ""
if !newPresent {
return false
}
newV := ""
if newA != nil {
newV = newA[meta.ReconcileAnnotation]
if !oldPresent {
return true
}
return newV != "" && newV != oldV
return oldValue != newValue
}
+4
View File
@@ -34,3 +34,7 @@ func NegateQuantity(in resource.Quantity) resource.Quantity {
return out
}
func QuantityEqual(a, b resource.Quantity) bool {
return a.Cmp(b) == 0
}
+333
View File
@@ -0,0 +1,333 @@
// Copyright 2020-2026 Project Capsule Authors
// SPDX-License-Identifier: Apache-2.0
package quota
import (
"strings"
"testing"
"k8s.io/apimachinery/pkg/api/resource"
)
func TestValidateQuantity(t *testing.T) {
t.Parallel()
tests := []struct {
name string
quantity resource.Quantity
wantErr bool
errContain string
}{
{
name: "positive integer quantity is valid",
quantity: resource.MustParse("1"),
wantErr: false,
},
{
name: "positive milli quantity is valid",
quantity: resource.MustParse("100m"),
wantErr: false,
},
{
name: "positive binary memory quantity is valid",
quantity: resource.MustParse("128Mi"),
wantErr: false,
},
{
name: "positive decimal memory quantity is valid",
quantity: resource.MustParse("1Gi"),
wantErr: false,
},
{
name: "zero quantity is invalid",
quantity: resource.MustParse("0"),
wantErr: true,
errContain: "quantity must not be negative or 0",
},
{
name: "negative integer quantity is invalid",
quantity: resource.MustParse("-1"),
wantErr: true,
errContain: "quantity must not be negative or 0",
},
{
name: "negative milli quantity is invalid",
quantity: resource.MustParse("-100m"),
wantErr: true,
errContain: "quantity must not be negative or 0",
},
{
name: "negative binary quantity is invalid",
quantity: resource.MustParse("-128Mi"),
wantErr: true,
errContain: "quantity must not be negative or 0",
},
{
name: "very small positive milli quantity is valid",
quantity: resource.MustParse("1m"),
wantErr: false,
},
{
name: "large positive quantity is valid",
quantity: resource.MustParse("999999999999"),
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
err := ValidateQuantity(tt.quantity)
if tt.wantErr {
if err == nil {
t.Fatal("expected error, got nil")
}
if tt.errContain != "" && !strings.Contains(err.Error(), tt.errContain) {
t.Fatalf("expected error to contain %q, got %q", tt.errContain, err.Error())
}
return
}
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
})
}
}
func TestClampQuantityToZero(t *testing.T) {
t.Parallel()
tests := []struct {
name string
in resource.Quantity
want resource.Quantity
}{
{
name: "positive quantity is unchanged",
in: resource.MustParse("5"),
want: resource.MustParse("5"),
},
{
name: "positive milli quantity is unchanged",
in: resource.MustParse("250m"),
want: resource.MustParse("250m"),
},
{
name: "zero quantity is unchanged",
in: resource.MustParse("0"),
want: resource.MustParse("0"),
},
{
name: "negative integer quantity is clamped to zero",
in: resource.MustParse("-5"),
want: resource.MustParse("0"),
},
{
name: "negative milli quantity is clamped to zero",
in: resource.MustParse("-250m"),
want: resource.MustParse("0"),
},
{
name: "negative memory quantity is clamped to zero",
in: resource.MustParse("-1Gi"),
want: resource.MustParse("0"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := tt.in.DeepCopy()
ClampQuantityToZero(&got)
if !QuantityEqual(got, tt.want) {
t.Fatalf("expected %q, got %q", tt.want.String(), got.String())
}
})
}
}
func TestNegateQuantity(t *testing.T) {
t.Parallel()
tests := []struct {
name string
in resource.Quantity
want resource.Quantity
}{
{
name: "positive integer becomes negative",
in: resource.MustParse("5"),
want: resource.MustParse("-5"),
},
{
name: "negative integer becomes positive",
in: resource.MustParse("-5"),
want: resource.MustParse("5"),
},
{
name: "zero remains zero",
in: resource.MustParse("0"),
want: resource.MustParse("0"),
},
{
name: "positive milli becomes negative",
in: resource.MustParse("250m"),
want: resource.MustParse("-250m"),
},
{
name: "negative milli becomes positive",
in: resource.MustParse("-250m"),
want: resource.MustParse("250m"),
},
{
name: "positive binary memory becomes negative",
in: resource.MustParse("1Gi"),
want: resource.MustParse("-1Gi"),
},
{
name: "negative binary memory becomes positive",
in: resource.MustParse("-1Gi"),
want: resource.MustParse("1Gi"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
original := tt.in.DeepCopy()
got := NegateQuantity(tt.in)
if !QuantityEqual(got, tt.want) {
t.Fatalf("expected %q, got %q", tt.want.String(), got.String())
}
if !QuantityEqual(tt.in, original) {
t.Fatalf("NegateQuantity mutated input: expected original %q, got %q", original.String(), tt.in.String())
}
})
}
}
func TestQuantityEqual(t *testing.T) {
t.Parallel()
tests := []struct {
name string
a resource.Quantity
b resource.Quantity
want bool
}{
{
name: "same integer quantities are equal",
a: resource.MustParse("1"),
b: resource.MustParse("1"),
want: true,
},
{
name: "equivalent decimal and milli quantities are equal",
a: resource.MustParse("1"),
b: resource.MustParse("1000m"),
want: true,
},
{
name: "equivalent binary quantities are equal",
a: resource.MustParse("1Gi"),
b: resource.MustParse("1024Mi"),
want: true,
},
{
name: "different integer quantities are not equal",
a: resource.MustParse("1"),
b: resource.MustParse("2"),
want: false,
},
{
name: "positive and negative quantities are not equal",
a: resource.MustParse("1"),
b: resource.MustParse("-1"),
want: false,
},
{
name: "zero quantities are equal",
a: resource.MustParse("0"),
b: resource.MustParse("0"),
want: true,
},
{
name: "equivalent CPU quantities are equal",
a: resource.MustParse("500m"),
b: resource.MustParse("0.5"),
want: true,
},
{
name: "nearby milli quantities are not equal",
a: resource.MustParse("500m"),
b: resource.MustParse("501m"),
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := QuantityEqual(tt.a, tt.b)
if got != tt.want {
t.Fatalf("expected %t for %q and %q, got %t", tt.want, tt.a.String(), tt.b.String(), got)
}
})
}
}
func TestClampQuantityToZeroMutatesOnlyNegativeInput(t *testing.T) {
t.Parallel()
q := resource.MustParse("-10")
ClampQuantityToZero(&q)
if !QuantityEqual(q, resource.MustParse("0")) {
t.Fatalf("expected quantity to be clamped to zero, got %q", q.String())
}
q = resource.MustParse("10")
ClampQuantityToZero(&q)
if !QuantityEqual(q, resource.MustParse("10")) {
t.Fatalf("expected positive quantity to remain unchanged, got %q", q.String())
}
}
func TestNegateQuantityReturnsIndependentCopy(t *testing.T) {
t.Parallel()
in := resource.MustParse("10")
out := NegateQuantity(in)
if !QuantityEqual(in, resource.MustParse("10")) {
t.Fatalf("expected input to remain unchanged, got %q", in.String())
}
if !QuantityEqual(out, resource.MustParse("-10")) {
t.Fatalf("expected output to be negated, got %q", out.String())
}
ClampQuantityToZero(&out)
if !QuantityEqual(out, resource.MustParse("0")) {
t.Fatalf("expected output to be clampable independently, got %q", out.String())
}
if !QuantityEqual(in, resource.MustParse("10")) {
t.Fatalf("expected input to remain unchanged after mutating output, got %q", in.String())
}
}
+155
View File
@@ -0,0 +1,155 @@
// Copyright 2020-2026 Project Capsule Authors
// SPDX-License-Identifier: Apache-2.0
package workloads
import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
)
func GetPodQoSClass(pod *corev1.Pod) corev1.PodQOSClass {
if pod == nil {
return corev1.PodQOSBestEffort
}
// Pod Can not change QOSClass during it's lifetime. Therefore we can use the status value if present.
// Docs: The QoS class is determined when the Pod is created and remains unchanged for the lifetime of the Pod. If you later attempt an in-place resize that would result in a different QoS class, the resize is rejected by admission.
if pod.Status.QOSClass != "" {
return pod.Status.QOSClass
}
return computePodQoSClass(pod)
}
func computePodQoSClass(pod *corev1.Pod) corev1.PodQOSClass {
if podLevelQoS, ok := computePodLevelQoSClass(pod); ok {
return podLevelQoS
}
return computeContainerLevelQoSClass(pod)
}
func computePodLevelQoSClass(pod *corev1.Pod) (corev1.PodQOSClass, bool) {
if pod == nil {
return corev1.PodQOSBestEffort, false
}
if pod.Spec.Resources == nil {
return corev1.PodQOSBestEffort, false
}
requests := pod.Spec.Resources.Requests
limits := pod.Spec.Resources.Limits
if !hasSupportedQoSResource(requests) && !hasSupportedQoSResource(limits) {
return corev1.PodQOSBestEffort, false
}
cpuRequest, hasCPURequest := positiveResource(requests, corev1.ResourceCPU)
memoryRequest, hasMemoryRequest := positiveResource(requests, corev1.ResourceMemory)
cpuLimit, hasCPULimit := positiveResource(limits, corev1.ResourceCPU)
memoryLimit, hasMemoryLimit := positiveResource(limits, corev1.ResourceMemory)
if hasCPURequest &&
hasMemoryRequest &&
hasCPULimit &&
hasMemoryLimit &&
cpuRequest.Cmp(cpuLimit) == 0 &&
memoryRequest.Cmp(memoryLimit) == 0 {
return corev1.PodQOSGuaranteed, true
}
return corev1.PodQOSBurstable, true
}
func computeContainerLevelQoSClass(pod *corev1.Pod) corev1.PodQOSClass {
if pod == nil {
return corev1.PodQOSBestEffort
}
hasSupportedRequestOrLimit := false
isGuaranteed := true
containers := make([]corev1.Container, 0,
len(pod.Spec.Containers)+
len(pod.Spec.InitContainers)+
len(pod.Spec.EphemeralContainers),
)
containers = append(containers, pod.Spec.Containers...)
containers = append(containers, pod.Spec.InitContainers...)
for _, container := range pod.Spec.EphemeralContainers {
containers = append(containers, corev1.Container{
Name: container.Name,
Resources: container.Resources,
})
}
if len(containers) == 0 {
return corev1.PodQOSBestEffort
}
for _, container := range containers {
requests := container.Resources.Requests
limits := container.Resources.Limits
cpuRequest, hasCPURequest := positiveResource(requests, corev1.ResourceCPU)
memoryRequest, hasMemoryRequest := positiveResource(requests, corev1.ResourceMemory)
cpuLimit, hasCPULimit := positiveResource(limits, corev1.ResourceCPU)
memoryLimit, hasMemoryLimit := positiveResource(limits, corev1.ResourceMemory)
if hasCPURequest || hasMemoryRequest || hasCPULimit || hasMemoryLimit {
hasSupportedRequestOrLimit = true
}
if !hasCPURequest ||
!hasMemoryRequest ||
!hasCPULimit ||
!hasMemoryLimit ||
cpuRequest.Cmp(cpuLimit) != 0 ||
memoryRequest.Cmp(memoryLimit) != 0 {
isGuaranteed = false
}
}
if !hasSupportedRequestOrLimit {
return corev1.PodQOSBestEffort
}
if isGuaranteed {
return corev1.PodQOSGuaranteed
}
return corev1.PodQOSBurstable
}
func hasSupportedQoSResource(resources corev1.ResourceList) bool {
for name, quantity := range resources {
if isSupportedQoSComputeResource(name) && quantity.Sign() > 0 {
return true
}
}
return false
}
func positiveResource(resources corev1.ResourceList, name corev1.ResourceName) (resource.Quantity, bool) {
quantity, ok := resources[name]
if !ok || quantity.Sign() <= 0 {
return resource.Quantity{}, false
}
return quantity, true
}
//nolint:exhaustive
func isSupportedQoSComputeResource(name corev1.ResourceName) bool {
switch name {
case corev1.ResourceCPU, corev1.ResourceMemory:
return true
default:
return false
}
}
+835
View File
@@ -0,0 +1,835 @@
package workloads
import (
"testing"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
)
func TestGetPodQoSClass(t *testing.T) {
t.Parallel()
tests := []struct {
name string
pod *corev1.Pod
want corev1.PodQOSClass
}{
{
name: "nil pod returns BestEffort",
pod: nil,
want: corev1.PodQOSBestEffort,
},
{
name: "status QoS class takes precedence over computed value",
pod: &corev1.Pod{
Status: corev1.PodStatus{
QOSClass: corev1.PodQOSGuaranteed,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
bestEffortContainer("app"),
},
},
},
want: corev1.PodQOSGuaranteed,
},
{
name: "empty status computes QoS class",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
bestEffortContainer("app"),
},
},
},
want: corev1.PodQOSBestEffort,
},
{
name: "pod-level Guaranteed takes precedence over BestEffort containers",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: guaranteedPodResourcesPtr("100m", "128Mi"),
Containers: []corev1.Container{
bestEffortContainer("app"),
},
},
},
want: corev1.PodQOSGuaranteed,
},
{
name: "pod-level Burstable takes precedence over Guaranteed containers",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: requestOnlyPodResourcesPtr("100m", "128Mi"),
Containers: []corev1.Container{
guaranteedContainer("app", "100m", "128Mi"),
},
},
},
want: corev1.PodQOSBurstable,
},
{
name: "BestEffort without pod-level or container-level resources",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
bestEffortContainer("app"),
},
},
},
want: corev1.PodQOSBestEffort,
},
{
name: "Burstable with container requests only",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
requestOnlyContainer("app", "100m", "128Mi"),
},
},
},
want: corev1.PodQOSBurstable,
},
{
name: "Burstable with container limits only",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
limitOnlyContainer("app", "100m", "128Mi"),
},
},
},
want: corev1.PodQOSBurstable,
},
{
name: "Guaranteed with equal CPU and memory requests and limits",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
guaranteedContainer("app", "100m", "128Mi"),
},
},
},
want: corev1.PodQOSGuaranteed,
},
{
name: "Burstable when CPU request and limit differ",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
containerWithResources(
"app",
corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
),
},
},
},
want: corev1.PodQOSBurstable,
},
{
name: "Burstable when memory request and limit differ",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
containerWithResources(
"app",
corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("256Mi"),
},
),
},
},
},
want: corev1.PodQOSBurstable,
},
{
name: "Burstable when one container is BestEffort",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
guaranteedContainer("app", "100m", "128Mi"),
bestEffortContainer("sidecar"),
},
},
},
want: corev1.PodQOSBurstable,
},
{
name: "Guaranteed with multiple guaranteed containers",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
guaranteedContainer("app", "100m", "128Mi"),
guaranteedContainer("sidecar", "50m", "64Mi"),
},
},
},
want: corev1.PodQOSGuaranteed,
},
{
name: "Burstable when init container has requests only",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{
requestOnlyContainer("init", "100m", "128Mi"),
},
Containers: []corev1.Container{
guaranteedContainer("app", "100m", "128Mi"),
},
},
},
want: corev1.PodQOSBurstable,
},
{
name: "Guaranteed when init and regular containers are guaranteed",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{
guaranteedContainer("init", "100m", "128Mi"),
},
Containers: []corev1.Container{
guaranteedContainer("app", "100m", "128Mi"),
},
},
},
want: corev1.PodQOSGuaranteed,
},
{
name: "Burstable when ephemeral container has requests only",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
guaranteedContainer("app", "100m", "128Mi"),
},
EphemeralContainers: []corev1.EphemeralContainer{
{
EphemeralContainerCommon: corev1.EphemeralContainerCommon{
Name: "debug",
Resources: requestOnlyRequirements("100m", "128Mi"),
},
},
},
},
},
want: corev1.PodQOSBurstable,
},
{
name: "Guaranteed when ephemeral container is guaranteed",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
guaranteedContainer("app", "100m", "128Mi"),
},
EphemeralContainers: []corev1.EphemeralContainer{
{
EphemeralContainerCommon: corev1.EphemeralContainerCommon{
Name: "debug",
Resources: guaranteedRequirements("100m", "128Mi"),
},
},
},
},
},
want: corev1.PodQOSGuaranteed,
},
{
name: "zero CPU and memory requests and limits are ignored",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
containerWithResources(
"app",
corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("0"),
corev1.ResourceMemory: resource.MustParse("0"),
},
corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("0"),
corev1.ResourceMemory: resource.MustParse("0"),
},
),
},
},
},
want: corev1.PodQOSBestEffort,
},
{
name: "unsupported resources do not influence QoS",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
containerWithResources(
"app",
corev1.ResourceList{
corev1.ResourceName("example.com/gpu"): resource.MustParse("1"),
},
corev1.ResourceList{
corev1.ResourceName("example.com/gpu"): resource.MustParse("1"),
},
),
},
},
},
want: corev1.PodQOSBestEffort,
},
{
name: "hugepages do not influence QoS",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
containerWithResources(
"app",
corev1.ResourceList{
corev1.ResourceName("hugepages-2Mi"): resource.MustParse("2Mi"),
},
corev1.ResourceList{
corev1.ResourceName("hugepages-2Mi"): resource.MustParse("2Mi"),
},
),
},
},
},
want: corev1.PodQOSBestEffort,
},
{
name: "pod-level unsupported resources do not influence QoS",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceName("example.com/gpu"): resource.MustParse("1"),
},
Limits: corev1.ResourceList{
corev1.ResourceName("example.com/gpu"): resource.MustParse("1"),
},
},
Containers: []corev1.Container{
bestEffortContainer("app"),
},
},
},
want: corev1.PodQOSBestEffort,
},
{
name: "pod-level zero resources do not mask container-level Guaranteed",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("0"),
corev1.ResourceMemory: resource.MustParse("0"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("0"),
corev1.ResourceMemory: resource.MustParse("0"),
},
},
Containers: []corev1.Container{
guaranteedContainer("app", "100m", "128Mi"),
},
},
},
want: corev1.PodQOSGuaranteed,
},
{
name: "pod-level CPU only request is Burstable",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
},
},
Containers: []corev1.Container{
bestEffortContainer("app"),
},
},
},
want: corev1.PodQOSBurstable,
},
{
name: "pod-level memory only limit is Burstable",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: &corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
},
Containers: []corev1.Container{
bestEffortContainer("app"),
},
},
},
want: corev1.PodQOSBurstable,
},
{
name: "pod-level CPU and memory unequal request and limit is Burstable",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
},
Containers: []corev1.Container{
bestEffortContainer("app"),
},
},
},
want: corev1.PodQOSBurstable,
},
{
name: "no containers and no resources is BestEffort",
pod: &corev1.Pod{
Spec: corev1.PodSpec{},
},
want: corev1.PodQOSBestEffort,
},
{
name: "aggregate container requests and limits equal returns Guaranteed",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
guaranteedContainer("app", "100m", "128Mi"),
guaranteedContainer("sidecar", "200m", "256Mi"),
},
},
},
want: corev1.PodQOSGuaranteed,
},
{
name: "mismatched container requests/limits are Burstable even if totals match",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
containerWithResources(
"a",
corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
),
containerWithResources(
"b",
corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
),
},
},
},
want: corev1.PodQOSBurstable,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := GetPodQoSClass(tt.pod)
if got != tt.want {
t.Fatalf("expected QoS class %q, got %q", tt.want, got)
}
})
}
}
func TestComputePodLevelQoSClassNilPodResources(t *testing.T) {
t.Parallel()
got, ok := computePodLevelQoSClass(&corev1.Pod{
Spec: corev1.PodSpec{
Resources: nil,
},
})
if got != corev1.PodQOSBestEffort {
t.Fatalf("expected QoS class %q, got %q", corev1.PodQOSBestEffort, got)
}
if ok {
t.Fatalf("expected ok=false, got true")
}
}
func TestComputePodLevelQoSClass(t *testing.T) {
t.Parallel()
tests := []struct {
name string
pod *corev1.Pod
want corev1.PodQOSClass
ok bool
}{
{
name: "nil pod returns false",
pod: nil,
want: corev1.PodQOSBestEffort,
ok: false,
},
{
name: "no pod-level resources returns false",
pod: &corev1.Pod{
Spec: corev1.PodSpec{},
},
want: corev1.PodQOSBestEffort,
ok: false,
},
{
name: "pod-level Guaranteed",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: guaranteedPodResourcesPtr("100m", "128Mi"),
},
},
want: corev1.PodQOSGuaranteed,
ok: true,
},
{
name: "pod-level request only is Burstable",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: requestOnlyPodResourcesPtr("100m", "128Mi"),
},
},
want: corev1.PodQOSBurstable,
ok: true,
},
{
name: "pod-level limit only is Burstable",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: limitOnlyPodResourcesPtr("100m", "128Mi"),
},
},
want: corev1.PodQOSBurstable,
ok: true,
},
{
name: "pod-level CPU only equal request and limit is Burstable",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
},
},
},
},
want: corev1.PodQOSBurstable,
ok: true,
},
{
name: "pod-level memory only equal request and limit is Burstable",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
},
},
},
want: corev1.PodQOSBurstable,
ok: true,
},
{
name: "pod-level zero resources return false",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("0"),
corev1.ResourceMemory: resource.MustParse("0"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("0"),
corev1.ResourceMemory: resource.MustParse("0"),
},
},
},
},
want: corev1.PodQOSBestEffort,
ok: false,
},
{
name: "pod-level unsupported resources return false",
pod: &corev1.Pod{
Spec: corev1.PodSpec{
Resources: &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceName("example.com/gpu"): resource.MustParse("1"),
},
Limits: corev1.ResourceList{
corev1.ResourceName("example.com/gpu"): resource.MustParse("1"),
},
},
},
},
want: corev1.PodQOSBestEffort,
ok: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got, ok := computePodLevelQoSClass(tt.pod)
if got != tt.want {
t.Fatalf("expected QoS class %q, got %q", tt.want, got)
}
if ok != tt.ok {
t.Fatalf("expected ok=%t, got %t", tt.ok, ok)
}
})
}
}
func TestQoSHelpers(t *testing.T) {
t.Parallel()
t.Run("hasSupportedQoSResource", func(t *testing.T) {
t.Parallel()
tests := []struct {
name string
resources corev1.ResourceList
want bool
}{
{
name: "nil resources",
resources: nil,
want: false,
},
{
name: "empty resources",
resources: corev1.ResourceList{},
want: false,
},
{
name: "CPU positive",
resources: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
},
want: true,
},
{
name: "memory positive",
resources: corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("1Mi"),
},
want: true,
},
{
name: "CPU zero",
resources: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("0"),
},
want: false,
},
{
name: "unsupported positive",
resources: corev1.ResourceList{
corev1.ResourceName("example.com/gpu"): resource.MustParse("1"),
},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := hasSupportedQoSResource(tt.resources)
if got != tt.want {
t.Fatalf("expected %t, got %t", tt.want, got)
}
})
}
})
t.Run("positiveResource", func(t *testing.T) {
t.Parallel()
resources := corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("0"),
}
if got, ok := positiveResource(resources, corev1.ResourceCPU); !ok || got.Cmp(resource.MustParse("100m")) != 0 {
t.Fatalf("expected positive CPU resource, got %q ok=%t", got.String(), ok)
}
if got, ok := positiveResource(resources, corev1.ResourceMemory); ok {
t.Fatalf("expected zero memory to be ignored, got %q ok=%t", got.String(), ok)
}
if got, ok := positiveResource(resources, corev1.ResourceStorage); ok {
t.Fatalf("expected missing storage to be ignored, got %q ok=%t", got.String(), ok)
}
})
t.Run("isSupportedQoSComputeResource", func(t *testing.T) {
t.Parallel()
tests := []struct {
name string
in corev1.ResourceName
want bool
}{
{name: "CPU", in: corev1.ResourceCPU, want: true},
{name: "memory", in: corev1.ResourceMemory, want: true},
{name: "storage", in: corev1.ResourceStorage, want: false},
{name: "ephemeral storage", in: corev1.ResourceEphemeralStorage, want: false},
{name: "extended resource", in: corev1.ResourceName("example.com/gpu"), want: false},
{name: "hugepage", in: corev1.ResourceName("hugepages-2Mi"), want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := isSupportedQoSComputeResource(tt.in)
if got != tt.want {
t.Fatalf("expected %t, got %t", tt.want, got)
}
})
}
})
}
func bestEffortContainer(name string) corev1.Container {
return corev1.Container{Name: name}
}
func requestOnlyContainer(name, cpu, memory string) corev1.Container {
return corev1.Container{
Name: name,
Resources: requestOnlyRequirements(cpu, memory),
}
}
func limitOnlyContainer(name, cpu, memory string) corev1.Container {
return corev1.Container{
Name: name,
Resources: limitOnlyRequirements(cpu, memory),
}
}
func guaranteedContainer(name, cpu, memory string) corev1.Container {
return corev1.Container{
Name: name,
Resources: guaranteedRequirements(cpu, memory),
}
}
func containerWithResources(
name string,
requests corev1.ResourceList,
limits corev1.ResourceList,
) corev1.Container {
return corev1.Container{
Name: name,
Resources: corev1.ResourceRequirements{
Requests: requests,
Limits: limits,
},
}
}
func requestOnlyRequirements(cpu, memory string) corev1.ResourceRequirements {
return corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(cpu),
corev1.ResourceMemory: resource.MustParse(memory),
},
}
}
func limitOnlyRequirements(cpu, memory string) corev1.ResourceRequirements {
return corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(cpu),
corev1.ResourceMemory: resource.MustParse(memory),
},
}
}
func guaranteedRequirements(cpu, memory string) corev1.ResourceRequirements {
return corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(cpu),
corev1.ResourceMemory: resource.MustParse(memory),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(cpu),
corev1.ResourceMemory: resource.MustParse(memory),
},
}
}
func requestOnlyPodResources(cpu, memory string) corev1.ResourceRequirements {
return requestOnlyRequirements(cpu, memory)
}
func limitOnlyPodResources(cpu, memory string) corev1.ResourceRequirements {
return limitOnlyRequirements(cpu, memory)
}
func guaranteedPodResources(cpu, memory string) corev1.ResourceRequirements {
return guaranteedRequirements(cpu, memory)
}
func requestOnlyPodResourcesPtr(cpu, memory string) *corev1.ResourceRequirements {
resources := requestOnlyRequirements(cpu, memory)
return &resources
}
func limitOnlyPodResourcesPtr(cpu, memory string) *corev1.ResourceRequirements {
resources := limitOnlyRequirements(cpu, memory)
return &resources
}
func guaranteedPodResourcesPtr(cpu, memory string) *corev1.ResourceRequirements {
resources := guaranteedRequirements(cpu, memory)
return &resources
}
+6 -19
View File
@@ -33,9 +33,10 @@ func GetManagedRuleStatus(
return obj, err
}
// BuildNamespaceRuleBodyForNamespace returns the aggregated rule body that applies to `ns`.
// BuildNamespaceRuleBodyStatus returns the aggregated rule bodies that apply to ns.
// - Rules with nil NamespaceSelector match all namespaces.
// - Matching rules are combined in the order they appear in tnt.Spec.Rules (important for "later wins" semantics).
// - Matching rules are returned in the order they appear in tnt.Spec.Rules.
// - Order is important because registry/QoS evaluation uses "later allow/deny wins" semantics.
func BuildNamespaceRuleBodyStatus(
ctx context.Context,
c client.Reader,
@@ -46,7 +47,6 @@ func BuildNamespaceRuleBodyStatus(
return nil, nil
}
// Treat nil labels map as empty.
nsLabels := labels.Set{}
if ns.Labels != nil {
nsLabels = labels.Set(ns.Labels)
@@ -70,25 +70,12 @@ func BuildNamespaceRuleBodyStatus(
}
}
normalized := rules.NamespaceRuleBodyNamespace{
Enforce: rules.NamespaceRuleEnforceBody{
Action: rule.Enforce.Action,
Registries: append(
[]rules.OCIRegistry(nil),
rule.Enforce.Registries...,
),
},
}
if normalized.Enforce.Action == "" {
normalized.Enforce.Action = rules.ActionTypeDeny
}
if len(normalized.Enforce.Registries) == 0 {
body := rule.NamespaceRuleBodyNamespace
if body == nil || body.Enforce == nil {
continue
}
out = append(out, &normalized)
out = append(out, body.DeepCopy())
}
return out, nil