// Copyright 2020-2026 Project Capsule Authors. // SPDX-License-Identifier: Apache-2.0 package e2e import ( "context" "fmt" "strings" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/pkg/api/meta" "github.com/projectcapsule/capsule/pkg/api/rbac" "github.com/projectcapsule/capsule/pkg/api/rules" "github.com/projectcapsule/capsule/pkg/api/runtime" ) var _ = Describe("enforcing pod schedulerName namespace rules", Ordered, Label("tenant", "rules", "enforce", "workloads", "scheduler"), func() { const ownerName = "e2e-rules-scheduler" var tnt *capsulev1beta2.Tenant newTenant := func() *capsulev1beta2.Tenant { return &capsulev1beta2.Tenant{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-rule-scheduler", Labels: map[string]string{ "env": "e2e", }, }, Spec: capsulev1beta2.TenantSpec{ Owners: rbac.OwnerListSpec{ { CoreOwnerSpec: rbac.CoreOwnerSpec{ UserSpec: rbac.UserSpec{ Name: ownerName, Kind: "User", }, }, }, }, Rules: []*rules.NamespaceRuleBodyTenant{ { NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ Enforce: &rules.NamespaceRuleEnforceBody{ Action: rules.ActionTypeDeny, Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ Schedulers: []runtime.ExpressionMatch{ { Exact: []string{ "forbidden-scheduler", "legacy-scheduler", }, }, }, }, }, }, }, { NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ Enforce: &rules.NamespaceRuleEnforceBody{ Action: rules.ActionTypeAudit, Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ Schedulers: []runtime.ExpressionMatch{ { Exact: []string{ "audited-scheduler", }, }, }, }, }, }, }, { NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "allow-forbidden-scheduler": "true", }, }, NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ Enforce: &rules.NamespaceRuleEnforceBody{ Action: rules.ActionTypeAllow, Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ Schedulers: []runtime.ExpressionMatch{ { Exact: []string{ "forbidden-scheduler", }, }, }, }, }, }, }, { NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "deny-audited-scheduler": "true", }, }, NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ Enforce: &rules.NamespaceRuleEnforceBody{ Action: rules.ActionTypeDeny, Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ Schedulers: []runtime.ExpressionMatch{ { Exact: []string{ "audited-scheduler", }, }, }, }, }, }, }, { NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "allow-team-scheduler": "true", }, }, NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ Enforce: &rules.NamespaceRuleEnforceBody{ Action: rules.ActionTypeAllow, Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ Schedulers: []runtime.ExpressionMatch{ { ExpressionRegex: runtime.ExpressionRegex{ Expression: "^team-[a-z0-9-]+$", }, }, }, }, }, }, }, { NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "deny-team-scheduler": "true", }, }, NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{ Enforce: &rules.NamespaceRuleEnforceBody{ Action: rules.ActionTypeDeny, Workloads: rules.NamespaceRuleEnforceWorkloadsBody{ Schedulers: []runtime.ExpressionMatch{ { ExpressionRegex: runtime.ExpressionRegex{ Expression: "^team-blocked-[a-z0-9-]+$", }, }, }, }, }, }, }, }, }, } } type expectedSchedulerStatusRule struct { action rules.ActionType targets []rules.WorkloadValidationTarget schedulers []runtime.ExpressionMatch } expectNamespaceStatusRules := func(nsName string, want []expectedSchedulerStatusRule) { Eventually(func(g Gomega) { nsStatus := &capsulev1beta2.RuleStatus{} g.Expect(k8sClient.Get( context.Background(), client.ObjectKey{Name: meta.NameForManagedRuleStatus(), Namespace: nsName}, nsStatus, )).To(Succeed()) g.Expect(nsStatus.Status.Rules).To(HaveLen(len(want))) for i, expected := range want { gotRule := nsStatus.Status.Rules[i] g.Expect(gotRule).NotTo(BeNil()) g.Expect(gotRule.Enforce.Action).To(Equal(expected.action)) g.Expect(gotRule.Enforce.Workloads.Targets).To(Equal(expected.targets)) g.Expect(gotRule.Enforce.Workloads.Schedulers).To(Equal(expected.schedulers)) } }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } createPodAndExpectDenied := func(cs kubernetes.Interface, nsName string, pod *corev1.Pod, substrings ...string) { base := pod.DeepCopy() baseName := base.Name if baseName == "" { baseName = "pod" } Eventually(func() error { p := base.DeepCopy() p.Name = fmt.Sprintf("%s-%d", baseName, time.Now().UnixNano()%1e6) _, err := cs.CoreV1().Pods(nsName).Create(context.Background(), p, metav1.CreateOptions{}) if err == nil { _ = cs.CoreV1().Pods(nsName).Delete(context.Background(), p.Name, metav1.DeleteOptions{}) return fmt.Errorf("expected create to be denied, but it succeeded") } if apierrors.IsAlreadyExists(err) { return fmt.Errorf("unexpected AlreadyExists: %v", err) } msg := err.Error() for _, substring := range substrings { if !strings.Contains(msg, substring) { return fmt.Errorf("expected error to contain %q, got: %s", substring, msg) } } return nil }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } createPodAndExpectAllowed := func(cs kubernetes.Interface, nsName string, pod *corev1.Pod) { EventuallyCreation(func() error { _, err := cs.CoreV1().Pods(nsName).Create(context.Background(), pod, metav1.CreateOptions{}) return err }).Should(Succeed()) } expectAuditEvent := func(cs kubernetes.Interface, nsName string, podName string, substrings ...string) { Eventually(func() error { events, err := cs.EventsV1().Events(nsName).List(context.Background(), metav1.ListOptions{}) if err != nil { return err } for _, event := range events.Items { if event.Regarding.Name != podName { continue } if event.Reason != "NamespaceRuleAudit" { continue } msg := event.Note matched := true for _, substring := range substrings { if !strings.Contains(msg, substring) { matched = false break } } if matched { return nil } } return fmt.Errorf("expected audit event for pod %q containing %q", podName, substrings) }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) } podWithScheduler := func(name string, schedulerName string) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: corev1.PodSpec{ SchedulerName: schedulerName, SecurityContext: nobodyPodSecurityContext(), Containers: []corev1.Container{ { Name: "c", Image: "registry.k8s.io/pause:3.9", ImagePullPolicy: corev1.PullIfNotPresent, SecurityContext: restrictedContainerSecurityContext(), }, }, }, } } JustBeforeEach(func() { tnt = newTenant() EventuallyCreation(func() error { tnt.ResourceVersion = "" return k8sClient.Create(context.TODO(), tnt) }).Should(Succeed()) TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval) }) JustAfterEach(func() { EventuallyDeletion(tnt) }) It("stores scheduler workload rules as independent status rule blocks", func() { ns := NewNamespace("", map[string]string{ meta.TenantLabel: tnt.GetName(), }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) expectNamespaceStatusRules(ns.GetName(), []expectedSchedulerStatusRule{ { action: rules.ActionTypeDeny, schedulers: []runtime.ExpressionMatch{ { Exact: []string{ "forbidden-scheduler", "legacy-scheduler", }, }, }, }, { action: rules.ActionTypeAudit, schedulers: []runtime.ExpressionMatch{ { Exact: []string{ "audited-scheduler", }, }, }, }, }) }) It("stores namespace-selector matched scheduler rules as additional status rule blocks", func() { ns := NewNamespace("", map[string]string{ "allow-forbidden-scheduler": "true", meta.TenantLabel: tnt.GetName(), }) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) expectNamespaceStatusRules(ns.GetName(), []expectedSchedulerStatusRule{ { action: rules.ActionTypeDeny, schedulers: []runtime.ExpressionMatch{ { Exact: []string{ "forbidden-scheduler", "legacy-scheduler", }, }, }, }, { action: rules.ActionTypeAudit, schedulers: []runtime.ExpressionMatch{ { Exact: []string{ "audited-scheduler", }, }, }, }, { action: rules.ActionTypeAllow, schedulers: []runtime.ExpressionMatch{ { Exact: []string{ "forbidden-scheduler", }, }, }, }, }) }) It("ignores pods without an explicit schedulerName", func() { ns := NewNamespace("", map[string]string{ meta.TenantLabel: tnt.GetName(), }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) createPodAndExpectAllowed(cs, ns.Name, podWithScheduler("empty-scheduler-ignored", "")) }) It("denies pods using an exact denied schedulerName", func() { ns := NewNamespace("", map[string]string{ meta.TenantLabel: tnt.GetName(), }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) createPodAndExpectDenied(cs, ns.Name, podWithScheduler("forbidden-denied", "forbidden-scheduler"), "forbidden-scheduler", "spec.schedulerName", "denied", ) }) It("denies pods using another schedulerName from the same exact list", func() { ns := NewNamespace("", map[string]string{ meta.TenantLabel: tnt.GetName(), }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) createPodAndExpectDenied(cs, ns.Name, podWithScheduler("legacy-denied", "legacy-scheduler"), "legacy-scheduler", "spec.schedulerName", "denied", ) }) It("allows pods when no scheduler rule matches them", func() { ns := NewNamespace("", map[string]string{ meta.TenantLabel: tnt.GetName(), }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) createPodAndExpectAllowed(cs, ns.Name, podWithScheduler("unmatched-allowed", "neutral-scheduler")) }) It("audits matching schedulerName rules by allowing admission and emitting an event", func() { ns := NewNamespace("", map[string]string{ meta.TenantLabel: tnt.GetName(), }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) pod := podWithScheduler("scheduler-audited", "audited-scheduler") createPodAndExpectAllowed(cs, ns.Name, pod) expectAuditEvent(clusterAdminClient(), ns.Name, pod.Name, `scheduler "audited-scheduler"`, "spec.schedulerName", "matched audit namespace rule", ) }) It("allows a previously denied schedulerName when a later namespace-selected allow rule matches", func() { ns := NewNamespace("", map[string]string{ "allow-forbidden-scheduler": "true", meta.TenantLabel: tnt.GetName(), }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) createPodAndExpectAllowed(cs, ns.Name, podWithScheduler("forbidden-allowed", "forbidden-scheduler")) }) It("denies an audited schedulerName when a later namespace-selected deny rule matches", func() { ns := NewNamespace("", map[string]string{ "deny-audited-scheduler": "true", meta.TenantLabel: tnt.GetName(), }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) createPodAndExpectDenied(cs, ns.Name, podWithScheduler("audited-denied", "audited-scheduler"), "audited-scheduler", "spec.schedulerName", "denied", ) }) It("allows schedulerName values matching a namespace-selected regex allow rule", func() { ns := NewNamespace("", map[string]string{ "allow-team-scheduler": "true", meta.TenantLabel: tnt.GetName(), }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) createPodAndExpectAllowed(cs, ns.Name, podWithScheduler("team-scheduler-allowed", "team-solar")) }) It("denies schedulerName values that do not match a namespace-selected regex allow rule", func() { ns := NewNamespace("", map[string]string{ "allow-team-scheduler": "true", meta.TenantLabel: tnt.GetName(), }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) createPodAndExpectDenied(cs, ns.Name, podWithScheduler("team-scheduler-denied", "team_Solar"), "team_Solar", "spec.schedulerName", "not allowed", ) }) It("denies schedulerName values matching a namespace-selected regex deny rule", func() { ns := NewNamespace("", map[string]string{ "deny-team-scheduler": "true", meta.TenantLabel: tnt.GetName(), }) cs := ownerClient(tnt.Spec.Owners[0].UserSpec) NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed()) NamespaceIsPartOfTenant(tnt, ns).Should(Succeed()) createPodAndExpectDenied(cs, ns.Name, podWithScheduler("team-blocked-denied", "team-blocked-solar"), "team-blocked-solar", "spec.schedulerName", "denied", ) }) })