feat: add scheduler enforcement rule (#1971)

* fix(controller): decode old object for delete requests

Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com>

* chore: modernize golang

Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com>

* chore: modernize golang

Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com>

* chore: modernize golang

Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com>

* fix: preserve ca-bundles injected from external providers

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: abstract ruling

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: migrate events api

* feat: migrate events api

* feat: migrate events api

* feat: migrate events api

* feat: migrate events  api

* feat: migrate events api

* feat: migrate events api

* feat: migrate events api

* feat: migrate events api

* feat: migrate events api

---------

Signed-off-by: Oliver Bähler <oliverbaehler@hotmail.com>
Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

pkg/runtime/configuration/client.go

@@ -54,7 +54,7 @@ func DefaultCapsuleConfiguration() capsulev1beta2.CapsuleConfigurationSpec {
 	}
 }
 
-func NewCapsuleConfiguration(ctx context.Context, c client.Client, rest *rest.Config, name string) Configuration {
+func NewCapsuleConfiguration(ctx context.Context, c client.Client, reader client.Reader, rest *rest.Config, name string) Configuration {
 	return &capsuleConfiguration{
 		client: c,
 		rest:   rest,
@@ -62,7 +62,7 @@ func NewCapsuleConfiguration(ctx context.Context, c client.Client, rest *rest.Co
 			cfg := &capsulev1beta2.CapsuleConfiguration{}
 			key := types.NamespacedName{Name: name}
 
-			if err := c.Get(ctx, key, cfg); err == nil {
+			if err := reader.Get(ctx, key, cfg); err == nil {
 				return cfg
 			} else if !apierrors.IsNotFound(err) {
 				panic(errors.Wrap(err, "cannot retrieve Capsule configuration with name "+name))
@@ -86,7 +86,7 @@ func NewCapsuleConfiguration(ctx context.Context, c client.Client, rest *rest.Co
 
 			if err := c.Create(ctx, cfg); err != nil {
 				if apierrors.IsAlreadyExists(err) {
-					if err := c.Get(ctx, key, cfg); err != nil {
+					if err := reader.Get(ctx, key, cfg); err != nil {
 						panic(errors.Wrap(err, "configuration created concurrently but cannot be retrieved"))
 					}
 
@@ -232,6 +232,10 @@ func (c *capsuleConfiguration) Admission() capsulev1beta2.DynamicAdmission {
 	return c.retrievalFn().Spec.Admission
 }
 
+func (c *capsuleConfiguration) Events() capsulev1beta2.EventsConfiguration {
+	return c.retrievalFn().Spec.Events
+}
+
 func (c *capsuleConfiguration) RBAC() *capsulev1beta2.RBACConfiguration {
 	return c.retrievalFn().Spec.RBAC
 }

pkg/runtime/configuration/configuration.go

@@ -43,6 +43,7 @@ type Configuration interface {
 	ServiceAccountClientProperties() capsulev1beta2.ServiceAccountClient
 	ServiceAccountClient(context.Context) (*rest.Config, error)
 	Admission() capsulev1beta2.DynamicAdmission
+	Events() capsulev1beta2.EventsConfiguration
 	RBAC() *capsulev1beta2.RBACConfiguration
 	CacheInvalidation() metav1.Duration
 }

pkg/runtime/events/actions.go

@@ -11,4 +11,5 @@ const (
 
 	ActionMutated          string = "Mutated"
 	ActionValidationDenied string = "ValidationDenied"
+	ActionRuleAudit        string = "RuleAudit"
 )

pkg/runtime/events/event_types.go

@@ -0,0 +1,97 @@
+// Copyright 2020-2026 Project Capsule Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package events
+
+import (
+	"context"
+	"maps"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+	capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
+	"github.com/projectcapsule/capsule/pkg/api/meta"
+)
+
+type LabeledEvent struct {
+	recorder *EventRecorder
+
+	regarding runtime.Object
+	related   runtime.Object
+
+	eventType string
+	reason    string
+	action    string
+	note      string
+
+	labels      map[string]string
+	annotations map[string]string
+}
+
+func (r *EventRecorder) LabeledEvent(
+	regarding runtime.Object,
+	eventType string,
+	reason string,
+	action string,
+	note string,
+) *LabeledEvent {
+	return &LabeledEvent{
+		recorder:    r,
+		regarding:   regarding,
+		eventType:   eventType,
+		reason:      reason,
+		action:      action,
+		note:        note,
+		labels:      map[string]string{},
+		annotations: map[string]string{},
+	}
+}
+
+func (e *LabeledEvent) Emit(ctx context.Context) {
+	if e == nil || e.recorder == nil {
+		return
+	}
+
+	e.recorder.emitLabeledEvent(ctx, e)
+}
+
+func (e *LabeledEvent) WithRelated(obj runtime.Object) *LabeledEvent {
+	e.related = obj
+
+	return e
+}
+
+func (e *LabeledEvent) WithLabels(labels map[string]string) *LabeledEvent {
+	maps.Copy(e.labels, labels)
+
+	return e
+}
+
+func (e *LabeledEvent) WithAnnotations(annotations map[string]string) *LabeledEvent {
+	maps.Copy(e.annotations, annotations)
+
+	return e
+}
+
+func (e *LabeledEvent) WithTenantLabel(tnt *capsulev1beta2.Tenant) *LabeledEvent {
+	if tnt == nil {
+		return e
+	}
+
+	e.labels[meta.NewTenantLabel] = tnt.Name
+
+	return e
+}
+
+func (e *LabeledEvent) WithRequestAnnotations(req admission.Request) *LabeledEvent {
+	if req.UID != "" {
+		e.annotations[meta.AuditRequestUID] = string(req.UID)
+	}
+
+	if req.UserInfo.Username != "" {
+		e.annotations[meta.AuditUsername] = req.UserInfo.Username
+	}
+
+	return e
+}

pkg/runtime/events/reasons.go

@@ -12,6 +12,10 @@ const (
 	ReasonForbiddenLabel string = "ForbiddenLabel"
 	// ForbiddenAnnotationReason used as reason string to deny forbidden annotations.
 	ReasonForbiddenAnnotation string = "ForbiddenAnnotation"
+	ReasonAdmissionFailure    string = "AdmissionFailed"
+
+	// RuleStatus.
+	ReasonNamespaceRuleAudit string = "NamespaceRuleAudit"
 
 	// Namespace.
 	ReasonNamespaceHijack string = "ReasonNamespacePatch"
@@ -34,11 +38,15 @@ const (
 	ReasonMissingDeviceClass     string = "MissingDeviceClass"
 	ReasonForbiddenDeviceClass   string = "ForbiddenDeviceClass"
 
+	// Nodes.
+	ReasonForbiddenNodeSelectorUpdate string = "ForbiddenNodeSelectorUpdate"
+
 	// Pods.
 	ReasonMissingFQCI                string = "MissingFQCI"
 	ReasonForbiddenContainerRegistry string = "ForbiddenContainerRegistry"
 	ReasonForbiddenPullPolicy        string = "ForbiddenPullPolicy"
-	ReasonForbiddenPodQoSClass       string = "ForbiddenPodQoSClass"
+	ReasonForbiddenPodQoSClass       string = "ForbiddenQoSClass"
+	ReasonForbiddenPodScheduler      string = "ForbiddenScheduler"
 
 	// Ingress.
 	ReasonWildcardDenied           string = "WildcardDenied"

pkg/runtime/events/recorder.go

@@ -0,0 +1,161 @@
+// Copyright 2020-2026 Project Capsule Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package events
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	eventsv1 "k8s.io/api/events/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	k8sevents "k8s.io/client-go/tools/events"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/projectcapsule/capsule/pkg/runtime/configuration"
+)
+
+const (
+	ReportingController = "controller.projectcapsule.dev"
+	ReportingInstance   = "capsule-admission"
+)
+
+type EventRecorder struct {
+	k8sevents.EventRecorder
+
+	client        client.Client
+	configuration configuration.Configuration
+	log           logr.Logger
+}
+
+func NewEventRecorder(
+	c client.Client,
+	log logr.Logger,
+	recorder k8sevents.EventRecorder,
+	configuration configuration.Configuration,
+) *EventRecorder {
+	return &EventRecorder{
+		EventRecorder: recorder,
+		client:        c,
+		log:           log.WithName("event-recorder"),
+		configuration: configuration,
+	}
+}
+
+func (r *EventRecorder) emitLabeledEvent(
+	ctx context.Context,
+	e *LabeledEvent,
+) {
+	if r == nil {
+		return
+	}
+
+	if r.client == nil {
+		r.log.Error(nil, "cannot emit labeled event: client is nil")
+
+		return
+	}
+
+	if e == nil {
+		r.log.Error(nil, "cannot emit labeled event: event is nil")
+
+		return
+	}
+
+	if e.reason == "" {
+		r.log.Error(nil, "cannot emit labeled event: reason is empty")
+
+		return
+	}
+
+	if e.action == "" {
+		r.log.Error(nil, "cannot emit labeled event: action is empty")
+
+		return
+	}
+
+	regardingRef, metaObj, err := objectReference(e.regarding)
+	if err != nil {
+		r.log.Error(err, "cannot emit labeled event: build regarding reference")
+
+		return
+	}
+
+	namespace := metaObj.GetNamespace()
+	if namespace == "" {
+		namespace = r.configuration.Events().ClusterEventNamespace
+	}
+
+	if namespace == "" {
+		r.log.Error(nil, "cannot emit labeled event: namespace is empty")
+
+		return
+	}
+
+	event := &eventsv1.Event{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: metaObj.GetName(),
+			Namespace:    namespace,
+			Labels:       e.labels,
+			Annotations:  e.annotations,
+		},
+		EventTime:           metav1.MicroTime{Time: time.Now()},
+		ReportingController: ReportingController,
+		ReportingInstance:   ReportingInstance,
+		Action:              e.action,
+		Reason:              e.reason,
+		Regarding:           regardingRef,
+		Note:                e.note,
+		Type:                e.eventType,
+	}
+
+	if e.related != nil {
+		relatedRef, _, err := objectReference(e.related)
+		if err != nil {
+			r.log.Error(err, "cannot emit labeled event: build related reference")
+
+			return
+		}
+
+		event.Related = &relatedRef
+	}
+
+	if err := r.client.Create(ctx, event); err != nil {
+		r.log.Error(
+			err,
+			"cannot emit labeled event",
+			"reason", e.reason,
+			"action", e.action,
+			"type", e.eventType,
+			"regarding", regardingRef.Name,
+			"namespace", namespace,
+		)
+
+		return
+	}
+}
+
+func objectReference(obj runtime.Object) (corev1.ObjectReference, metav1.Object, error) {
+	if obj == nil {
+		return corev1.ObjectReference{}, nil, fmt.Errorf("object is nil")
+	}
+
+	metaObj, ok := obj.(metav1.Object)
+	if !ok {
+		return corev1.ObjectReference{}, nil, fmt.Errorf("%T does not implement metav1.Object", obj)
+	}
+
+	gvk := obj.GetObjectKind().GroupVersionKind()
+
+	return corev1.ObjectReference{
+		APIVersion: gvk.GroupVersion().String(),
+		Kind:       gvk.Kind,
+		Namespace:  metaObj.GetNamespace(),
+		Name:       metaObj.GetName(),
+		UID:        metaObj.GetUID(),
+	}, metaObj, nil
+}

pkg/runtime/handlers/handlers.go

@@ -6,11 +6,11 @@ package handlers
 import (
 	"context"
 
-	"k8s.io/client-go/tools/events"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
+	"github.com/projectcapsule/capsule/pkg/runtime/events"
 	"github.com/projectcapsule/capsule/pkg/users"
 )
 

pkg/runtime/handlers/in_capsule_groups.go

@@ -7,11 +7,11 @@ package handlers
 import (
 	"context"
 
-	"k8s.io/client-go/tools/events"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	"github.com/projectcapsule/capsule/pkg/runtime/configuration"
+	"github.com/projectcapsule/capsule/pkg/runtime/events"
 	"github.com/projectcapsule/capsule/pkg/users"
 )
 

pkg/runtime/handlers/is_not_privileged.go

@@ -6,11 +6,11 @@ package handlers
 import (
 	"context"
 
-	"k8s.io/client-go/tools/events"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	"github.com/projectcapsule/capsule/pkg/runtime/configuration"
+	"github.com/projectcapsule/capsule/pkg/runtime/events"
 	"github.com/projectcapsule/capsule/pkg/users"
 )
 

pkg/runtime/handlers/typed_tenant_object.go

@@ -7,11 +7,11 @@ package handlers
 import (
 	"context"
 
-	"k8s.io/client-go/tools/events"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
+	"github.com/projectcapsule/capsule/pkg/runtime/events"
 	"github.com/projectcapsule/capsule/pkg/tenant"
 )
 

pkg/runtime/handlers/typed_tenant_ruleset.go

@@ -10,13 +10,13 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/tools/events"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
 	"github.com/projectcapsule/capsule/pkg/api/meta"
 	"github.com/projectcapsule/capsule/pkg/api/rules"
+	"github.com/projectcapsule/capsule/pkg/runtime/events"
 	"github.com/projectcapsule/capsule/pkg/tenant"
 )
 
@@ -187,7 +187,7 @@ func (h *TypedTenantWithRulesetHandler[T]) resolveTenant(
 // If not yet present, try to calculate it.
 func (h *TypedTenantWithRulesetHandler[T]) resolveRuleset(
 	ctx context.Context,
-	c client.Reader,
+	c client.Client,
 	req admission.Request,
 	namespace string,
 	tnt *capsulev1beta2.Tenant,
@@ -209,5 +209,5 @@ func (h *TypedTenantWithRulesetHandler[T]) resolveRuleset(
 		return nil, err
 	}
 
-	return tenant.BuildNamespaceRuleBodyStatus(ctx, c, ns, tnt)
+	return tenant.BuildNamespaceRuleBodyStatus(c.Scheme(), ns, tnt)
 }

pkg/runtime/handlers/typed_tenant_user_object.go

@@ -7,12 +7,12 @@ package handlers
 import (
 	"context"
 
-	"k8s.io/client-go/tools/events"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
 	"github.com/projectcapsule/capsule/pkg/runtime/configuration"
+	"github.com/projectcapsule/capsule/pkg/runtime/events"
 	"github.com/projectcapsule/capsule/pkg/tenant"
 	"github.com/projectcapsule/capsule/pkg/users"
 )