feat: add runtimeclass control

Signed-off-by: Oliver Baehler <oliver.baehler@hotmail.com>

Makefile

@@ -257,7 +257,7 @@ e2e-build/%:
 		capsule \
 		./charts/capsule
 
-e2e-exec:
+e2e-exec: ginkgo
 	$(GINKGO) -v -tags e2e ./e2e
 
 e2e-destroy:

api/v1beta2/tenant_types.go

@@ -35,6 +35,8 @@ type TenantSpec struct {
 	AdditionalRoleBindings []api.AdditionalRoleBindingsSpec `json:"additionalRoleBindings,omitempty"`
 	// Specify the allowed values for the imagePullPolicies option in Pod resources. Capsule assures that all Pod resources created in the Tenant can use only one of the allowed policy. Optional.
 	ImagePullPolicies []api.ImagePullPolicySpec `json:"imagePullPolicies,omitempty"`
+	// Specifies the allowed RuntimeClasses assigned to the Tenant. Capsule assures that all Pods resources created in the Tenant can use only one of the allowed RuntimeClasses. Optional.
+	RuntimeClasses *api.SelectorAllowedListSpec `json:"runtimeClasses,omitempty"`
 	// Specifies the allowed priorityClasses assigned to the Tenant. Capsule assures that all Pods resources created in the Tenant can use only one of the allowed PriorityClasses. Optional.
 	PriorityClasses *api.SelectorAllowedListSpec `json:"priorityClasses,omitempty"`
 	// Toggling the Tenant resources cordoning, when enable resources cannot be deleted.

api/v1beta2/zz_generated.deepcopy.go

@@ -749,6 +749,11 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) {
 		*out = make([]api.ImagePullPolicySpec, len(*in))
 		copy(*out, *in)
 	}
+	if in.RuntimeClasses != nil {
+		in, out := &in.RuntimeClasses, &out.RuntimeClasses
+		*out = new(api.SelectorAllowedListSpec)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.PriorityClasses != nil {
 		in, out := &in.PriorityClasses, &out.PriorityClasses
 		*out = new(api.SelectorAllowedListSpec)

config/crd/bases/capsule.clastix.io_tenants.yaml

@@ -3003,6 +3003,59 @@ spec:
                     - Namespace
                     type: string
                 type: object
+              runtimeClasses:
+                description: Specifies the allowed RuntimeClasses assigned to the
+                  Tenant. Capsule assures that all Pods resources created in the Tenant
+                  can use only one of the allowed RuntimeClasses. Optional.
+                properties:
+                  allowed:
+                    items:
+                      type: string
+                    type: array
+                  allowedRegex:
+                    type: string
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements.
+                      The requirements are ANDed.
+                    items:
+                      description: A label selector requirement is a selector that
+                        contains values, a key, and an operator that relates the key
+                        and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies
+                            to.
+                          type: string
+                        operator:
+                          description: operator represents a key's relationship to
+                            a set of values. Valid operators are In, NotIn, Exists
+                            and DoesNotExist.
+                          type: string
+                        values:
+                          description: values is an array of string values. If the
+                            operator is In or NotIn, the values array must be non-empty.
+                            If the operator is Exists or DoesNotExist, the values
+                            array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: matchLabels is a map of {key,value} pairs. A single
+                      {key,value} in the matchLabels map is equivalent to an element
+                      of matchExpressions, whose key field is "key", the operator
+                      is "In", and the values array contains only "value". The requirements
+                      are ANDed.
+                    type: object
+                type: object
+                x-kubernetes-map-type: atomic
               serviceOptions:
                 description: Specifies options for the Service, such as additional
                   metadata or block of certain type of Services. Optional.

config/install.yaml

@@ -2536,6 +2536,43 @@ spec:
                     - Namespace
                     type: string
                 type: object
+              runtimeClasses:
+                description: Specifies the allowed RuntimeClasses assigned to the Tenant. Capsule assures that all Pods resources created in the Tenant can use only one of the allowed RuntimeClasses. Optional.
+                properties:
+                  allowed:
+                    items:
+                      type: string
+                    type: array
+                  allowedRegex:
+                    type: string
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                    items:
+                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies to.
+                          type: string
+                        operator:
+                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+                          type: string
+                        values:
+                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+                          items:
+                            type: string
+                          type: array
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+                    type: object
+                type: object
+                x-kubernetes-map-type: atomic
               serviceOptions:
                 description: Specifies options for the Service, such as additional metadata or block of certain type of Services. Optional.
                 properties:

docs/content/general/crds-apis.md

@@ -2984,6 +2984,13 @@ TenantSpec defines the desired state of Tenant.
           Specifies a list of ResourceQuota resources assigned to the Tenant. The assigned values are inherited by any namespace created in the Tenant. The Capsule operator aggregates ResourceQuota at Tenant level, so that the hard quota is never crossed for the given Tenant. This permits the Tenant owner to consume resources in the Tenant regardless of the namespace. Optional.<br/>
         </td>
         <td>false</td>
+      </tr><tr>
+        <td><b><a href="#tenantspecruntimeclasses">runtimeClasses</a></b></td>
+        <td>object</td>
+        <td>
+          Specifies the allowed RuntimeClasses assigned to the Tenant. Capsule assures that all Pods resources created in the Tenant can use only one of the allowed RuntimeClasses. Optional.<br/>
+        </td>
+        <td>false</td>
       </tr><tr>
         <td><b><a href="#tenantspecserviceoptions-1">serviceOptions</a></b></td>
         <td>object</td>
@@ -4612,6 +4619,93 @@ A scoped-resource selector requirement is a selector that contains values, a sco
 </table>
 
 
+### Tenant.spec.runtimeClasses
+
+
+
+Specifies the allowed RuntimeClasses assigned to the Tenant. Capsule assures that all Pods resources created in the Tenant can use only one of the allowed RuntimeClasses. Optional.
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
+        <td><b>allowed</b></td>
+        <td>[]string</td>
+        <td>
+          <br/>
+        </td>
+        <td>false</td>
+      </tr><tr>
+        <td><b>allowedRegex</b></td>
+        <td>string</td>
+        <td>
+          <br/>
+        </td>
+        <td>false</td>
+      </tr><tr>
+        <td><b><a href="#tenantspecruntimeclassesmatchexpressionsindex">matchExpressions</a></b></td>
+        <td>[]object</td>
+        <td>
+          matchExpressions is a list of label selector requirements. The requirements are ANDed.<br/>
+        </td>
+        <td>false</td>
+      </tr><tr>
+        <td><b>matchLabels</b></td>
+        <td>map[string]string</td>
+        <td>
+          matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.<br/>
+        </td>
+        <td>false</td>
+      </tr></tbody>
+</table>
+
+
+### Tenant.spec.runtimeClasses.matchExpressions[index]
+
+
+
+A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
+        <td><b>key</b></td>
+        <td>string</td>
+        <td>
+          key is the label key that the selector applies to.<br/>
+        </td>
+        <td>true</td>
+      </tr><tr>
+        <td><b>operator</b></td>
+        <td>string</td>
+        <td>
+          operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.<br/>
+        </td>
+        <td>true</td>
+      </tr><tr>
+        <td><b>values</b></td>
+        <td>[]string</td>
+        <td>
+          values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.<br/>
+        </td>
+        <td>false</td>
+      </tr></tbody>
+</table>
+
+
 ### Tenant.spec.serviceOptions
 
 

docs/content/general/tutorial.md

@@ -838,6 +838,42 @@ With the said Tenant specification, Alice can create a Pod resource if `spec.pri
 
 If a Pod is going to use a non-allowed _Priority Class_, it will be rejected by the Validation Webhook enforcing it.
 
+
+## Assign Pod Runtime Classes
+
+Pods can be assigned different runtime classes. With the assigned runtime you can control Container Runtime Interface (CRI) is used for each pod. See [Kubernetes documentation](https://kubernetes.io/docs/concepts/containers/runtime-class/). 
+
+To prevent misuses of Pod Runtime Classes, Bill, the cluster admin, can enforce the allowed Pod Runtime Class at tenant level:
+
+```yaml
+kubectl apply -f - << EOF
+apiVersion: capsule.clastix.io/v1beta2
+kind: Tenant
+metadata:
+  name: oil
+spec:
+  owners:
+  - name: alice
+    kind: User
+  runtimeClasses:
+    allowed:
+    - legacy
+    allowedRegex: "^hardened-.*$"
+    selector:
+      matchLabels:
+        env: "production"
+EOF
+```
+
+With the said Tenant specification, Alice can create a Pod resource if `spec.RuntimeClasses` equals to:
+
+- `legacy`
+- `hardened-crio` or `hardened-containerd`, since these compile the allowed regex.
+- Any RuntimeClass which has the label `env` with the value `production`
+
+If a Pod is going to use a non-allowed _Runtime Class_, it will be rejected by the Validation Webhook enforcing it.
+
+
 ## Assign Nodes Pool
 Bill, the cluster admin, can dedicate a pool of worker nodes to the `oil` tenant, to isolate the tenant applications from other noisy neighbors.
 

e2e/pod_priority_class_test.go

@@ -8,14 +8,15 @@ package e2e
 import (
 	"context"
 
+	capsulev1beta2 "github.com/clastix/capsule/api/v1beta2"
+	"github.com/clastix/capsule/pkg/api"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/scheduling/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	capsulev1beta2 "github.com/clastix/capsule/api/v1beta2"
-	"github.com/clastix/capsule/pkg/api"
+	"strconv"
+	"strings"
 )
 
 var _ = Describe("enforcing a Priority Class", func() {
@@ -35,6 +36,11 @@ var _ = Describe("enforcing a Priority Class", func() {
 					Exact: []string{"gold"},
 					Regex: "pc\\-\\w+",
 				},
+				Selector: metav1.LabelSelector{
+					MatchLabels: map[string]string{
+						"env": "customers",
+					},
+				},
 			},
 		},
 	}
@@ -155,4 +161,51 @@ var _ = Describe("enforcing a Priority Class", func() {
 			Expect(k8sClient.Delete(context.TODO(), class)).Should(Succeed())
 		}
 	})
+
+	It("should allow selector match", func() {
+		ns := NewNamespace("priority-selector-match")
+
+		NamespaceCreation(ns, tnt.Spec.Owners[0], defaultTimeoutInterval).Should(Succeed())
+
+		for i, pc := range []string{"customer-bronze", "customer-silver", "customer-gold"} {
+			priorityName := strings.Join([]string{pc, "-", strconv.Itoa(i)}, "")
+			class := &v1.PriorityClass{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: pc,
+					Labels: map[string]string{
+						"name": priorityName,
+						"env":  "customers",
+					},
+				},
+				Description: "fake PriorityClass for e2e",
+				Value:       int32(10000 * (i + 2)),
+			}
+			Expect(k8sClient.Create(context.TODO(), class)).Should(Succeed())
+
+			pod := &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: pc,
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "container",
+							Image: "quay.io/google-containers/pause-amd64:3.0",
+						},
+					},
+					PriorityClassName: class.GetName(),
+				},
+			}
+
+			cs := ownerClient(tnt.Spec.Owners[0])
+
+			EventuallyCreation(func() error {
+				_, err := cs.CoreV1().Pods(ns.GetName()).Create(context.Background(), pod, metav1.CreateOptions{})
+				return err
+			}).Should(Succeed())
+
+			Expect(k8sClient.Delete(context.TODO(), class)).Should(Succeed())
+		}
+	})
+
 })

e2e/pod_runtime_class_test.go

@@ -0,0 +1,222 @@
+//go:build e2e
+
+// Copyright 2020-2021 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+	"github.com/clastix/capsule/pkg/api"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	nodev1 "k8s.io/api/node/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"strconv"
+	"strings"
+
+	capsulev1beta2 "github.com/clastix/capsule/api/v1beta2"
+)
+
+var _ = Describe("enforcing a Runtime Class", func() {
+	tnt := &capsulev1beta2.Tenant{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "runtime-class",
+		},
+		Spec: capsulev1beta2.TenantSpec{
+			Owners: capsulev1beta2.OwnerListSpec{
+				{
+					Name: "george",
+					Kind: "User",
+				},
+			},
+			RuntimeClasses: &api.SelectorAllowedListSpec{
+				AllowedListSpec: api.AllowedListSpec{
+					Exact: []string{"legacy"},
+					Regex: "^hardened-.*$",
+				},
+				Selector: metav1.LabelSelector{
+					MatchLabels: map[string]string{
+						"env": "customers",
+					},
+				},
+			},
+		},
+	}
+
+	JustBeforeEach(func() {
+		EventuallyCreation(func() error {
+			tnt.ResourceVersion = ""
+			return k8sClient.Create(context.TODO(), tnt)
+		}).Should(Succeed())
+	})
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
+	})
+
+	It("should block non allowed Runtime Class", func() {
+		runtime := &nodev1.RuntimeClass{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "disallowed",
+			},
+			Handler: "custom-handler",
+		}
+		Expect(k8sClient.Create(context.TODO(), runtime)).Should(Succeed())
+
+		defer func() {
+			Expect(k8sClient.Delete(context.TODO(), runtime)).Should(Succeed())
+		}()
+
+		ns := NewNamespace("rt-disallow")
+		NamespaceCreation(ns, tnt.Spec.Owners[0], defaultTimeoutInterval).Should(Succeed())
+		runtimeName := "disallowed"
+		pod := &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "container",
+			},
+			Spec: corev1.PodSpec{
+				Containers: []corev1.Container{
+					{
+						Name:  "container",
+						Image: "quay.io/google-containers/pause-amd64:3.0",
+					},
+				},
+				RuntimeClassName: &runtimeName,
+			},
+		}
+
+		cs := ownerClient(tnt.Spec.Owners[0])
+		EventuallyCreation(func() error {
+			_, err := cs.CoreV1().Pods(ns.GetName()).Create(context.Background(), pod, metav1.CreateOptions{})
+			return err
+		}).ShouldNot(Succeed())
+	})
+
+	It("should allow exact match", func() {
+		runtime := &nodev1.RuntimeClass{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "legacy",
+			},
+			Handler: "custom-handler",
+		}
+		Expect(k8sClient.Create(context.TODO(), runtime)).Should(Succeed())
+
+		defer func() {
+			Expect(k8sClient.Delete(context.TODO(), runtime)).Should(Succeed())
+		}()
+
+		ns := NewNamespace("rt-exact-match")
+		NamespaceCreation(ns, tnt.Spec.Owners[0], defaultTimeoutInterval).Should(Succeed())
+		runtimeName := "legacy"
+		pod := &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "container",
+			},
+			Spec: corev1.PodSpec{
+				Containers: []corev1.Container{
+					{
+						Name:  "container",
+						Image: "quay.io/google-containers/pause-amd64:3.0",
+					},
+				},
+				RuntimeClassName: &runtimeName,
+			},
+		}
+
+		cs := ownerClient(tnt.Spec.Owners[0])
+		EventuallyCreation(func() error {
+			_, err := cs.CoreV1().Pods(ns.GetName()).Create(context.Background(), pod, metav1.CreateOptions{})
+			return err
+		}).Should(Succeed())
+	})
+
+	It("should allow regex match", func() {
+		ns := NewNamespace("rc-regex-match")
+
+		NamespaceCreation(ns, tnt.Spec.Owners[0], defaultTimeoutInterval).Should(Succeed())
+
+		for i, rt := range []string{"hardened-crio", "hardened-containerd", "hardened-dockerd"} {
+			runtimeName := strings.Join([]string{rt, "-", strconv.Itoa(i)}, "")
+			runtime := &nodev1.RuntimeClass{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: runtimeName,
+				},
+				Handler: "custom-handler",
+			}
+
+			Expect(k8sClient.Create(context.TODO(), runtime)).Should(Succeed())
+
+			pod := &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: rt,
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "container",
+							Image: "quay.io/google-containers/pause-amd64:3.0",
+						},
+					},
+					RuntimeClassName: &runtimeName,
+				},
+			}
+
+			cs := ownerClient(tnt.Spec.Owners[0])
+
+			EventuallyCreation(func() error {
+				_, err := cs.CoreV1().Pods(ns.GetName()).Create(context.Background(), pod, metav1.CreateOptions{})
+				return err
+			}).Should(Succeed())
+
+			Expect(k8sClient.Delete(context.TODO(), runtime)).Should(Succeed())
+		}
+	})
+
+	It("should allow selector match", func() {
+		ns := NewNamespace("rc-selector-match")
+
+		NamespaceCreation(ns, tnt.Spec.Owners[0], defaultTimeoutInterval).Should(Succeed())
+
+		for i, rt := range []string{"customer-containerd", "customer-crio", "customer-dockerd"} {
+			runtimeName := strings.Join([]string{rt, "-", strconv.Itoa(i)}, "")
+			runtime := &nodev1.RuntimeClass{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: runtimeName,
+					Labels: map[string]string{
+						"name": runtimeName,
+						"env":  "customers",
+					},
+				},
+				Handler: "custom-handler",
+			}
+
+			Expect(k8sClient.Create(context.TODO(), runtime)).Should(Succeed())
+
+			pod := &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: rt,
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "container",
+							Image: "quay.io/google-containers/pause-amd64:3.0",
+						},
+					},
+					RuntimeClassName: &runtimeName,
+				},
+			}
+
+			cs := ownerClient(tnt.Spec.Owners[0])
+
+			EventuallyCreation(func() error {
+				_, err := cs.CoreV1().Pods(ns.GetName()).Create(context.Background(), pod, metav1.CreateOptions{})
+				return err
+			}).Should(Succeed())
+
+			Expect(k8sClient.Delete(context.TODO(), runtime)).Should(Succeed())
+		}
+	})
+
+})

main.go

@@ -235,7 +235,7 @@ func main() {
 	// webhooks: the order matters, don't change it and just append
 	webhooksList := append(
 		make([]webhook.Webhook, 0),
-		route.Pod(pod.ImagePullPolicy(), pod.ContainerRegistry(), pod.PriorityClass()),
+		route.Pod(pod.ImagePullPolicy(), pod.ContainerRegistry(), pod.PriorityClass(), pod.RuntimeClass()),
 		route.Namespace(utils.InCapsuleGroups(cfg, namespacewebhook.PatchHandler(), namespacewebhook.QuotaHandler(), namespacewebhook.FreezeHandler(cfg), namespacewebhook.PrefixHandler(cfg), namespacewebhook.UserMetadataHandler())),
 		route.Ingress(ingress.Class(cfg), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()),
 		route.PVC(pvc.Handler()),

pkg/webhook/pod/priorityclass_errors.go

@@ -5,9 +5,9 @@ package pod
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/clastix/capsule/pkg/api"
+	"github.com/clastix/capsule/pkg/webhook/utils"
 )
 
 type podPriorityClassForbiddenError struct {
@@ -25,21 +25,5 @@ func NewPodPriorityClassForbidden(priorityClassName string, spec api.SelectorAll
 func (f podPriorityClassForbiddenError) Error() (err string) {
 	err = fmt.Sprintf("Pod Priorioty Class %s is forbidden for the current Tenant: ", f.priorityClassName)
 
-	var extra []string
-
-	if len(f.spec.Exact) > 0 {
-		extra = append(extra, fmt.Sprintf("use one from the following list (%s)", strings.Join(f.spec.Exact, ", ")))
-	}
-
-	if len(f.spec.Regex) > 0 {
-		extra = append(extra, fmt.Sprintf(" use one matching the following regex (%s)", f.spec.Regex))
-	}
-
-	if len(f.spec.Selector.MatchLabels) > 0 || len(f.spec.Selector.MatchExpressions) > 0 {
-		extra = append(extra, ", or matching the label selector defined in the Tenant")
-	}
-
-	err += strings.Join(extra, " or ")
-
-	return
+	return utils.AllowedValuesErrorMessage(f.spec, err)
 }

pkg/webhook/pod/runtimeclass.go

@@ -0,0 +1,108 @@
+// Copyright 2020-2021 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package pod
+
+import (
+	"context"
+	"net/http"
+
+	corev1 "k8s.io/api/core/v1"
+	nodev1 "k8s.io/api/node/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+	capsulev1beta2 "github.com/clastix/capsule/api/v1beta2"
+	capsulewebhook "github.com/clastix/capsule/pkg/webhook"
+	"github.com/clastix/capsule/pkg/webhook/utils"
+)
+
+type runtimeClass struct{}
+
+func RuntimeClass() capsulewebhook.Handler {
+	return &runtimeClass{}
+}
+
+func (h *runtimeClass) class(ctx context.Context, c client.Client, name string) (client.Object, error) {
+	if len(name) == 0 {
+		return nil, nil
+	}
+
+	obj := &nodev1.RuntimeClass{}
+	if err := c.Get(ctx, types.NamespacedName{Name: name}, obj); err != nil {
+		return nil, err
+	}
+
+	return obj, nil
+}
+
+func (h *runtimeClass) OnCreate(c client.Client, decoder *admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func {
+	return func(ctx context.Context, req admission.Request) *admission.Response {
+		return h.validate(ctx, c, decoder, recorder, req)
+	}
+}
+
+func (h *runtimeClass) OnDelete(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func {
+	return func(ctx context.Context, req admission.Request) *admission.Response {
+		return nil
+	}
+}
+
+func (h *runtimeClass) OnUpdate(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func {
+	return func(ctx context.Context, req admission.Request) *admission.Response {
+		return nil
+	}
+}
+
+func (h *runtimeClass) validate(ctx context.Context, c client.Client, decoder *admission.Decoder, recorder record.EventRecorder, req admission.Request) *admission.Response {
+	pod := &corev1.Pod{}
+	if err := decoder.Decode(req, pod); err != nil {
+		return utils.ErroredResponse(err)
+	}
+
+	tntList := &capsulev1beta2.TenantList{}
+
+	if err := c.List(ctx, tntList, client.MatchingFieldsSelector{
+		Selector: fields.OneTermEqualSelector(".status.namespaces", pod.Namespace),
+	}); err != nil {
+		return utils.ErroredResponse(err)
+	}
+
+	if len(tntList.Items) == 0 {
+		return nil
+	}
+
+	allowed := tntList.Items[0].Spec.RuntimeClasses
+
+	runtimeClassName := ""
+	if pod.Spec.RuntimeClassName != nil {
+		runtimeClassName = *pod.Spec.RuntimeClassName
+	}
+
+	class, err := h.class(ctx, c, runtimeClassName)
+	if err != nil {
+		response := admission.Errored(http.StatusInternalServerError, err)
+
+		return &response
+	}
+
+	switch {
+	case allowed == nil:
+		// Enforcement is not in place, skipping it at all
+		return nil
+	case len(runtimeClassName) == 0:
+		// We don't have to force Pod to specify a RuntimeClass
+		return nil
+	case !allowed.ExactMatch(runtimeClassName) && !allowed.RegexMatch(runtimeClassName) && !allowed.SelectorMatch(class):
+		recorder.Eventf(&tntList.Items[0], corev1.EventTypeWarning, "ForbiddenRuntimeClass", "Pod %s/%s is using Runtime Class %s is forbidden for the current Tenant", pod.Namespace, pod.Name, runtimeClassName)
+
+		response := admission.Denied(NewPodRuntimeClassForbidden(runtimeClassName, *allowed).Error())
+
+		return &response
+	default:
+		return nil
+	}
+}

pkg/webhook/pod/runtimeclass_errors.go

@@ -0,0 +1,29 @@
+// Copyright 2020-2021 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package pod
+
+import (
+	"fmt"
+
+	"github.com/clastix/capsule/pkg/api"
+	"github.com/clastix/capsule/pkg/webhook/utils"
+)
+
+type podRuntimeClassForbiddenError struct {
+	runtimeClassName string
+	spec             api.SelectorAllowedListSpec
+}
+
+func NewPodRuntimeClassForbidden(runtimeClassName string, spec api.SelectorAllowedListSpec) error {
+	return &podRuntimeClassForbiddenError{
+		runtimeClassName: runtimeClassName,
+		spec:             spec,
+	}
+}
+
+func (f podRuntimeClassForbiddenError) Error() (err string) {
+	err = fmt.Sprintf("Pod Runtime Class %s is forbidden for the current Tenant: ", f.runtimeClassName)
+
+	return utils.AllowedValuesErrorMessage(f.spec, err)
+}

pkg/webhook/utils/error.go

@@ -4,9 +4,13 @@
 package utils
 
 import (
+	"fmt"
 	"net/http"
+	"strings"
 
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+	"github.com/clastix/capsule/pkg/api"
 )
 
 func ErroredResponse(err error) *admission.Response {
@@ -14,3 +18,22 @@ func ErroredResponse(err error) *admission.Response {
 
 	return &response
 }
+
+func AllowedValuesErrorMessage(allowed api.SelectorAllowedListSpec, err string) string {
+	var extra []string
+	if len(allowed.Exact) > 0 {
+		extra = append(extra, fmt.Sprintf("use one from the following list (%s)", strings.Join(allowed.Exact, ", ")))
+	}
+
+	if len(allowed.Regex) > 0 {
+		extra = append(extra, fmt.Sprintf(" use one matching the following regex (%s)", allowed.Regex))
+	}
+
+	if len(allowed.Selector.MatchLabels) > 0 || len(allowed.Selector.MatchExpressions) > 0 {
+		extra = append(extra, ", or matching the label selector defined in the Tenant")
+	}
+
+	err += strings.Join(extra, " or ")
+
+	return err
+}