capsule/e2e/tenant_container_registry_test.go

// Copyright 2020-2023 Project Capsule Authors.
// SPDX-License-Identifier: Apache-2.0

package e2e

import (
	"context"
	"encoding/json"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
	corev1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"sigs.k8s.io/controller-runtime/pkg/client"

	capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
	"github.com/projectcapsule/capsule/pkg/api"
	"github.com/projectcapsule/capsule/pkg/api/meta"
	"github.com/projectcapsule/capsule/pkg/api/rbac"
)

type Patch struct {
	Op    string `json:"op"`
	Path  string `json:"path"`
	Value string `json:"value"`
}

var _ = Describe("enforcing a Container Registry", Ordered, Label("tenant", "images", "registry"), func() {
	originConfig := &capsulev1beta2.CapsuleConfiguration{}

	tnt := &capsulev1beta2.Tenant{
		ObjectMeta: metav1.ObjectMeta{
			Name: "e2e-container-registry",
			Labels: map[string]string{
				"env": "e2e",
			},
		},
		Spec: capsulev1beta2.TenantSpec{
			Owners: rbac.OwnerListSpec{
				{
					CoreOwnerSpec: rbac.CoreOwnerSpec{
						UserSpec: rbac.UserSpec{
							Name: "e2e-container-registry",
							Kind: "User",
						},
					},
				},
			},
			ContainerRegistries: &api.AllowedListSpec{
				Exact: []string{"docker.io", "myregistry.azurecr.io"},
				Regex: `quay\.\w+`,
			},
		},
	}

	JustBeforeEach(func() {
		Expect(k8sClient.Get(context.Background(), client.ObjectKey{Name: defaultConfigurationName}, originConfig)).To(Succeed())

		EventuallyCreation(func() error {
			tnt.ResourceVersion = ""
			return k8sClient.Create(context.TODO(), tnt)
		}).Should(Succeed())

		TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval)
	})

	JustAfterEach(func() {
		EventuallyDeletion(tnt)
	})

	It("should add labels to Namespace", func() {
		ns := NewNamespace("", map[string]string{
			meta.TenantLabel: tnt.GetName(),
		})
		NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
		Eventually(func() (ok bool) {
			Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: ns.Name}, ns)).Should(Succeed())
			ok, _ = HaveKeyWithValue("capsule.clastix.io/allowed-registries", "docker.io,myregistry.azurecr.io").Match(ns.Annotations)
			if !ok {
				return
			}
			ok, _ = HaveKeyWithValue("capsule.clastix.io/allowed-registries-regexp", `quay\.\w+`).Match(ns.Annotations)
			if !ok {
				return
			}
			return true
		}, defaultTimeoutInterval, defaultPollInterval).Should(BeTrue())
	})

	It("should deny running a gcr.io container", func() {
		ns := NewNamespace("", map[string]string{
			meta.TenantLabel: tnt.GetName(),
		})

		pod := &corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "container",
			},
			Spec: corev1.PodSpec{
				SecurityContext: nobodyPodSecurityContext(),
				Containers: []corev1.Container{
					{
						Name:            "container",
						Image:           "gcr.io/invalid/pause:3.10",
						SecurityContext: restrictedContainerSecurityContext(),
					},
				},
			},
		}

		cs := ownerClient(tnt.Spec.Owners[0].UserSpec)

		NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
		NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())

		EventuallyCreation(func() error {
			_, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{})

			return err
		}).ShouldNot(Succeed())
	})

	It("should allow using a registry only match", func() {
		ns := NewNamespace("", map[string]string{
			meta.TenantLabel: tnt.GetName(),
		})

		pod := &corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "container",
			},
			Spec: corev1.PodSpec{
				SecurityContext: nobodyPodSecurityContext(),
				Containers: []corev1.Container{
					{
						Name:            "container",
						Image:           "myregistry.azurecr.io/myapp:latest",
						SecurityContext: restrictedContainerSecurityContext(),
					},
				},
			},
		}

		cs := ownerClient(tnt.Spec.Owners[0].UserSpec)

		NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
		NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())

		EventuallyCreation(func() error {
			_, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{})

			return err
		}).Should(Succeed())

		By("verifying the image was correctly mutated", func() {
			created := &corev1.Pod{}
			Expect(k8sClient.Get(context.Background(), types.NamespacedName{
				Namespace: ns.Name,
				Name:      pod.Name,
			}, created)).To(Succeed())

			Expect(created.Spec.Containers).To(HaveLen(1))
			Expect(created.Spec.Containers[0].Image).To(Equal("myregistry.azurecr.io/myapp:latest"))
		})
	})

	It("should deny patching a not matching registry after applying with a matching (Container)", func() {
		ns := NewNamespace("", map[string]string{
			meta.TenantLabel: tnt.GetName(),
		})

		pod := &corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "container",
			},
			Spec: corev1.PodSpec{
				SecurityContext: nobodyPodSecurityContext(),
				Containers: []corev1.Container{
					{
						Name:            "container",
						Image:           "myregistry.azurecr.io/myapp:latest",
						SecurityContext: restrictedContainerSecurityContext(),
					},
				},
			},
		}

		cs := ownerClient(tnt.Spec.Owners[0].UserSpec)

		NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
		NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())

		EventuallyCreation(func() error {
			_, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{})

			return err
		}).Should(Succeed())

		By("verifying the image was correctly mutated", func() {
			created := &corev1.Pod{}
			Expect(k8sClient.Get(context.Background(), types.NamespacedName{
				Namespace: ns.Name,
				Name:      pod.Name,
			}, created)).To(Succeed())

			Expect(created.Spec.Containers).To(HaveLen(1))
			Expect(created.Spec.Containers[0].Image).To(Equal("myregistry.azurecr.io/myapp:latest"))
		})

		Eventually(func() error {
			payload := []Patch{{
				Op:    "replace",
				Path:  "/spec/containers/0/image",
				Value: "attacker/pause:3.10",
			}}
			payloadBytes, _ := json.Marshal(payload)
			_, err := cs.CoreV1().Pods(ns.GetName()).Patch(context.TODO(), pod.GetName(), types.JSONPatchType, payloadBytes, metav1.PatchOptions{})
			if err != nil {
				return err
			}
			return nil
		}).ShouldNot(Succeed())
	})

	It("should deny patching a not matching registry after applying with a matching (EphemeralContainer)", func() {
		ns := NewNamespace("", map[string]string{
			meta.TenantLabel: tnt.GetName(),
		})

		pod := &corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "container",
			},
			Spec: corev1.PodSpec{
				SecurityContext: nobodyPodSecurityContext(),
				Containers: []corev1.Container{
					{
						Name:            "container",
						Image:           "docker.io/library/busybox:1.36.1",
						SecurityContext: restrictedContainerSecurityContext(),
					},
				},
			},
		}

		cs := ownerClient(tnt.Spec.Owners[0].UserSpec)

		NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
		NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())

		role := &rbacv1.Role{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "ephemeralcontainers-editor",
				Namespace: ns.GetName(),
			},
			Rules: []rbacv1.PolicyRule{
				{
					APIGroups: []string{""}, // core API group
					Resources: []string{"pods/ephemeralcontainers"},
					Verbs:     []string{"update", "patch"},
				},
			},
		}

		rb := &rbacv1.RoleBinding{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "ephemeralcontainers-editor-binding",
				Namespace: ns.GetName(),
			},
			Subjects: []rbacv1.Subject{
				{
					Kind: rbacv1.UserKind,
					Name: tnt.Spec.Owners[0].Name,
				},
			},
			RoleRef: rbacv1.RoleRef{
				APIGroup: "rbac.authorization.k8s.io",
				Kind:     "Role",
				Name:     role.Name,
			},
		}

		// Create role and binding before test logic
		Expect(k8sClient.Create(context.TODO(), role)).To(Succeed())
		Expect(k8sClient.Create(context.TODO(), rb)).To(Succeed())

		EventuallyCreation(func() error {
			_, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{})

			return err
		}).Should(Succeed())

		Eventually(func() error {
			pod.Spec.EphemeralContainers = []corev1.EphemeralContainer{
				{
					EphemeralContainerCommon: corev1.EphemeralContainerCommon{
						Name:            "dbg",
						Image:           "attacker/pause:3.10",
						ImagePullPolicy: corev1.PullIfNotPresent,
					},
				},
			}

			_, err := cs.CoreV1().Pods(ns.Name).UpdateEphemeralContainers(context.Background(), pod.Name, pod, metav1.UpdateOptions{})
			if err != nil {
				return err
			}
			return nil
		}).ShouldNot(Succeed())
	})

	It("should deny patching a not matching registry after applying with a matching (initContainer)", func() {
		ns := NewNamespace("", map[string]string{
			meta.TenantLabel: tnt.GetName(),
		})

		pod := &corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "container",
			},
			Spec: corev1.PodSpec{
				SecurityContext: nobodyPodSecurityContext(),
				InitContainers: []corev1.Container{
					{
						Name:            "init",
						Image:           "myregistry.azurecr.io/myapp:latest",
						SecurityContext: restrictedContainerSecurityContext(),
					},
				},
				Containers: []corev1.Container{
					{
						Name:            "container",
						Image:           "myregistry.azurecr.io/myapp:latest",
						SecurityContext: restrictedContainerSecurityContext(),
					},
				},
			},
		}

		cs := ownerClient(tnt.Spec.Owners[0].UserSpec)

		NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
		NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())

		EventuallyCreation(func() error {
			_, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{})

			return err
		}).Should(Succeed())

		Eventually(func() error {
			payload := []Patch{{
				Op:    "replace",
				Path:  "/spec/initContainers/0/image",
				Value: "attacker/pause:3.10",
			}}
			payloadBytes, _ := json.Marshal(payload)
			_, err := cs.CoreV1().Pods(ns.GetName()).Patch(context.TODO(), pod.GetName(), types.JSONPatchType, payloadBytes, metav1.PatchOptions{})
			if err != nil {
				return err
			}
			return nil
		}).ShouldNot(Succeed())
	})

	It("should deny patching a not matching registry after applying with a matching (Container)", func() {
		ns := NewNamespace("", map[string]string{
			meta.TenantLabel: tnt.GetName(),
		})

		pod := &corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "container",
			},
			Spec: corev1.PodSpec{
				SecurityContext: nobodyPodSecurityContext(),
				Containers: []corev1.Container{
					{
						Name:            "container",
						Image:           "myregistry.azurecr.io/myapp:latest",
						SecurityContext: restrictedContainerSecurityContext(),
					},
				},
			},
		}

		cs := ownerClient(tnt.Spec.Owners[0].UserSpec)

		NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
		NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())

		EventuallyCreation(func() error {
			_, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{})

			return err
		}).Should(Succeed())

		Eventually(func() error {
			payload := []Patch{{
				Op:    "replace",
				Path:  "/spec/initContainers/0/image",
				Value: "attacker/pause:3.10",
			}}
			payloadBytes, _ := json.Marshal(payload)
			_, err := cs.CoreV1().Pods(ns.GetName()).Patch(context.TODO(), pod.GetName(), types.JSONPatchType, payloadBytes, metav1.PatchOptions{})
			if err != nil {
				return err
			}
			return nil
		}).ShouldNot(Succeed())
	})

	It("should allow patching a matching registry after applying with a matching (EphemeralContainer)", Label("skip-on-openshift"), func() {
		ns := NewNamespace("", map[string]string{
			meta.TenantLabel: tnt.GetName(),
		})

		pod := &corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "container",
			},
			Spec: corev1.PodSpec{
				SecurityContext: nobodyPodSecurityContext(),
				Containers: []corev1.Container{
					{
						Name:            "container",
						Image:           "docker.io/library/busybox:1.36.1",
						SecurityContext: restrictedContainerSecurityContext(),
					},
				},
			},
		}

		cs := ownerClient(tnt.Spec.Owners[0].UserSpec)

		NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
		NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())

		role := &rbacv1.Role{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "ephemeralcontainers-editor",
				Namespace: ns.GetName(),
			},
			Rules: []rbacv1.PolicyRule{
				{
					APIGroups: []string{""}, // core API group
					Resources: []string{"pods/ephemeralcontainers"},
					Verbs:     []string{"update", "patch"},
				},
			},
		}

		rb := &rbacv1.RoleBinding{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "ephemeralcontainers-editor-binding",
				Namespace: ns.GetName(),
			},
			Subjects: []rbacv1.Subject{
				{
					Kind: rbacv1.UserKind,
					Name: tnt.Spec.Owners[0].Name,
				},
			},
			RoleRef: rbacv1.RoleRef{
				APIGroup: "rbac.authorization.k8s.io",
				Kind:     "Role",
				Name:     role.Name,
			},
		}

		// Create role and binding before test logic
		Expect(k8sClient.Create(context.TODO(), role)).To(Succeed())
		Expect(k8sClient.Create(context.TODO(), rb)).To(Succeed())

		EventuallyCreation(func() error {
			_, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{})

			return err
		}).Should(Succeed())

		Eventually(func() error {
			pod.Spec.EphemeralContainers = []corev1.EphemeralContainer{
				{
					EphemeralContainerCommon: corev1.EphemeralContainerCommon{
						Name:            "dbg",
						Image:           "myregistry.azurecr.io/pause:3.10",
						ImagePullPolicy: corev1.PullIfNotPresent,
					},
				},
			}

			_, err := cs.CoreV1().Pods(ns.Name).UpdateEphemeralContainers(context.Background(), pod.Name, pod, metav1.UpdateOptions{})
			if err != nil {
				return err
			}
			return nil
		}).Should(Succeed())
	})

	It("should allow patching a matching registry after applying with a matching (initContainer)", func() {
		ns := NewNamespace("", map[string]string{
			meta.TenantLabel: tnt.GetName(),
		})

		pod := &corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "container",
			},
			Spec: corev1.PodSpec{
				SecurityContext: nobodyPodSecurityContext(),
				InitContainers: []corev1.Container{
					{
						Name:            "init",
						Image:           "myregistry.azurecr.io/myapp:latest",
						SecurityContext: restrictedContainerSecurityContext(),
					},
				},

				Containers: []corev1.Container{
					{
						Name:            "container",
						Image:           "docker.io/library/busybox:1.36.1",
						SecurityContext: restrictedContainerSecurityContext(),
					},
				},
			},
		}

		cs := ownerClient(tnt.Spec.Owners[0].UserSpec)

		NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
		NamespaceIsPartOfTenant(tnt, ns).Should(Succeed())

		EventuallyCreation(func() error {
			_, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{})

			return err
		}).Should(Succeed())

		Eventually(func() error {
			payload := []Patch{{
				Op:    "replace",
				Path:  "/spec/initContainers/0/image",
				Value: "myregistry.azurecr.io/pause:3.10",
			}}
			payloadBytes, _ := json.Marshal(payload)
			_, err := cs.CoreV1().Pods(ns.GetName()).Patch(context.TODO(), pod.GetName(), types.JSONPatchType, payloadBytes, metav1.PatchOptions{})
			if err != nil {
				return err
			}
			return nil
		}).Should(Succeed())
	})

	It("should allow using an exact match", func() {
		ns := NewNamespace("", map[string]string{
			meta.TenantLabel: tnt.GetName(),
		})
		NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())

		pod := &corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "container",
			},
			Spec: corev1.PodSpec{
				Containers: []corev1.Container{
					{
						Name:  "container",
						Image: "docker.io/library/nginx:alpine",
					},
				},
			},
		}

		cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
		EventuallyCreation(func() error {
			_, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{})
			return err
		}).Should(Succeed())
	})

	It("should allow using a regex match", func() {
		ns := NewNamespace("", map[string]string{
			meta.TenantLabel: tnt.GetName(),
		})
		NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())

		pod := &corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "container",
			},
			Spec: corev1.PodSpec{
				SecurityContext: nobodyPodSecurityContext(),
				Containers: []corev1.Container{
					{
						Name:            "container",
						Image:           "quay.io/prometheus/busybox:latest",
						SecurityContext: restrictedContainerSecurityContext(),
					},
				},
			},
		}

		cs := ownerClient(tnt.Spec.Owners[0].UserSpec)
		EventuallyCreation(func() error {
			_, err := cs.CoreV1().Pods(ns.Name).Create(context.Background(), pod, metav1.CreateOptions{})
			return err
		}).Should(Succeed())
	})
})