feat: pv labelling and preventing cross-tenant mount

controllers/pv/controller.go

@@ -0,0 +1,117 @@
+// Copyright 2020-2021 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package pv
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/client-go/util/retry"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	log2 "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	capsulev1beta2 "github.com/clastix/capsule/api/v1beta2"
+	capsuleutils "github.com/clastix/capsule/pkg/utils"
+	webhookutils "github.com/clastix/capsule/pkg/webhook/utils"
+)
+
+type Controller struct {
+	client client.Client
+	label  string
+}
+
+func (c *Controller) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
+	log := log2.FromContext(ctx)
+
+	persistentVolume := corev1.PersistentVolume{}
+	if err := c.client.Get(ctx, request.NamespacedName, &persistentVolume); err != nil {
+		if errors.IsNotFound(err) {
+			log.Info("skipping reconciliation, resource may have been deleted")
+
+			return reconcile.Result{}, nil
+		}
+
+		log.Error(err, "cannot retrieve corev1.PersistentVolume")
+
+		return reconcile.Result{}, err
+	}
+
+	if persistentVolume.Spec.ClaimRef == nil {
+		log.Info("skipping reconciliation, missing claimRef")
+
+		return reconcile.Result{}, nil
+	}
+
+	tnt, err := webhookutils.TenantByStatusNamespace(ctx, c.client, persistentVolume.Spec.ClaimRef.Namespace)
+	if err != nil {
+		log.Error(err, "unable to retrieve Tenant from the claimRef")
+
+		return reconcile.Result{}, err
+	}
+
+	if tnt == nil {
+		log.Info("skipping reconciliation, PV is claimed by a PVC not managed in a Tenant")
+
+		return reconcile.Result{}, nil
+	}
+
+	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		pv := persistentVolume
+
+		if err = c.client.Get(ctx, request.NamespacedName, &pv); err != nil {
+			return err
+		}
+
+		labels := pv.GetLabels()
+		if labels == nil {
+			labels = map[string]string{}
+		}
+
+		labels[c.label] = tnt.GetName()
+
+		pv.SetLabels(labels)
+
+		return c.client.Update(ctx, &pv)
+	})
+	if retryErr != nil {
+		log.Error(retryErr, "unable to update PersistentVolume with Capsule label")
+
+		return reconcile.Result{}, retryErr
+	}
+
+	return reconcile.Result{}, nil
+}
+
+func (c *Controller) SetupWithManager(mgr ctrl.Manager) error {
+	label, err := capsuleutils.GetTypeLabel(&capsulev1beta2.Tenant{})
+	if err != nil {
+		return err
+	}
+
+	c.client = mgr.GetClient()
+	c.label = label
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.PersistentVolume{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			pv, ok := object.(*corev1.PersistentVolume)
+			if !ok {
+				return false
+			}
+
+			if pv.Spec.ClaimRef == nil {
+				return false
+			}
+
+			labels := object.GetLabels()
+			_, ok = labels[c.label]
+
+			return !ok
+		}))).
+		Complete(c)
+}

main.go

@@ -30,6 +30,7 @@ import (
 	capsulev1beta1 "github.com/clastix/capsule/api/v1beta1"
 	capsulev1beta2 "github.com/clastix/capsule/api/v1beta2"
 	configcontroller "github.com/clastix/capsule/controllers/config"
+	"github.com/clastix/capsule/controllers/pv"
 	rbaccontroller "github.com/clastix/capsule/controllers/rbac"
 	"github.com/clastix/capsule/controllers/resources"
 	servicelabelscontroller "github.com/clastix/capsule/controllers/servicelabels"
@@ -95,7 +96,7 @@ func newDelegatingClient(cache cache.Cache, config *rest.Config, options client.
 	return delegatingClient, nil
 }
 
-// nolint:maintidx
+// nolint:maintidx,cyclop
 func main() {
 	var enableLeaderElection, version bool
 
@@ -239,7 +240,7 @@ func main() {
 		route.Pod(pod.ImagePullPolicy(), pod.ContainerRegistry(), pod.PriorityClass(), pod.RuntimeClass()),
 		route.Namespace(utils.InCapsuleGroups(cfg, namespacewebhook.PatchHandler(), namespacewebhook.QuotaHandler(), namespacewebhook.FreezeHandler(cfg), namespacewebhook.PrefixHandler(cfg), namespacewebhook.UserMetadataHandler())),
 		route.Ingress(ingress.Class(cfg, kubeVersion), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()),
-		route.PVC(pvc.Handler()),
+		route.PVC(pvc.Validating(), pvc.PersistentVolumeReuse()),
 		route.Service(service.Handler()),
 		route.NetworkPolicy(utils.InCapsuleGroups(cfg, networkpolicy.Handler())),
 		route.Tenant(tenant.NameHandler(), tenant.RoleBindingRegexHandler(), tenant.IngressClassRegexHandler(), tenant.StorageClassRegexHandler(), tenant.ContainerRegistryRegexHandler(), tenant.HostnameRegexHandler(), tenant.FreezedEmitter(), tenant.ServiceAccountNameHandler(), tenant.ForbiddenAnnotationsRegexHandler(), tenant.ProtectedHandler()),
@@ -296,6 +297,11 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels")
 	}
 
+	if err = (&pv.Controller{}).SetupWithManager(manager); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "PersistentVolume")
+		os.Exit(1)
+	}
+
 	if err = (&configcontroller.Manager{
 		Log: ctrl.Log.WithName("controllers").WithName("CapsuleConfiguration"),
 	}).SetupWithManager(manager, configurationName); err != nil {

pkg/webhook/pvc/errors.go

@@ -43,3 +43,51 @@ func (f storageClassForbiddenError) Error() string {
 
 	return utils.DefaultAllowedValuesErrorMessage(f.spec, msg)
 }
+
+type missingPVLabelsError struct {
+	name string
+}
+
+func NewMissingPVLabelsError(name string) error {
+	return &missingPVLabelsError{name: name}
+}
+
+func (m missingPVLabelsError) Error() string {
+	return fmt.Sprintf("PeristentVolume %s is missing any label, please, ask the Cluster Administrator to label it", m.name)
+}
+
+type missingPVTenantLabelsError struct {
+	name string
+}
+
+func NewMissingTenantPVLabelsError(name string) error {
+	return &missingPVTenantLabelsError{name: name}
+}
+
+func (m missingPVTenantLabelsError) Error() string {
+	return fmt.Sprintf("PeristentVolume %s is missing the Capsule Tenant label, preventing a potential cross-tenant mount", m.name)
+}
+
+type crossTenantPVMountError struct {
+	name string
+}
+
+func NewCrossTenantPVMountError(name string) error {
+	return &crossTenantPVMountError{
+		name: name,
+	}
+}
+
+func (m crossTenantPVMountError) Error() string {
+	return fmt.Sprintf("PeristentVolume %s cannot be used by the following Tenant, preventing a cross-tenant mount", m.name)
+}
+
+type pvSelectorError struct{}
+
+func NewPVSelectorError() error {
+	return &pvSelectorError{}
+}
+
+func (m pvSelectorError) Error() string {
+	return "PersistentVolume selectors are not allowed since unable to prevent cross-tenant mount"
+}

pkg/webhook/pvc/pv.go

@@ -0,0 +1,98 @@
+// Copyright 2020-2021 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package pvc
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+	capsulev1beta2 "github.com/clastix/capsule/api/v1beta2"
+	capsulewebhook "github.com/clastix/capsule/pkg/webhook"
+	"github.com/clastix/capsule/pkg/webhook/utils"
+)
+
+type PV struct {
+	capsuleLabel string
+}
+
+func PersistentVolumeReuse() capsulewebhook.Handler {
+	value, err := capsulev1beta2.GetTypeLabel(&capsulev1beta2.Tenant{})
+	if err != nil {
+		panic(fmt.Sprintf("this shouldn't happen: %s", err.Error()))
+	}
+
+	return &PV{
+		capsuleLabel: value,
+	}
+}
+
+func (p PV) OnCreate(client client.Client, decoder *admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func {
+	return func(ctx context.Context, req admission.Request) *admission.Response {
+		pvc := corev1.PersistentVolumeClaim{}
+		if err := decoder.Decode(req, &pvc); err != nil {
+			return utils.ErroredResponse(err)
+		}
+
+		tnt, err := utils.TenantByStatusNamespace(ctx, client, pvc.GetNamespace())
+		if err != nil {
+			return utils.ErroredResponse(err)
+		}
+		// PVC is not in a Tenant Namespace, skipping
+		if tnt == nil {
+			return nil
+		}
+		// A PersistentVolume selector cannot help in preventing a cross-tenant mount:
+		// thus, disallowing that in first place.
+		if pvc.Spec.Selector != nil {
+			return utils.ErroredResponse(NewPVSelectorError())
+		}
+		// The PVC hasn't any volumeName pre-claimed, it can be skipped
+		if len(pvc.Spec.VolumeName) == 0 {
+			return nil
+		}
+		// Checking if the PV is labelled with the Tenant name
+		pv := corev1.PersistentVolume{}
+		if err = client.Get(ctx, types.NamespacedName{Name: pvc.Spec.VolumeName}, &pv); err != nil {
+			if errors.IsNotFound(err) {
+				err = fmt.Errorf("cannot create a PVC referring to a not yet existing PV")
+			}
+
+			return utils.ErroredResponse(err)
+		}
+
+		if pv.GetLabels() == nil {
+			return utils.ErroredResponse(NewMissingPVLabelsError(pv.GetName()))
+		}
+
+		value, ok := pv.GetLabels()[p.capsuleLabel]
+		if !ok {
+			return utils.ErroredResponse(NewMissingTenantPVLabelsError(pv.GetName()))
+		}
+
+		if value != p.capsuleLabel {
+			return utils.ErroredResponse(NewCrossTenantPVMountError(pv.GetName()))
+		}
+
+		return nil
+	}
+}
+
+func (p PV) OnDelete(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func {
+	return func(context.Context, admission.Request) *admission.Response {
+		return nil
+	}
+}
+
+func (p PV) OnUpdate(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func {
+	return func(context.Context, admission.Request) *admission.Response {
+		return nil
+	}
+}

pkg/webhook/pvc/validating.go

@@ -17,13 +17,13 @@ import (
 	"github.com/clastix/capsule/pkg/webhook/utils"
 )
 
-type handler struct{}
+type validating struct{}
 
-func Handler() capsulewebhook.Handler {
-	return &handler{}
+func Validating() capsulewebhook.Handler {
+	return &validating{}
 }
 
-func (h *handler) OnCreate(c client.Client, decoder *admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func {
+func (h *validating) OnCreate(c client.Client, decoder *admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func {
 	return func(ctx context.Context, req admission.Request) *admission.Response {
 		pvc := &corev1.PersistentVolumeClaim{}
 		if err := decoder.Decode(req, pvc); err != nil {
@@ -87,13 +87,13 @@ func (h *handler) OnCreate(c client.Client, decoder *admission.Decoder, recorder
 	}
 }
 
-func (h *handler) OnDelete(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func {
+func (h *validating) OnDelete(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func {
 	return func(ctx context.Context, req admission.Request) *admission.Response {
 		return nil
 	}
 }
 
-func (h *handler) OnUpdate(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func {
+func (h *validating) OnUpdate(client.Client, *admission.Decoder, record.EventRecorder) capsulewebhook.Func {
 	return func(ctx context.Context, req admission.Request) *admission.Response {
 		return nil
 	}