Supporting additional Role Bindings per Tenant (#133)

* Enabling Capsule to run on a cluster with PodSecurityPolicy enabled

* Supporting additional Role Binding per Tenant

* Documenting the additionalRoleBindings specification

api/v1alpha1/tenant_labels.go

@@ -21,6 +21,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -34,6 +35,8 @@ func GetTypeLabel(t runtime.Object) (label string, err error) {
 		return "capsule.clastix.io/network-policy", nil
 	case *corev1.ResourceQuota:
 		return "capsule.clastix.io/resource-quota", nil
+	case *rbacv1.RoleBinding:
+		return "capsule.clastix.io/role-binding", nil
 	default:
 		err = fmt.Errorf("type %T is not mapped as Capsule label recognized", v)
 	}

api/v1alpha1/tenant_types.go

@@ -19,6 +19,7 @@ package v1alpha1
 import (
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -61,7 +62,14 @@ type TenantSpec struct {
 	NetworkPolicies []networkingv1.NetworkPolicySpec `json:"networkPolicies,omitempty"`
 	LimitRanges     []corev1.LimitRangeSpec          `json:"limitRanges"`
 	// +kubebuilder:validation:Optional
-	ResourceQuota []corev1.ResourceQuotaSpec `json:"resourceQuotas"`
+	ResourceQuota          []corev1.ResourceQuotaSpec `json:"resourceQuotas"`
+	AdditionalRoleBindings []AdditionalRoleBindings   `json:"additionalRoleBindings,omitempty"`
+}
+
+type AdditionalRoleBindings struct {
+	ClusterRoleName string `json:"clusterRoleName"`
+	// kubebuilder:validation:Minimum=1
+	Subjects []rbacv1.Subject `json:"subjects"`
 }
 
 // OwnerSpec defines tenant owner name and kind

api/v1alpha1/zz_generated.deepcopy.go

@@ -23,6 +23,7 @@ package v1alpha1
 import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -55,6 +56,26 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AdditionalRoleBindings) DeepCopyInto(out *AdditionalRoleBindings) {
+	*out = *in
+	if in.Subjects != nil {
+		in, out := &in.Subjects, &out.Subjects
+		*out = make([]rbacv1.Subject, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalRoleBindings.
+func (in *AdditionalRoleBindings) DeepCopy() *AdditionalRoleBindings {
+	if in == nil {
+		return nil
+	}
+	out := new(AdditionalRoleBindings)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in IngressClassList) DeepCopyInto(out *IngressClassList) {
 	{
@@ -262,6 +283,13 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.AdditionalRoleBindings != nil {
+		in, out := &in.AdditionalRoleBindings, &out.AdditionalRoleBindings
+		*out = make([]AdditionalRoleBindings, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantSpec.

config/crd/bases/capsule.clastix.io_tenants.yaml

@@ -59,6 +59,49 @@ spec:
         spec:
           description: TenantSpec defines the desired state of Tenant
           properties:
+            additionalRoleBindings:
+              items:
+                properties:
+                  clusterRoleName:
+                    type: string
+                  subjects:
+                    description: kubebuilder:validation:Minimum=1
+                    items:
+                      description: Subject contains a reference to the object or user
+                        identities a role binding applies to.  This can either hold
+                        a direct API object reference, or a value for non-objects
+                        such as user and group names.
+                      properties:
+                        apiGroup:
+                          description: APIGroup holds the API group of the referenced
+                            subject. Defaults to "" for ServiceAccount subjects. Defaults
+                            to "rbac.authorization.k8s.io" for User and Group subjects.
+                          type: string
+                        kind:
+                          description: Kind of object being referenced. Values defined
+                            by this API group are "User", "Group", and "ServiceAccount".
+                            If the Authorizer does not recognized the kind value,
+                            the Authorizer should report an error.
+                          type: string
+                        name:
+                          description: Name of the object being referenced.
+                          type: string
+                        namespace:
+                          description: Namespace of the referenced object.  If the
+                            object kind is non-namespace, such as "User" or "Group",
+                            and this value is not empty the Authorizer should report
+                            an error.
+                          type: string
+                      required:
+                      - kind
+                      - name
+                      type: object
+                    type: array
+                required:
+                - clusterRoleName
+                - subjects
+                type: object
+              type: array
             ingressClasses:
               properties:
                 allowed:

config/rbac/kustomization.yaml

@@ -7,3 +7,9 @@ resources:
 - auth_proxy_role.yaml
 - auth_proxy_role_binding.yaml
 - auth_proxy_client_clusterrole.yaml
+# Uncomment the following 3 lines if you are running Capsule
+# in a cluster where [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/)
+# are enabled.
+# - psp_policy.yaml
+# - psp_role.yaml
+# - psp_role_binding.yaml

config/rbac/psp_policy.yaml

@@ -0,0 +1,18 @@
+kind: PodSecurityPolicy
+apiVersion: policy/v1beta1
+metadata:
+  name: capsule
+spec:
+  fsGroup:
+    rule: RunAsAny
+  hostPorts:
+    - max: 0
+      min: 0
+  runAsUser:
+    rule: RunAsAny
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  volumes:
+    - secret

config/rbac/psp_role.yaml

@@ -0,0 +1,9 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: capsule-psp
+rules:
+  - apiGroups: ['extensions']
+    resources: ['podsecuritypolicies']
+    resourceNames: ['capsule-psp']
+    verbs: ['use']

config/rbac/psp_role_binding.yaml

@@ -0,0 +1,12 @@
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: capsule-use-psp
+  namespace: system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: capsule-psp
+subjects:
+  - kind: ServiceAccount
+    name: default

config/webhook/manifests.yaml

@@ -23,6 +23,7 @@ webhooks:
     - CREATE
     resources:
     - namespaces
+
 ---
 apiVersion: admissionregistration.k8s.io/v1beta1
 kind: ValidatingWebhookConfiguration

controllers/tenant_controller.go

@@ -19,6 +19,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"hash/fnv"
 	"strconv"
 	"strings"
 	"sync"
@@ -116,6 +117,12 @@ func (r TenantReconciler) Reconcile(ctx context.Context, request ctrl.Request) (
 		return reconcile.Result{}, err
 	}
 
+	r.Log.Info("Ensuring PSP for owner")
+	if err := r.syncAdditionalRoleBindings(instance); err != nil {
+		r.Log.Error(err, "Cannot sync additional Role Bindings items")
+		return reconcile.Result{}, err
+	}
+
 	r.Log.Info("Ensuring RoleBinding for owner")
 	if err := r.ownerRoleBinding(instance); err != nil {
 		r.Log.Error(err, "Cannot sync owner RoleBinding")
@@ -218,6 +225,70 @@ func (r *TenantReconciler) resourceQuotasUpdate(resourceName corev1.ResourceName
 	return
 }
 
+// Additional Role Bindings can be used in many ways: applying Pod Security Policies or giving
+// access to CRDs or specific API groups.
+func (r *TenantReconciler) syncAdditionalRoleBindings(tenant *capsulev1alpha1.Tenant) (err error) {
+	// hashing the RoleBinding name due to DNS RFC-1123 applied to Kubernetes labels
+	hash := func(value string) string {
+		h := fnv.New64a()
+		_, _ = h.Write([]byte(value))
+		return fmt.Sprintf("%x", h.Sum64())
+	}
+	// getting requested Role Binding keys
+	var keys []string
+	for _, i := range tenant.Spec.AdditionalRoleBindings {
+		keys = append(keys, hash(i.ClusterRoleName))
+	}
+
+	var tl, ll string
+	tl, err = capsulev1alpha1.GetTypeLabel(&capsulev1alpha1.Tenant{})
+	if err != nil {
+		return
+	}
+	ll, err = capsulev1alpha1.GetTypeLabel(&rbacv1.RoleBinding{})
+	if err != nil {
+		return
+	}
+
+	for _, ns := range tenant.Status.Namespaces {
+		if err = r.pruningResources(ns, keys, &rbacv1.RoleBinding{}); err != nil {
+			return err
+		}
+		for _, i := range tenant.Spec.AdditionalRoleBindings {
+			lv := hash(i.ClusterRoleName)
+			rb := &rbacv1.RoleBinding{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      fmt.Sprintf("capsule-%s-%s", tenant.Name, i.ClusterRoleName),
+					Namespace: ns,
+				},
+			}
+			var res controllerutil.OperationResult
+			res, err = controllerutil.CreateOrUpdate(context.TODO(), r.Client, rb, func() error {
+				rb.ObjectMeta.Labels = map[string]string{
+					tl: tenant.Name,
+					ll: lv,
+				}
+				rb.RoleRef = rbacv1.RoleRef{
+					APIGroup: "rbac.authorization.k8s.io",
+					Kind:     "ClusterRole",
+					Name:     i.ClusterRoleName,
+				}
+				rb.Subjects = i.Subjects
+				return controllerutil.SetControllerReference(tenant, rb, r.Scheme)
+			})
+			if err != nil {
+				r.Log.Error(err, "Cannot sync Additional RoleBinding")
+			}
+			r.Log.Info(fmt.Sprintf("Additional RoleBindings sync result: %s", string(res)), "name", rb.Name, "namespace", rb.Namespace)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	return nil
+}
+
 // We're relying on the ResourceQuota resource to represent the resource quota for the single Tenant rather than the
 // single Namespace, so abusing of this API although its Namespaced scope.
 // Since a Namespace could take-up all the available resource quota, the Namespace ResourceQuota will be a 1:1 mapping

e2e/additional_role_bindings_test.go

@@ -0,0 +1,92 @@
+//+build e2e
+
+/*
+Copyright 2020 Clastix Labs.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/clastix/capsule/api/v1alpha1"
+)
+
+var _ = Describe("creating a Namespace with additional Role Binding", func() {
+	tnt := &v1alpha1.Tenant{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "additional-role-binding",
+		},
+		Spec: v1alpha1.TenantSpec{
+			Owner: v1alpha1.OwnerSpec{
+				Name: "dale",
+				Kind: "User",
+			},
+			NamespacesMetadata: v1alpha1.AdditionalMetadata{},
+			ServicesMetadata:   v1alpha1.AdditionalMetadata{},
+			IngressClasses:     v1alpha1.IngressClassesSpec{},
+			StorageClasses:     v1alpha1.StorageClassesSpec{},
+			LimitRanges:        []corev1.LimitRangeSpec{},
+			NamespaceQuota:     10,
+			NodeSelector:       map[string]string{},
+			ResourceQuota:      []corev1.ResourceQuotaSpec{},
+			AdditionalRoleBindings: []v1alpha1.AdditionalRoleBindings{
+				{
+					ClusterRoleName: "crds-rolebinding",
+					Subjects: []rbacv1.Subject{
+						{
+							Kind:     "Group",
+							APIGroup: "rbac.authorization.k8s.io",
+							Name:     "system:authenticated",
+						},
+					},
+				},
+			},
+		},
+	}
+	JustBeforeEach(func() {
+		EventuallyCreation(func() error {
+			tnt.ResourceVersion = ""
+			return k8sClient.Create(context.TODO(), tnt)
+		}).Should(Succeed())
+	})
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
+	})
+	It("should be assigned on each Namespace", func() {
+		for _, ns := range []string{"rb-1", "rb-2", "rb-3"} {
+			ns := NewNamespace(ns)
+			NamespaceCreation(ns, tnt, defaultTimeoutInterval).Should(Succeed())
+			TenantNamespaceList(tnt, podRecreationTimeoutInterval).Should(ContainElement(ns.GetName()))
+
+			var rb *rbacv1.RoleBinding
+
+			Eventually(func() (err error) {
+				cs := ownerClient(tnt)
+				rb, err = cs.RbacV1().RoleBindings(ns.Name).Get(context.Background(), fmt.Sprintf("capsule-%s-%s", tnt.Name, "crds-rolebinding"), metav1.GetOptions{})
+				return err
+			}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
+			Expect(rb.RoleRef.Name).Should(Equal(tnt.Spec.AdditionalRoleBindings[0].ClusterRoleName))
+			Expect(rb.Subjects).Should(Equal(tnt.Spec.AdditionalRoleBindings[0].Subjects))
+		}
+	})
+})

use_cases.md

@@ -893,4 +893,73 @@ alice@caas# kubectl -n oil-production delete networkpolicy capsule-oil-0
 Error from server (Capsule Network Policies cannot be deleted: please, reach out the system administrators): admission webhook "validating.network-policy.capsule.clastix.io" denied the request: Capsule Network Policies cannot be deleted: please, reach out the system administrators
 ```
 
+### Adding additional Role Binding to Tenant Namespaces
 
+In a Container as a Service scenario could be useful to add some specific
+features or capabilities, like _interacting with a CRDs_ or enforcing the Pod
+policies using _Pod Security Policies_.
+
+In one case or the another, there are some mandatory steps:
+
+1. Install the _CRD_ or define the _Pod Security Policy_
+
+```yaml
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: psp:restricted
+spec:
+  privileged: false
+  allowPrivilegeEscalation: false
+...
+```
+
+2. Create a _ClusterRole_ using or granting the said item
+
+```yaml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: psp:restricted
+rules:
+- apiGroups: ['policy']
+  resources: ['podsecuritypolicies']
+  resourceNames: ['psp:restricted']
+  verbs: ['use']
+```
+
+3. For each Tenant Namespace, create a _RoleBinding_ pointing to the said
+   _ClusterRole_.
+
+```
+kubectl create rolebinding -n ${NAMESPACE} psp:privileged \
+    --clusterrole=psp:privileged --group=systems:authenticated
+```
+
+This can be easily achieved and automated using the Tenant\
+`additionalRoleBindings` specification.
+
+```yaml
+apiVersion: capsule.clastix.io/v1alpha1
+kind: Tenant
+metadata:
+  labels:
+  annotations:
+  name: oil
+spec:
+  additionalRoleBindings:
+  - clusterRoleName: psp:privileged
+    subjects:
+    - kind: "Group"
+      apiGroup: "rbac.authorization.k8s.io"
+      name: "system:authenticated"
+...
+```
+
+With the given specification, Capsule will ensure that each Namespace will
+contain the desired _RoleBinding_ for the specified _Cluster Role_ bounded to
+the given subjects.
+
+> With the following example, Capsule is forbidding to any authenticated user
+> to run privileged pods and let them to performs privilege escalation as
+> declared by the Cluster Role `psp:privileged`.