From ea599ba6e6bf1954e51fc2e68bde83ae90e5703e Mon Sep 17 00:00:00 2001 From: Dario Tranchitella Date: Mon, 16 Nov 2020 13:51:44 +0100 Subject: [PATCH] Supporting additional Role Bindings per Tenant (#133) * Enabling Capsule to run on a cluster with PodSecurityPolicy enabled * Supporting additional Role Binding per Tenant * Documenting the additionalRoleBindings specification --- api/v1alpha1/tenant_labels.go | 3 + api/v1alpha1/tenant_types.go | 10 +- api/v1alpha1/zz_generated.deepcopy.go | 28 ++++++ .../crd/bases/capsule.clastix.io_tenants.yaml | 43 +++++++++ config/rbac/kustomization.yaml | 6 ++ config/rbac/psp_policy.yaml | 18 ++++ config/rbac/psp_role.yaml | 9 ++ config/rbac/psp_role_binding.yaml | 12 +++ config/webhook/manifests.yaml | 1 + controllers/tenant_controller.go | 71 ++++++++++++++ e2e/additional_role_bindings_test.go | 92 +++++++++++++++++++ use_cases.md | 69 ++++++++++++++ 12 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 config/rbac/psp_policy.yaml create mode 100644 config/rbac/psp_role.yaml create mode 100644 config/rbac/psp_role_binding.yaml create mode 100644 e2e/additional_role_bindings_test.go diff --git a/api/v1alpha1/tenant_labels.go b/api/v1alpha1/tenant_labels.go index e384bab6..c6417bd0 100644 --- a/api/v1alpha1/tenant_labels.go +++ b/api/v1alpha1/tenant_labels.go @@ -21,6 +21,7 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -34,6 +35,8 @@ func GetTypeLabel(t runtime.Object) (label string, err error) { return "capsule.clastix.io/network-policy", nil case *corev1.ResourceQuota: return "capsule.clastix.io/resource-quota", nil + case *rbacv1.RoleBinding: + return "capsule.clastix.io/role-binding", nil default: err = fmt.Errorf("type %T is not mapped as Capsule label recognized", v) } diff --git a/api/v1alpha1/tenant_types.go b/api/v1alpha1/tenant_types.go index dd4dabba..b592e531 100644 --- a/api/v1alpha1/tenant_types.go +++ b/api/v1alpha1/tenant_types.go @@ -19,6 +19,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -61,7 +62,14 @@ type TenantSpec struct { NetworkPolicies []networkingv1.NetworkPolicySpec `json:"networkPolicies,omitempty"` LimitRanges []corev1.LimitRangeSpec `json:"limitRanges"` // +kubebuilder:validation:Optional - ResourceQuota []corev1.ResourceQuotaSpec `json:"resourceQuotas"` + ResourceQuota []corev1.ResourceQuotaSpec `json:"resourceQuotas"` + AdditionalRoleBindings []AdditionalRoleBindings `json:"additionalRoleBindings,omitempty"` +} + +type AdditionalRoleBindings struct { + ClusterRoleName string `json:"clusterRoleName"` + // kubebuilder:validation:Minimum=1 + Subjects []rbacv1.Subject `json:"subjects"` } // OwnerSpec defines tenant owner name and kind diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 98c97c2f..82d80b20 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -23,6 +23,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -55,6 +56,26 @@ func (in *AdditionalMetadata) DeepCopy() *AdditionalMetadata { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AdditionalRoleBindings) DeepCopyInto(out *AdditionalRoleBindings) { + *out = *in + if in.Subjects != nil { + in, out := &in.Subjects, &out.Subjects + *out = make([]rbacv1.Subject, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalRoleBindings. +func (in *AdditionalRoleBindings) DeepCopy() *AdditionalRoleBindings { + if in == nil { + return nil + } + out := new(AdditionalRoleBindings) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in IngressClassList) DeepCopyInto(out *IngressClassList) { { @@ -262,6 +283,13 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.AdditionalRoleBindings != nil { + in, out := &in.AdditionalRoleBindings, &out.AdditionalRoleBindings + *out = make([]AdditionalRoleBindings, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantSpec. diff --git a/config/crd/bases/capsule.clastix.io_tenants.yaml b/config/crd/bases/capsule.clastix.io_tenants.yaml index 0b6f5a2a..10e4e28a 100644 --- a/config/crd/bases/capsule.clastix.io_tenants.yaml +++ b/config/crd/bases/capsule.clastix.io_tenants.yaml @@ -59,6 +59,49 @@ spec: spec: description: TenantSpec defines the desired state of Tenant properties: + additionalRoleBindings: + items: + properties: + clusterRoleName: + type: string + subjects: + description: kubebuilder:validation:Minimum=1 + items: + description: Subject contains a reference to the object or user + identities a role binding applies to. This can either hold + a direct API object reference, or a value for non-objects + such as user and group names. + properties: + apiGroup: + description: APIGroup holds the API group of the referenced + subject. Defaults to "" for ServiceAccount subjects. Defaults + to "rbac.authorization.k8s.io" for User and Group subjects. + type: string + kind: + description: Kind of object being referenced. Values defined + by this API group are "User", "Group", and "ServiceAccount". + If the Authorizer does not recognized the kind value, + the Authorizer should report an error. + type: string + name: + description: Name of the object being referenced. + type: string + namespace: + description: Namespace of the referenced object. If the + object kind is non-namespace, such as "User" or "Group", + and this value is not empty the Authorizer should report + an error. + type: string + required: + - kind + - name + type: object + type: array + required: + - clusterRoleName + - subjects + type: object + type: array ingressClasses: properties: allowed: diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 2f8909ba..2d3d67eb 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -7,3 +7,9 @@ resources: - auth_proxy_role.yaml - auth_proxy_role_binding.yaml - auth_proxy_client_clusterrole.yaml +# Uncomment the following 3 lines if you are running Capsule +# in a cluster where [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) +# are enabled. +# - psp_policy.yaml +# - psp_role.yaml +# - psp_role_binding.yaml diff --git a/config/rbac/psp_policy.yaml b/config/rbac/psp_policy.yaml new file mode 100644 index 00000000..16cbc9c0 --- /dev/null +++ b/config/rbac/psp_policy.yaml @@ -0,0 +1,18 @@ +kind: PodSecurityPolicy +apiVersion: policy/v1beta1 +metadata: + name: capsule +spec: + fsGroup: + rule: RunAsAny + hostPorts: + - max: 0 + min: 0 + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - secret diff --git a/config/rbac/psp_role.yaml b/config/rbac/psp_role.yaml new file mode 100644 index 00000000..e56aa21c --- /dev/null +++ b/config/rbac/psp_role.yaml @@ -0,0 +1,9 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: capsule-psp +rules: + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + resourceNames: ['capsule-psp'] + verbs: ['use'] diff --git a/config/rbac/psp_role_binding.yaml b/config/rbac/psp_role_binding.yaml new file mode 100644 index 00000000..6041a6e9 --- /dev/null +++ b/config/rbac/psp_role_binding.yaml @@ -0,0 +1,12 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: capsule-use-psp + namespace: system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: capsule-psp +subjects: + - kind: ServiceAccount + name: default diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 1d35d920..7369a25c 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -23,6 +23,7 @@ webhooks: - CREATE resources: - namespaces + --- apiVersion: admissionregistration.k8s.io/v1beta1 kind: ValidatingWebhookConfiguration diff --git a/controllers/tenant_controller.go b/controllers/tenant_controller.go index a5f80118..e009fb89 100644 --- a/controllers/tenant_controller.go +++ b/controllers/tenant_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "hash/fnv" "strconv" "strings" "sync" @@ -116,6 +117,12 @@ func (r TenantReconciler) Reconcile(ctx context.Context, request ctrl.Request) ( return reconcile.Result{}, err } + r.Log.Info("Ensuring PSP for owner") + if err := r.syncAdditionalRoleBindings(instance); err != nil { + r.Log.Error(err, "Cannot sync additional Role Bindings items") + return reconcile.Result{}, err + } + r.Log.Info("Ensuring RoleBinding for owner") if err := r.ownerRoleBinding(instance); err != nil { r.Log.Error(err, "Cannot sync owner RoleBinding") @@ -218,6 +225,70 @@ func (r *TenantReconciler) resourceQuotasUpdate(resourceName corev1.ResourceName return } +// Additional Role Bindings can be used in many ways: applying Pod Security Policies or giving +// access to CRDs or specific API groups. +func (r *TenantReconciler) syncAdditionalRoleBindings(tenant *capsulev1alpha1.Tenant) (err error) { + // hashing the RoleBinding name due to DNS RFC-1123 applied to Kubernetes labels + hash := func(value string) string { + h := fnv.New64a() + _, _ = h.Write([]byte(value)) + return fmt.Sprintf("%x", h.Sum64()) + } + // getting requested Role Binding keys + var keys []string + for _, i := range tenant.Spec.AdditionalRoleBindings { + keys = append(keys, hash(i.ClusterRoleName)) + } + + var tl, ll string + tl, err = capsulev1alpha1.GetTypeLabel(&capsulev1alpha1.Tenant{}) + if err != nil { + return + } + ll, err = capsulev1alpha1.GetTypeLabel(&rbacv1.RoleBinding{}) + if err != nil { + return + } + + for _, ns := range tenant.Status.Namespaces { + if err = r.pruningResources(ns, keys, &rbacv1.RoleBinding{}); err != nil { + return err + } + for _, i := range tenant.Spec.AdditionalRoleBindings { + lv := hash(i.ClusterRoleName) + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("capsule-%s-%s", tenant.Name, i.ClusterRoleName), + Namespace: ns, + }, + } + var res controllerutil.OperationResult + res, err = controllerutil.CreateOrUpdate(context.TODO(), r.Client, rb, func() error { + rb.ObjectMeta.Labels = map[string]string{ + tl: tenant.Name, + ll: lv, + } + rb.RoleRef = rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: i.ClusterRoleName, + } + rb.Subjects = i.Subjects + return controllerutil.SetControllerReference(tenant, rb, r.Scheme) + }) + if err != nil { + r.Log.Error(err, "Cannot sync Additional RoleBinding") + } + r.Log.Info(fmt.Sprintf("Additional RoleBindings sync result: %s", string(res)), "name", rb.Name, "namespace", rb.Namespace) + if err != nil { + return + } + } + } + + return nil +} + // We're relying on the ResourceQuota resource to represent the resource quota for the single Tenant rather than the // single Namespace, so abusing of this API although its Namespaced scope. // Since a Namespace could take-up all the available resource quota, the Namespace ResourceQuota will be a 1:1 mapping diff --git a/e2e/additional_role_bindings_test.go b/e2e/additional_role_bindings_test.go new file mode 100644 index 00000000..9de7e223 --- /dev/null +++ b/e2e/additional_role_bindings_test.go @@ -0,0 +1,92 @@ +//+build e2e + +/* +Copyright 2020 Clastix Labs. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/clastix/capsule/api/v1alpha1" +) + +var _ = Describe("creating a Namespace with additional Role Binding", func() { + tnt := &v1alpha1.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "additional-role-binding", + }, + Spec: v1alpha1.TenantSpec{ + Owner: v1alpha1.OwnerSpec{ + Name: "dale", + Kind: "User", + }, + NamespacesMetadata: v1alpha1.AdditionalMetadata{}, + ServicesMetadata: v1alpha1.AdditionalMetadata{}, + IngressClasses: v1alpha1.IngressClassesSpec{}, + StorageClasses: v1alpha1.StorageClassesSpec{}, + LimitRanges: []corev1.LimitRangeSpec{}, + NamespaceQuota: 10, + NodeSelector: map[string]string{}, + ResourceQuota: []corev1.ResourceQuotaSpec{}, + AdditionalRoleBindings: []v1alpha1.AdditionalRoleBindings{ + { + ClusterRoleName: "crds-rolebinding", + Subjects: []rbacv1.Subject{ + { + Kind: "Group", + APIGroup: "rbac.authorization.k8s.io", + Name: "system:authenticated", + }, + }, + }, + }, + }, + } + JustBeforeEach(func() { + EventuallyCreation(func() error { + tnt.ResourceVersion = "" + return k8sClient.Create(context.TODO(), tnt) + }).Should(Succeed()) + }) + JustAfterEach(func() { + Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed()) + }) + It("should be assigned on each Namespace", func() { + for _, ns := range []string{"rb-1", "rb-2", "rb-3"} { + ns := NewNamespace(ns) + NamespaceCreation(ns, tnt, defaultTimeoutInterval).Should(Succeed()) + TenantNamespaceList(tnt, podRecreationTimeoutInterval).Should(ContainElement(ns.GetName())) + + var rb *rbacv1.RoleBinding + + Eventually(func() (err error) { + cs := ownerClient(tnt) + rb, err = cs.RbacV1().RoleBindings(ns.Name).Get(context.Background(), fmt.Sprintf("capsule-%s-%s", tnt.Name, "crds-rolebinding"), metav1.GetOptions{}) + return err + }, defaultTimeoutInterval, defaultPollInterval).Should(Succeed()) + Expect(rb.RoleRef.Name).Should(Equal(tnt.Spec.AdditionalRoleBindings[0].ClusterRoleName)) + Expect(rb.Subjects).Should(Equal(tnt.Spec.AdditionalRoleBindings[0].Subjects)) + } + }) +}) diff --git a/use_cases.md b/use_cases.md index 21cb17b1..80016d0c 100644 --- a/use_cases.md +++ b/use_cases.md @@ -893,4 +893,73 @@ alice@caas# kubectl -n oil-production delete networkpolicy capsule-oil-0 Error from server (Capsule Network Policies cannot be deleted: please, reach out the system administrators): admission webhook "validating.network-policy.capsule.clastix.io" denied the request: Capsule Network Policies cannot be deleted: please, reach out the system administrators ``` +### Adding additional Role Binding to Tenant Namespaces +In a Container as a Service scenario could be useful to add some specific +features or capabilities, like _interacting with a CRDs_ or enforcing the Pod +policies using _Pod Security Policies_. + +In one case or the another, there are some mandatory steps: + +1. Install the _CRD_ or define the _Pod Security Policy_ + +```yaml +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: psp:restricted +spec: + privileged: false + allowPrivilegeEscalation: false +... +``` + +2. Create a _ClusterRole_ using or granting the said item + +```yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp:restricted +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + resourceNames: ['psp:restricted'] + verbs: ['use'] +``` + +3. For each Tenant Namespace, create a _RoleBinding_ pointing to the said + _ClusterRole_. + +``` +kubectl create rolebinding -n ${NAMESPACE} psp:privileged \ + --clusterrole=psp:privileged --group=systems:authenticated +``` + +This can be easily achieved and automated using the Tenant\ +`additionalRoleBindings` specification. + +```yaml +apiVersion: capsule.clastix.io/v1alpha1 +kind: Tenant +metadata: + labels: + annotations: + name: oil +spec: + additionalRoleBindings: + - clusterRoleName: psp:privileged + subjects: + - kind: "Group" + apiGroup: "rbac.authorization.k8s.io" + name: "system:authenticated" +... +``` + +With the given specification, Capsule will ensure that each Namespace will +contain the desired _RoleBinding_ for the specified _Cluster Role_ bounded to +the given subjects. + +> With the following example, Capsule is forbidding to any authenticated user +> to run privileged pods and let them to performs privilege escalation as +> declared by the Cluster Role `psp:privileged`.