Add capsule-user-group CLI flag (#67)

* add capsule-user-group param

* Implementing RBAC controller

Co-authored-by: Maksim Fedotov <m_fedotov@wargaming.net>
Co-authored-by: Dario Tranchitella <dario@tranchitella.eu>

README.md

@@ -46,12 +46,9 @@ make deploy
 # /usr/local/bin/kustomize build config/default | kubectl apply -f -
 # namespace/capsule-system created
 # customresourcedefinition.apiextensions.k8s.io/tenants.capsule.clastix.io created
-# clusterrole.rbac.authorization.k8s.io/capsule-namespace:deleter created
-# clusterrole.rbac.authorization.k8s.io/capsule-namespace:provisioner created
 # clusterrole.rbac.authorization.k8s.io/capsule-proxy-role created
 # clusterrole.rbac.authorization.k8s.io/capsule-metrics-reader created
 # clusterrolebinding.rbac.authorization.k8s.io/capsule-manager-rolebinding created
-# clusterrolebinding.rbac.authorization.k8s.io/capsule-namespace:provisioner created
 # clusterrolebinding.rbac.authorization.k8s.io/capsule-proxy-rolebinding created
 # secret/capsule-ca created
 # secret/capsule-tls created
@@ -64,6 +61,8 @@ make deploy
 
 Log verbosity of the Capsule controller can be increased by passing the `--zap-log-level` option with a value from `1` to `10` or the [basic keywords](https://godoc.org/go.uber.org/zap/zapcore#Level) although it is suggested to use the `--zap-devel` flag to get also stack traces.
 
+During startup Capsule controller will create additional ClusterRoles `capsule-namespace:deleter`, `capsule-namespace:provisioner` and ClusterRoleBinding `capsule-namespace:provisioner`. These resources are used in order to allow Capsule users to manage their namespaces in tenants.
+
 ## Admission Controllers
 Capsule implements Kubernetes multi-tenancy capabilities using a minimum set of standard [Admission Controllers](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/) enabled on the Kubernetes APIs server: `--enable-admission-plugins=PodNodeSelector,LimitRanger,ResourceQuota,MutatingAdmissionWebhook,ValidatingAdmissionWebhook`. In addition to these default controllers, Capsule implements its own set of Admission Controllers through the [Dynamic Admission Controller](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), providing callbacks to add further validation or resource patching.
 
@@ -73,9 +72,9 @@ the API Server is communicating with the right client. Capsule upon installation
 ## Tenant users
 Each tenant comes with a delegated user acting as the tenant admin. In the Capsule jargon, this user is called the _Tenant Owner_. Other users can operate inside a tenant with different levels of permissions and authorizations assigned directly by the Tenant owner.
 
-Capsule does not care about the authentication strategy used in the cluster and all the Kubernetes methods of [authentication](https://kubernetes.io/docs/reference/access-authn-authz/authentication/) are supported. The only requirement to use Capsule is to assign tenant users to the `capsule.clastix.io` group.
+Capsule does not care about the authentication strategy used in the cluster and all the Kubernetes methods of [authentication](https://kubernetes.io/docs/reference/access-authn-authz/authentication/) are supported. The only requirement to use Capsule is to assign tenant users to the the group defined by `--capsule-user-group` option, which defaults to `capsule.clastix.io`.
 
-Assignment to a group depends on the authentication strategy in your cluster. For example, users authenticated through a _X.509_ certificate must have `capsule.clastix.io` as _Organization_: `-subj "/CN=${USER}/O=capsule.clastix.io"`
+Assignment to a group depends on the authentication strategy in your cluster. For example, if you are using `capsule.clastix.io` as your `--capsule-user-group`, users authenticated through a _X.509_ certificate must have `capsule.clastix.io` as _Organization_: `-subj "/CN=${USER}/O=capsule.clastix.io"`
 
 Users authenticated through an _OIDC token_ must have
 

config/default/kustomization.yaml

@@ -18,7 +18,6 @@ bases:
 - ../manager
 - ../secret
 - ../webhook
-- ../tenants
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
 

config/tenants/kustomization.yaml

@@ -1,3 +0,0 @@
-resources:
-- namespace-deleter.yaml
-- namespace-provisioner.yaml

config/tenants/namespace-deleter.yaml

@@ -1,8 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: namespace:deleter
-rules:
-  - apiGroups: [""]
-    resources: ["namespaces"]
-    verbs: ["delete"]

config/tenants/namespace-provisioner.yaml

@@ -1,22 +0,0 @@
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  labels:
-  name: namespace:provisioner
-rules:
-  - apiGroups: [""]
-    resources: ["namespaces"]
-    verbs: ["create"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: namespace:provisioner
-subjects:
-  - kind: Group
-    name: capsule.clastix.io
-roleRef:
-  kind: ClusterRole
-  name: namespace:provisioner
-  apiGroup: rbac.authorization.k8s.io

controllers/rbac/const.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2020 Clastix Labs.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbac
+
+import (
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	ProvisionerRoleName = "capsule-namespace:provisioner"
+	DeleterRoleName     = "capsule-namespace:deleter"
+)
+
+var (
+	clusterRoles = map[string]*rbacv1.ClusterRole{
+		ProvisionerRoleName: {
+			ObjectMeta: metav1.ObjectMeta{
+				Name: ProvisionerRoleName,
+			},
+			Rules: []rbacv1.PolicyRule{
+				{
+					APIGroups: []string{""},
+					Resources: []string{"namespaces"},
+					Verbs:     []string{"create"},
+				},
+			},
+		},
+		DeleterRoleName: {
+			ObjectMeta: metav1.ObjectMeta{
+				Name: DeleterRoleName,
+			},
+			Rules: []rbacv1.PolicyRule{
+				{
+					APIGroups: []string{""},
+					Resources: []string{"namespaces"},
+					Verbs:     []string{"delete"},
+				},
+			},
+		},
+	}
+
+	provisionerClusterRoleBinding = &rbacv1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: ProvisionerRoleName,
+		},
+		RoleRef: rbacv1.RoleRef{
+			Kind:     "ClusterRole",
+			Name:     ProvisionerRoleName,
+			APIGroup: "rbac.authorization.k8s.io",
+		},
+	}
+)

controllers/rbac/errors.go

@@ -0,0 +1,24 @@
+/*
+Copyright 2020 Clastix Labs.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbac
+
+type ImmutableClusterRoleBindingError struct {
+}
+
+func (i ImmutableClusterRoleBindingError) Error() string {
+	return "The ClusterRoleBinding Role reference is immutable, deletion must be processed first"
+}

controllers/rbac/manager.go

@@ -0,0 +1,182 @@
+/*
+Copyright 2020 Clastix Labs.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbac
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	"github.com/hashicorp/go-multierror"
+	rbacv1 "k8s.io/api/rbac/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+type Manager struct {
+	CapsuleGroup string
+	Log          logr.Logger
+	Client       client.Client
+}
+
+// Using the Client interface, required by the Runnable interface
+func (r *Manager) InjectClient(c client.Client) error {
+	r.Client = c
+	return nil
+}
+
+func (r *Manager) filterByClusterRolesNames(name string) bool {
+	return name == ProvisionerRoleName || name == DeleterRoleName
+}
+
+func (r *Manager) SetupWithManager(mgr ctrl.Manager) (err error) {
+	crErr := ctrl.NewControllerManagedBy(mgr).
+		For(&rbacv1.ClusterRole{}, builder.WithPredicates(predicate.Funcs{
+			CreateFunc: func(event event.CreateEvent) bool {
+				return r.filterByClusterRolesNames(event.Meta.GetName())
+			},
+			DeleteFunc: func(deleteEvent event.DeleteEvent) bool {
+				return r.filterByClusterRolesNames(deleteEvent.Meta.GetName())
+			},
+			UpdateFunc: func(updateEvent event.UpdateEvent) bool {
+				return r.filterByClusterRolesNames(updateEvent.MetaNew.GetName())
+			},
+			GenericFunc: func(genericEvent event.GenericEvent) bool {
+				return r.filterByClusterRolesNames(genericEvent.Meta.GetName())
+			},
+		})).
+		Complete(r)
+	if crErr != nil {
+		err = multierror.Append(err, crErr)
+	}
+	crbErr := ctrl.NewControllerManagedBy(mgr).
+		For(&rbacv1.ClusterRoleBinding{}, builder.WithPredicates(predicate.Funcs{
+			CreateFunc: func(event event.CreateEvent) bool {
+				return event.Meta.GetName() == ProvisionerRoleName
+			},
+			DeleteFunc: func(deleteEvent event.DeleteEvent) bool {
+				return deleteEvent.Meta.GetName() == ProvisionerRoleName
+			},
+			UpdateFunc: func(updateEvent event.UpdateEvent) bool {
+				return updateEvent.MetaNew.GetName() == ProvisionerRoleName
+			},
+			GenericFunc: func(genericEvent event.GenericEvent) bool {
+				return genericEvent.Meta.GetName() == ProvisionerRoleName
+			},
+		})).
+		Complete(r)
+	if crbErr != nil {
+		err = multierror.Append(err, crbErr)
+	}
+	return
+}
+
+// This reconcile function is serving both ClusterRole and ClusterRoleBinding: that's ok, we're watching for multiple
+// Resource kinds and we're just interested to the ones with the said name since they're bounded together.
+func (r *Manager) Reconcile(request reconcile.Request) (res reconcile.Result, err error) {
+	switch request.Name {
+	case ProvisionerRoleName:
+		if err = r.EnsureClusterRole(ProvisionerRoleName); err != nil {
+			r.Log.Error(err, "Reconciliation for ClusterRole failed", "ClusterRole", ProvisionerRoleName)
+			break
+		}
+		if err = r.EnsureClusterRoleBinding(); err != nil {
+			r.Log.Error(err, "Reconciliation for ClusterRoleBinding failed", "ClusterRoleBinding", ProvisionerRoleName)
+			break
+		}
+	case DeleterRoleName:
+		if err = r.EnsureClusterRole(DeleterRoleName); err != nil {
+			r.Log.Error(err, "Reconciliation for ClusterRole failed", "ClusterRole", DeleterRoleName)
+			break
+		}
+	}
+	return reconcile.Result{}, err
+}
+
+func (r *Manager) EnsureClusterRoleBinding() (err error) {
+	crb := &rbacv1.ClusterRoleBinding{
+		ObjectMeta: v1.ObjectMeta{
+			Name: ProvisionerRoleName,
+		},
+	}
+
+	_, err = controllerutil.CreateOrUpdate(context.TODO(), r.Client, crb, func() error {
+		// RoleRef is immutable, so we need to delete and recreate ClusterRoleBinding if it changed
+		if crb.ResourceVersion != "" && !equality.Semantic.DeepDerivative(provisionerClusterRoleBinding.RoleRef, crb.RoleRef) {
+			return ImmutableClusterRoleBindingError{}
+		}
+		crb.RoleRef = provisionerClusterRoleBinding.RoleRef
+		crb.Subjects = []rbacv1.Subject{
+			{
+				Kind: "Group",
+				Name: r.CapsuleGroup,
+			},
+		}
+		return nil
+	})
+	if err != nil {
+		if _, ok := err.(ImmutableClusterRoleBindingError); ok {
+			if err = r.Client.Delete(context.TODO(), crb); err != nil {
+				r.Log.Error(err, "Cannot delete CRB during reset due to RoleRef change")
+				return
+			}
+			return r.Client.Create(context.TODO(), provisionerClusterRoleBinding, &client.CreateOptions{})
+		}
+		r.Log.Error(err, "Cannot CreateOrUpdate CRB")
+		return
+	}
+	return
+}
+
+func (r *Manager) EnsureClusterRole(roleName string) (err error) {
+	role, ok := clusterRoles[roleName]
+	if !ok {
+		return fmt.Errorf("ClusterRole %s is not mapped", roleName)
+	}
+	clusterRole := &rbacv1.ClusterRole{
+		ObjectMeta: v1.ObjectMeta{
+			Name: role.GetName(),
+		},
+	}
+	_, err = controllerutil.CreateOrUpdate(context.TODO(), r.Client, clusterRole, func() error {
+		clusterRole.Rules = role.Rules
+		return nil
+	})
+	return
+}
+
+// This is the Runnable function that is triggered upon Manager start-up to perform the first RBAC reconciliation
+// since we're not creating empty CR and CRB upon Capsule installation: it's a run-once task, since the reconciliation
+// is handled by the Reconciler implemented interface.
+func (r *Manager) Start(<-chan struct{}) (err error) {
+	for roleName := range clusterRoles {
+		r.Log.Info("setting up ClusterRoles", "ClusterRole", roleName)
+		if err = r.EnsureClusterRole(roleName); err != nil {
+			return
+		}
+	}
+	r.Log.Info("setting up ClusterRoleBindings", "ClusterRoleBinding", ProvisionerRoleName)
+	err = r.EnsureClusterRoleBinding()
+	return
+}

controllers/tenant_controller.go

@@ -43,6 +43,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	capsulev1alpha1 "github.com/clastix/capsule/api/v1alpha1"
+	"github.com/clastix/capsule/controllers/rbac"
 )
 
 // TenantReconciler reconciles a Tenant object
@@ -516,7 +517,7 @@ func (r *TenantReconciler) ownerRoleBinding(tenant *capsulev1alpha1.Tenant) erro
 		rbl[types.NamespacedName{Namespace: i, Name: "namespace:deleter"}] = rbacv1.RoleRef{
 			APIGroup: "rbac.authorization.k8s.io",
 			Kind:     "ClusterRole",
-			Name:     "capsule-namespace:deleter",
+			Name:     rbac.DeleterRoleName,
 		}
 	}
 

e2e/custom_capsule_group_test.go

@@ -0,0 +1,69 @@
+//+build e2e
+
+/*
+Copyright 2020 Clastix Labs.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/clastix/capsule/api/v1alpha1"
+)
+
+var _ = Describe("creating a Namespace as Tenant owner with custom --capsule-group", func() {
+	tnt := &v1alpha1.Tenant{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "tenant-assigned-custom-group",
+		},
+		Spec: v1alpha1.TenantSpec{
+			Owner:          "alice",
+			StorageClasses: []string{},
+			IngressClasses: []string{},
+			LimitRanges:    []corev1.LimitRangeSpec{},
+			NamespaceQuota: 10,
+			NodeSelector:   map[string]string{},
+			ResourceQuota:  []corev1.ResourceQuotaSpec{},
+		},
+	}
+	JustBeforeEach(func() {
+		tnt.ResourceVersion = ""
+		Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
+	})
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
+	})
+	It("should fail", func() {
+		args := append(defaulManagerPodArgs, []string{"--capsule-user-group=test"}...)
+		ModifyCapsuleManagerPodArgs(args)
+		CapsuleClusterGroupParamShouldBeUpdated("test")
+		ns := NewNamespace("cg-namespace-fail")
+		NamespaceCreationShouldNotSucceed(ns, tnt)
+	})
+	It("should succeed and be available in Tenant namespaces list", func() {
+		ModifyCapsuleManagerPodArgs(defaulManagerPodArgs)
+		CapsuleClusterGroupParamShouldBeUpdated("capsule.clastix.io")
+		ns := NewNamespace("cg-namespace")
+		NamespaceCreationShouldSucceed(ns, tnt)
+		NamespaceShouldBeManagedByTenant(ns, tnt)
+
+	})
+})

e2e/suite_test.go

@@ -19,11 +19,14 @@ limitations under the License.
 package e2e
 
 import (
+	"context"
 	"path/filepath"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
@@ -40,9 +43,18 @@ import (
 // These tests use Ginkgo (BDD-style Go testing framework). Refer to
 // http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
 
-var cfg *rest.Config
-var k8sClient client.Client
-var testEnv *envtest.Environment
+var (
+	cfg                  *rest.Config
+	k8sClient            client.Client
+	testEnv              *envtest.Environment
+	defaulManagerPodArgs []string
+)
+
+const (
+	capsuleDeploymentName       = "capsule-controller-manager"
+	capsuleNamespace            = "capsule-system"
+	capsuleManagerContainerName = "manager"
+)
 
 func TestAPIs(t *testing.T) {
 	RegisterFailHandler(Fail)
@@ -75,6 +87,15 @@ var _ = BeforeSuite(func(done Done) {
 	Expect(err).ToNot(HaveOccurred())
 	Expect(k8sClient).ToNot(BeNil())
 
+	capsuleDeployment := &appsv1.Deployment{}
+	k8sClient.Get(context.TODO(), types.NamespacedName{Name: capsuleDeploymentName, Namespace: capsuleNamespace}, capsuleDeployment)
+	for _, container := range capsuleDeployment.Spec.Template.Spec.Containers {
+		if container.Name == capsuleManagerContainerName {
+			defaulManagerPodArgs = container.Args
+		}
+	}
+	Expect(defaulManagerPodArgs).ToNot(BeEmpty())
+
 	close(done)
 }, 60)
 

e2e/utils_test.go

@@ -23,15 +23,18 @@ import (
 	"time"
 
 	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/clastix/capsule/api/v1alpha1"
 )
 
 const (
-	defaultTimeoutInterval = 10 * time.Second
+	defaultTimeoutInterval = 25 * time.Second
 	defaultPollInterval    = time.Second
 )
 
@@ -51,9 +54,61 @@ func NamespaceCreationShouldSucceed(ns *corev1.Namespace, t *v1alpha1.Tenant) {
 	}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
 }
 
+func NamespaceCreationShouldNotSucceed(ns *corev1.Namespace, t *v1alpha1.Tenant) {
+	cs := ownerClient(t)
+	Eventually(func() (err error) {
+		_, err = cs.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
+		return
+	}, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed())
+}
+
 func NamespaceShouldBeManagedByTenant(ns *corev1.Namespace, t *v1alpha1.Tenant) {
 	Eventually(func() v1alpha1.NamespaceList {
 		Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: t.GetName()}, t)).Should(Succeed())
 		return t.Status.Namespaces
 	}, defaultTimeoutInterval, defaultPollInterval).Should(ContainElement(ns.GetName()))
 }
+
+func CapsuleClusterGroupParamShouldBeUpdated(capsuleClusterGroup string) {
+	capsuleCRB := &rbacv1.ClusterRoleBinding{}
+
+	Eventually(func() string {
+		Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: "capsule-namespace:provisioner"}, capsuleCRB)).Should(Succeed())
+		return capsuleCRB.Subjects[0].Name
+	}, defaultTimeoutInterval, defaultPollInterval).Should(BeIdenticalTo(capsuleClusterGroup))
+
+}
+
+func ModifyCapsuleManagerPodArgs(args []string) {
+	capsuleDeployment := &appsv1.Deployment{}
+	k8sClient.Get(context.TODO(), types.NamespacedName{Name: capsuleDeploymentName, Namespace: capsuleNamespace}, capsuleDeployment)
+	for i, container := range capsuleDeployment.Spec.Template.Spec.Containers {
+		if container.Name == capsuleManagerContainerName {
+			capsuleDeployment.Spec.Template.Spec.Containers[i].Args = args
+		}
+	}
+	capsuleDeployment.ResourceVersion = ""
+	err := k8sClient.Update(context.TODO(), capsuleDeployment)
+	Expect(err).ToNot(HaveOccurred())
+
+	Eventually(func() []string {
+		var containerArgs []string
+		Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: capsuleDeploymentName, Namespace: capsuleNamespace}, capsuleDeployment)).Should(Succeed())
+		for i, container := range capsuleDeployment.Spec.Template.Spec.Containers {
+			if container.Name == capsuleManagerContainerName {
+				containerArgs = capsuleDeployment.Spec.Template.Spec.Containers[i].Args
+			}
+		}
+		return containerArgs
+	}, defaultTimeoutInterval, defaultPollInterval).Should(HaveLen(len(args)))
+
+	pl := &corev1.PodList{}
+	Eventually(func() []corev1.Pod {
+		Expect(k8sClient.List(context.TODO(), pl, client.MatchingLabels{"control-plane": "controller-manager"})).Should(Succeed())
+		return pl.Items
+	}, defaultTimeoutInterval, defaultPollInterval).Should(HaveLen(2))
+	Eventually(func() []corev1.Pod {
+		Expect(k8sClient.List(context.TODO(), pl, client.MatchingLabels{"control-plane": "controller-manager"})).Should(Succeed())
+		return pl.Items
+	}, defaultTimeoutInterval, defaultPollInterval).Should(HaveLen(1))
+}

go.mod

@@ -8,6 +8,8 @@ require (
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.8.1
 	github.com/stretchr/testify v1.4.0
+	golang.org/x/net v0.0.0-20200625001655-4c5254603344 // indirect
+	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
 	k8s.io/api v0.18.2
 	k8s.io/apimachinery v0.18.2
 	k8s.io/client-go v0.18.2

go.sum

@@ -300,6 +300,8 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo=
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -323,6 +325,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
@@ -349,6 +353,8 @@ golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU=
 golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -370,9 +376,12 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72 h1:bw9doJza/SFBEweII/rHQh338oozWyiFsBRHtrflcws=
 golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0=
 gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -400,6 +409,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

main.go

@@ -27,12 +27,14 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	capsulev1alpha1 "github.com/clastix/capsule/api/v1alpha1"
 	"github.com/clastix/capsule/controllers"
+	"github.com/clastix/capsule/controllers/rbac"
 	"github.com/clastix/capsule/controllers/secret"
 	"github.com/clastix/capsule/pkg/indexer"
 	"github.com/clastix/capsule/pkg/webhook"
@@ -70,8 +72,10 @@ func main() {
 	var enableLeaderElection bool
 	var forceTenantPrefix bool
 	var v bool
+	var capsuleGroup string
 
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&capsuleGroup, "capsule-user-group", capsulev1alpha1.GroupVersion.Group, "Name of the group for capsule users")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -119,12 +123,25 @@ func main() {
 	//webhooks
 	wl := make([]webhook.Webhook, 0)
 	wl = append(wl, &ingress.ExtensionIngress{}, &ingress.NetworkIngress{}, pvc.Webhook{}, &owner_reference.Webhook{}, &namespace_quota.Webhook{}, network_policies.Webhook{}, tenant_prefix.Webhook{ForceTenantPrefix: forceTenantPrefix})
-	err = webhook.Register(mgr, wl...)
+	err = webhook.Register(mgr, capsuleGroup, wl...)
 	if err != nil {
 		setupLog.Error(err, "unable to setup webhooks")
 		os.Exit(1)
 	}
 
+	rbacManager := &rbac.Manager{
+		Log:          ctrl.Log.WithName("controllers").WithName("Rbac"),
+		CapsuleGroup: capsuleGroup,
+	}
+	if err := mgr.Add(rbacManager); err != nil {
+		setupLog.Error(err, "unable to create cluster roles")
+		os.Exit(1)
+	}
+	if err = rbacManager.SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "Rbac")
+		os.Exit(1)
+	}
+
 	if err = (&secret.CaReconciler{
 		Client: mgr.GetClient(),
 		Log:    ctrl.Log.WithName("controllers").WithName("CA"),

pkg/utils/user_group.go

@@ -19,8 +19,6 @@ package utils
 import (
 	"sort"
 	"strings"
-
-	"github.com/clastix/capsule/api/v1alpha1"
 )
 
 type UserGroupList []string
@@ -37,11 +35,9 @@ func (u UserGroupList) Swap(i, j int) {
 	u[i], u[j] = u[j], u[i]
 }
 
-func (u UserGroupList) IsInCapsuleGroup() (ok bool) {
-	v := v1alpha1.GroupVersion.Group
-
+func (u UserGroupList) IsInCapsuleGroup(capsuleGroup string) (ok bool) {
 	sort.Sort(u)
-	i := sort.SearchStrings(u, v)
-	ok = i < u.Len() && u[i] == v
+	i := sort.SearchStrings(u, capsuleGroup)
+	ok = i < u.Len() && u[i] == capsuleGroup
 	return
 }

pkg/webhook/router.go

@@ -29,7 +29,7 @@ import (
 	"github.com/clastix/capsule/pkg/utils"
 )
 
-func Register(mgr controllerruntime.Manager, webhookList ...Webhook) error {
+func Register(mgr controllerruntime.Manager, capsuleGroup string, webhookList ...Webhook) error {
 	// skipping webhook setup if certificate is missing
 	dat, _ := ioutil.ReadFile("/tmp/k8s-webhook-server/serving-certs/tls.crt")
 	if len(dat) == 0 {
@@ -40,7 +40,8 @@ func Register(mgr controllerruntime.Manager, webhookList ...Webhook) error {
 	for _, wh := range webhookList {
 		s.Register(wh.GetPath(), &webhook.Admission{
 			Handler: &handlerRouter{
-				handler: wh.GetHandler(),
+				handler:      wh.GetHandler(),
+				capsuleGroup: capsuleGroup,
 			},
 		})
 	}
@@ -48,13 +49,14 @@ func Register(mgr controllerruntime.Manager, webhookList ...Webhook) error {
 }
 
 type handlerRouter struct {
-	handler Handler
-	client  client.Client
-	decoder *admission.Decoder
+	handler      Handler
+	capsuleGroup string
+	client       client.Client
+	decoder      *admission.Decoder
 }
 
 func (r *handlerRouter) Handle(ctx context.Context, req admission.Request) admission.Response {
-	if !utils.UserGroupList(req.UserInfo.Groups).IsInCapsuleGroup() {
+	if !utils.UserGroupList(req.UserInfo.Groups).IsInCapsuleGroup(r.capsuleGroup) {
 		// not a Capsule user, can be skipped
 		return admission.Allowed("")
 	}