Support custom RBAC resources (#572)

Motivation: Allow users to change the default RBAC resources (ServiceAccount, ClusterRole, ClusterRoleBinding, Role and RoleBinding) without having Mizu delete them every run.

Adds app.kubernetes.io/created-by and app.kubernetes.io/managed-by labels to all resources.
The value of app.kubernetes.io/created-by is either mizu-cli or mizu-agent.
The value of app.kubernetes.io/managed-by is mizu.
When Mizu cleans resources (ctrl-c in tap cmd or mizu clean cmd) it removes all RBAC resources that have managed-by=mizu, and only those.

A user may have a ClusterRole named mizu-clusterrole. If it doesn't have the label app.kubernetes.io/managed-by=mizu, then Mizu won't overwrite it and won't delete it.
Other resources (deployments, services etc.) are always removed, regardless of their labels.

README.md

@@ -52,8 +52,8 @@ Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page
 ## How to Run
 
 1. Find pods you'd like to tap to in your Kubernetes cluster
-2. Run `mizu tap` or `mizu tap PODNAME`  
-3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI
+2. Run `mizu tap` or `mizu tap PODNAME`
+3. Open browser on `http://localhost:8899` **or** as instructed in the CLI
 4. Watch the API traffic flowing
 5. Type ^C to stop
 
@@ -172,12 +172,10 @@ Please see [CONTRACT MONITORING](docs/CONTRACT_MONITORING.md) page for more deta
 
 ### Configure proxy host 
 
-By default, mizu will be accessible via local host: 'http://localhost:8899/mizu/', it is possible to change the host,
-for instance, to '0.0.0.0' which can grant access via machine IP address.
-This setting can be changed via command line flag `--set tap.proxy-host=<value>` or via config file:
-tap
-    proxy-host: 0.0.0.0
-and when changed it will support accessing by IP
+By default, mizu will be accessible via local host: 'http://localhost:8899', it is possible to change the host, for
+instance, to '0.0.0.0' which can grant access via machine IP address. This setting can be changed via command line
+flag `--set tap.proxy-host=<value>` or via config file:
+tap proxy-host: 0.0.0.0 and when changed it will support accessing by IP
 
 ### Install Mizu standalone
 

agent/main.go

@@ -24,6 +24,8 @@ import (
 	"plugin"
 	"regexp"
 	"sort"
+	"strconv"
+	"strings"
 	"syscall"
 	"time"
 
@@ -59,6 +61,7 @@ const (
 	socketConnectionRetries    = 30
 	socketConnectionRetryDelay = time.Second * 2
 	socketHandshakeTimeout     = time.Second * 2
+	uiIndexPath                = "./site/index.html"
 )
 
 func main() {
@@ -251,7 +254,12 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 	}
 
 	app.Use(DisableRootStaticCache())
+
+	if err := setUIMode(); err != nil {
+		logger.Log.Panicf("Error setting ui mode, err: %v", err)
+	}
 	app.Use(static.ServeRoot("/", "./site"))
+
 	app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
 
 	api.WebSocketRoutes(app, &eventHandlers, startTime)
@@ -291,6 +299,22 @@ func CORSMiddleware() gin.HandlerFunc {
 	}
 }
 
+func setUIMode() error {
+	read, err := ioutil.ReadFile(uiIndexPath)
+	if err != nil {
+		return err
+	}
+
+	replacedContent := strings.Replace(string(read), "__IS_STANDALONE__", strconv.FormatBool(config.Config.StandaloneMode), 1)
+
+	err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func parseEnvVar(env string) map[string][]v1.Pod {
 	var mapOfList map[string][]v1.Pod
 

cli/cmd/installRunner.go

@@ -65,6 +65,7 @@ func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Reso
 		TapperResources:        tapperResources,
 		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
 		AgentDatabasePath:      shared.DataDirPath,
+		StandaloneMode:         true,
 	}
 
 	return &mizuAgentConfig

cli/resources/cleanResources.go

@@ -42,14 +42,29 @@ func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, ku
 		defer waitUntilNamespaceDeleted(ctx, cancel, kubernetesProvider, mizuResourcesNamespace)
 	}
 
-	if err := kubernetesProvider.RemoveClusterRole(ctx, kubernetes.ClusterRoleName); err != nil {
-		resourceDesc := fmt.Sprintf("ClusterRole %s", kubernetes.ClusterRoleName)
+	if resources, err := kubernetesProvider.ListManagedClusterRoles(ctx); err != nil {
+		resourceDesc := "ClusterRoles"
 		handleDeletionError(err, resourceDesc, &leftoverResources)
+	} else {
+		for _, resource := range resources.Items {
+			if err := kubernetesProvider.RemoveClusterRole(ctx, resource.Name); err != nil {
+				resourceDesc := fmt.Sprintf("ClusterRole %s", resource.Name)
+				handleDeletionError(err, resourceDesc, &leftoverResources)
+			}
+		}
 	}
 
-	if err := kubernetesProvider.RemoveClusterRoleBinding(ctx, kubernetes.ClusterRoleBindingName); err != nil {
-		resourceDesc := fmt.Sprintf("ClusterRoleBinding %s", kubernetes.ClusterRoleBindingName)
+
+	if resources, err := kubernetesProvider.ListManagedClusterRoleBindings(ctx); err != nil {
+		resourceDesc := "ClusterRoleBindings"
 		handleDeletionError(err, resourceDesc, &leftoverResources)
+	} else {
+		for _, resource := range resources.Items {
+			if err := kubernetesProvider.RemoveClusterRoleBinding(ctx, resource.Name); err != nil {
+				resourceDesc := fmt.Sprintf("ClusterRoleBinding %s", resource.Name)
+				handleDeletionError(err, resourceDesc, &leftoverResources)
+			}
+		}
 	}
 
 	return leftoverResources
@@ -91,14 +106,40 @@ func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.P
 		handleDeletionError(err, resourceDesc, &leftoverResources)
 	}
 
-	if err := kubernetesProvider.RemoveServicAccount(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName); err != nil {
-		resourceDesc := fmt.Sprintf("Service Account %s in namespace %s", kubernetes.ServiceAccountName, mizuResourcesNamespace)
+	if resources, err := kubernetesProvider.ListManagedServiceAccounts(ctx, mizuResourcesNamespace); err != nil {
+		resourceDesc := fmt.Sprintf("ServiceAccounts in namespace %s", mizuResourcesNamespace)
 		handleDeletionError(err, resourceDesc, &leftoverResources)
+	} else {
+		for _, resource := range resources.Items {
+			if err := kubernetesProvider.RemoveServicAccount(ctx, mizuResourcesNamespace, resource.Name); err != nil {
+				resourceDesc := fmt.Sprintf("ServiceAccount %s in namespace %s", resource.Name, mizuResourcesNamespace)
+				handleDeletionError(err, resourceDesc, &leftoverResources)
+			}
+		}
 	}
 
-	if err := kubernetesProvider.RemoveRole(ctx, mizuResourcesNamespace, kubernetes.RoleName); err != nil {
-		resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.RoleName, mizuResourcesNamespace)
+	if resources, err := kubernetesProvider.ListManagedRoles(ctx, mizuResourcesNamespace); err != nil {
+		resourceDesc := fmt.Sprintf("Roles in namespace %s", mizuResourcesNamespace)
 		handleDeletionError(err, resourceDesc, &leftoverResources)
+	} else {
+		for _, resource := range resources.Items {
+			if err := kubernetesProvider.RemoveRole(ctx, mizuResourcesNamespace, resource.Name); err != nil {
+				resourceDesc := fmt.Sprintf("Role %s in namespace %s", resource.Name, mizuResourcesNamespace)
+				handleDeletionError(err, resourceDesc, &leftoverResources)
+			}
+		}
+	}
+
+	if resources, err := kubernetesProvider.ListManagedRoleBindings(ctx, mizuResourcesNamespace); err != nil {
+		resourceDesc := fmt.Sprintf("RoleBindings in namespace %s", mizuResourcesNamespace)
+		handleDeletionError(err, resourceDesc, &leftoverResources)
+	} else {
+		for _, resource := range resources.Items {
+			if err := kubernetesProvider.RemoveRoleBinding(ctx, mizuResourcesNamespace, resource.Name); err != nil {
+				resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", resource.Name, mizuResourcesNamespace)
+				handleDeletionError(err, resourceDesc, &leftoverResources)
+			}
+		}
 	}
 
 	if err := kubernetesProvider.RemovePod(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
@@ -107,10 +148,6 @@ func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.P
 	}
 
 	//install mode resources
-	if err := kubernetesProvider.RemoveRoleBinding(ctx, mizuResourcesNamespace, kubernetes.RoleBindingName); err != nil {
-		resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.RoleBindingName, mizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
 
 	if err := kubernetesProvider.RemoveDeployment(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
 		resourceDesc := fmt.Sprintf("Deployment %s in namespace %s", kubernetes.ApiServerPodName, mizuResourcesNamespace)
@@ -122,16 +159,6 @@ func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.P
 		handleDeletionError(err, resourceDesc, &leftoverResources)
 	}
 
-	if err := kubernetesProvider.RemoveRole(ctx, mizuResourcesNamespace, kubernetes.DaemonRoleName); err != nil {
-		resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.DaemonRoleName, mizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveRoleBinding(ctx, mizuResourcesNamespace, kubernetes.DaemonRoleBindingName); err != nil {
-		resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.DaemonRoleBindingName, mizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
 	return leftoverResources
 }
 

shared/kubernetes/consts.go

@@ -17,3 +17,12 @@ const (
 	PersistentVolumeClaimName  = MizuResourcesPrefix + "volume-claim"
 	MinKubernetesServerVersion = "1.16.0"
 )
+
+const (
+	LabelPrefixApp      = "app.kubernetes.io/"
+	LabelManagedBy      = LabelPrefixApp + "managed-by"
+	LabelCreatedBy      = LabelPrefixApp + "created-by"
+	LabelValueMizu      = "mizu"
+	LabelValueMizuCLI   = "mizu-cli"
+	LabelValueMizuAgent = "mizu-agent"
+)

shared/kubernetes/provider.go

@@ -43,6 +43,8 @@ type Provider struct {
 	kubernetesConfig clientcmd.ClientConfig
 	clientConfig     restclient.Config
 	Namespace        string
+	managedBy   string
+	createdBy   string
 }
 
 const (
@@ -86,6 +88,8 @@ func NewProvider(kubeConfigPath string) (*Provider, error) {
 		clientSet:        clientSet,
 		kubernetesConfig: kubernetesConfig,
 		clientConfig:     *restClientConfig,
+		managedBy:        LabelValueMizu,
+		createdBy:        LabelValueMizuCLI,
 	}, nil
 }
 
@@ -103,6 +107,8 @@ func NewProviderInCluster() (*Provider, error) {
 		clientSet:        clientSet,
 		kubernetesConfig: nil, // not relevant in cluster
 		clientConfig:     *restClientConfig,
+		managedBy:        LabelValueMizu,
+		createdBy:        LabelValueMizuAgent,
 	}, nil
 }
 
@@ -158,6 +164,10 @@ func (provider *Provider) CreateNamespace(ctx context.Context, name string) (*co
 	namespaceSpec := &core.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name,
+			Labels: map[string]string{
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 	}
 	return provider.clientSet.CoreV1().Namespaces().Create(ctx, namespaceSpec, metav1.CreateOptions{})
@@ -243,7 +253,11 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 	pod := &core.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   opts.PodName,
-			Labels: map[string]string{"app": opts.PodName},
+			Labels: map[string]string{
+				"app": opts.PodName,
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 		Spec: core.PodSpec{
 			Containers: []core.Container{
@@ -303,6 +317,10 @@ func (provider *Provider) CreateDeployment(ctx context.Context, namespace string
 	deployment := &v1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: deploymentName,
+			Labels: map[string]string{
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 		Spec: v1.DeploymentSpec{
 			Selector: &metav1.LabelSelector{
@@ -319,6 +337,10 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s
 	service := core.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: serviceName,
+			Labels: map[string]string{
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 		Spec: core.ServiceSpec{
 			Ports:    []core.ServicePort{{TargetPort: intstr.FromInt(shared.DefaultApiServerPort), Port: 80}},
@@ -351,13 +373,21 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
 	serviceAccount := &core.ServiceAccount{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   serviceAccountName,
-			Labels: map[string]string{"mizu-cli-version": version},
+			Labels: map[string]string{
+				"mizu-cli-version": version,
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 	}
 	clusterRole := &rbac.ClusterRole{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   clusterRoleName,
-			Labels: map[string]string{"mizu-cli-version": version},
+			Labels: map[string]string{
+				"mizu-cli-version": version,
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 		Rules: []rbac.PolicyRule{
 			{
@@ -370,7 +400,11 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
 	clusterRoleBinding := &rbac.ClusterRoleBinding{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   clusterRoleBindingName,
-			Labels: map[string]string{"mizu-cli-version": version},
+			Labels: map[string]string{
+				"mizu-cli-version": version,
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 		RoleRef: rbac.RoleRef{
 			Name:     clusterRoleName,
@@ -404,13 +438,21 @@ func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context,
 	serviceAccount := &core.ServiceAccount{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   serviceAccountName,
-			Labels: map[string]string{"mizu-cli-version": version},
+			Labels: map[string]string{
+				"mizu-cli-version": version,
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 	}
 	role := &rbac.Role{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   roleName,
-			Labels: map[string]string{"mizu-cli-version": version},
+			Labels: map[string]string{
+				"mizu-cli-version": version,
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 		Rules: []rbac.PolicyRule{
 			{
@@ -423,7 +465,11 @@ func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context,
 	roleBinding := &rbac.RoleBinding{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   roleBindingName,
-			Labels: map[string]string{"mizu-cli-version": version},
+			Labels: map[string]string{
+				"mizu-cli-version": version,
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 		RoleRef: rbac.RoleRef{
 			Name:     roleName,
@@ -457,7 +503,11 @@ func (provider *Provider) CreateDaemonsetRBAC(ctx context.Context, namespace str
 	role := &rbac.Role{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   roleName,
-			Labels: map[string]string{"mizu-cli-version": version},
+			Labels: map[string]string{
+				"mizu-cli-version": version,
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 		Rules: []rbac.PolicyRule{
 			{
@@ -475,7 +525,11 @@ func (provider *Provider) CreateDaemonsetRBAC(ctx context.Context, namespace str
 	roleBinding := &rbac.RoleBinding{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   roleBindingName,
-			Labels: map[string]string{"mizu-cli-version": version},
+			Labels: map[string]string{
+				"mizu-cli-version": version,
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 		RoleRef: rbac.RoleRef{
 			Name:     roleName,
@@ -588,6 +642,10 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: configMapName,
+			Labels: map[string]string{
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 		Data: configMapData,
 	}
@@ -746,14 +804,23 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	podSpec.WithVolumes(&configMapVolume, procfsVolume)
 
 	podTemplate := applyconfcore.PodTemplateSpec()
-	podTemplate.WithLabels(map[string]string{"app": tapperPodName})
+	podTemplate.WithLabels(map[string]string{
+		"app": tapperPodName,
+		LabelManagedBy: provider.managedBy,
+		LabelCreatedBy: provider.createdBy,
+	})
 	podTemplate.WithSpec(podSpec)
 
 	labelSelector := applyconfmeta.LabelSelector()
 	labelSelector.WithMatchLabels(map[string]string{"app": tapperPodName})
 
 	daemonSet := applyconfapp.DaemonSet(daemonSetName, namespace)
-	daemonSet.WithSpec(applyconfapp.DaemonSetSpec().WithSelector(labelSelector).WithTemplate(podTemplate))
+	daemonSet.
+		WithLabels(map[string]string{
+			LabelManagedBy: provider.managedBy,
+			LabelCreatedBy: provider.createdBy,
+		}).
+		WithSpec(applyconfapp.DaemonSetSpec().WithSelector(labelSelector).WithTemplate(podTemplate))
 
 	_, err = provider.clientSet.AppsV1().DaemonSets(namespace).Apply(ctx, daemonSet, metav1.ApplyOptions{FieldManager: fieldManagerName})
 	return err
@@ -827,6 +894,41 @@ func (provider *Provider) GetNamespaceEvents(ctx context.Context, namespace stri
 	return eventList.String(), nil
 }
 
+func (provider *Provider) ListManagedServiceAccounts(ctx context.Context, namespace string) (*core.ServiceAccountList, error) {
+	listOptions := metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy),
+	}
+	return provider.clientSet.CoreV1().ServiceAccounts(namespace).List(ctx, listOptions)
+}
+
+func (provider *Provider) ListManagedClusterRoles(ctx context.Context) (*rbac.ClusterRoleList, error) {
+	listOptions := metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy),
+	}
+	return provider.clientSet.RbacV1().ClusterRoles().List(ctx, listOptions)
+}
+
+func (provider *Provider) ListManagedClusterRoleBindings(ctx context.Context) (*rbac.ClusterRoleBindingList, error) {
+	listOptions := metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy),
+	}
+	return provider.clientSet.RbacV1().ClusterRoleBindings().List(ctx, listOptions)
+}
+
+func (provider *Provider) ListManagedRoles(ctx context.Context, namespace string) (*rbac.RoleList, error) {
+	listOptions := metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy),
+	}
+	return provider.clientSet.RbacV1().Roles(namespace).List(ctx, listOptions)
+}
+
+func (provider *Provider) ListManagedRoleBindings(ctx context.Context, namespace string) (*rbac.RoleBindingList, error) {
+	listOptions := metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy),
+	}
+	return provider.clientSet.RbacV1().RoleBindings(namespace).List(ctx, listOptions)
+}
+
 func (provider *Provider) IsDefaultStorageProviderAvailable(ctx context.Context) (bool, error) {
 	storageClassList, err := provider.clientSet.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{})
 	if err != nil {
@@ -845,6 +947,10 @@ func (provider *Provider) CreatePersistentVolumeClaim(ctx context.Context, names
 	volumeClaim := &core.PersistentVolumeClaim{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: volumeClaimName,
+			Labels: map[string]string{
+				LabelManagedBy: provider.managedBy,
+				LabelCreatedBy: provider.createdBy,
+			},
 		},
 		Spec: core.PersistentVolumeClaimSpec{
 			AccessModes: []core.PersistentVolumeAccessMode{core.ReadWriteOnce},

shared/models.go

@@ -40,6 +40,7 @@ type MizuAgentConfig struct {
 	TapperResources        Resources     `json:"tapperResources"`
 	MizuResourcesNamespace string        `json:"mizuResourceNamespace"`
 	AgentDatabasePath      string        `json:"agentDatabasePath"`
+	StandaloneMode         bool          `json:"standaloneMode"`
 }
 
 type WebSocketMessageMetadata struct {