Hotfix - ignore not allowed set flags (#192)

#patch

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,38 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Run mizu <command> '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Logs**
-Upload logs:
-1. Run the mizu command with `--set dump-logs=true` (e.g `mizu tap --set dump-logs=true`) 
-2. Try to reproduce the issue
-3. CNTRL+C on terminal tab which runs mizu
-4. Upload the logs zip file from ~/.mizu/mizu_logs_**.zip
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Browser [e.g. chrome]
-
-**Additional context**
-Add any other context about the problem here.

.github/workflows/test.yaml

@@ -1,22 +0,0 @@
-name: test
-on:
-  pull_request:
-    branches:
-      - 'develop'
-      - 'main'
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Go 1.x
-        uses: actions/setup-go@v2
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Test
-        run: make test
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2

Dockerfile

@@ -48,6 +48,8 @@ WORKDIR /app
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 
+COPY agent/start.sh .
+
 # gin-gonic runs in debug mode without this
 ENV GIN_MODE=release
 

Makefile

@@ -65,5 +65,3 @@ clean-cli:  ## Clean CLI.
 clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
-test: ## Run tests.
-	@echo "running cli tests"; cd cli && $(MAKE) test

README.md

@@ -1,17 +1,16 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
-
 # The API Traffic Viewer for Kubernetes
 
-A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined
+A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined.
 
 ![Simple UI](assets/mizu-ui.png)
 
 ## Features
 
 - Simple and powerful CLI
-- Real-time view of all HTTP requests, REST and gRPC API calls
+- Real time view of all HTTP requests, REST and gRPC API calls
 - No installation or code instrumentation
-- Works completely on premises
+- Works completely on premises (on-prem)
 
 ## Download
 
@@ -33,10 +32,10 @@ https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 \
 && chmod 755 mizu
 ``` 
 
-SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page
+SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page.
 
 ### Development (unstable) Build
-Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page
+Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page.
 
 ## Prerequisites
 1. Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
@@ -49,8 +48,8 @@ For detailed list of k8s permissions see [PERMISSIONS](PERMISSIONS.md) document
 
 1. Find pods you'd like to tap to in your Kubernetes cluster
 2. Run `mizu tap` or `mizu tap PODNAME`  
-3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI
-4. Watch the API traffic flowing
+3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI .. 
+4. Watch the API traffic flowing ..
 5. Type ^C to stop
 
 ## Examples
@@ -76,7 +75,7 @@ To tap all pods in current namespace -
 
 
 To tap specific pod - 
-```bash
+``` 
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
  front-end-649fc5fd6-kqbtn       2/2     Running   0          7m
@@ -89,7 +88,7 @@ To tap specific pod -
 ```
 
 To tap multiple pods using regex - 
-```bash
+``` 
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
  carts-66c77f5fbb-fq65r          2/2     Running   0          20m
@@ -107,42 +106,30 @@ To tap multiple pods using regex -
 ## Configuration
 
 Mizu can work with config file which should be stored in ${HOME}/.mizu/config.yaml (macOS: ~/.mizu/config.yaml) <br />
-In case no config file found, defaults will be used <br />
-In case of partial configuration defined, all other fields will be used with defaults <br />
-You can always override the defaults or config file with CLI flags
+In case no config file found, defaults will be used. <br />
+In case of partial configuration defined, all other fields will be used with defaults. <br />
+You can always override the defaults or config file with CLI flags.
 
 To get the default config params run `mizu config` <br />
 To generate a new config file with default values use `mizu config -r`
 
-### Telemetry
-
-By default, mizu reports usage telemetry. It can be disabled by adding a line of telemetry: false in the ${HOME}/.mizu/config.yaml file
+Mizu has several undocumented flags which can be set by using --set flag (e.g., `mizu tap --set dump-logs=true`)
+* **mizu-resources-namespace**: Type - String, See [Namespace-Restricted Mode](#namespace-restricted-mode)
+* **telemetry**: Type - Boolean, Reports telemetry
+* **dump-logs**: Type - Boolean, At the end of the execution it creates a zip file with logs (in .mizu folder)
+* **kube-config-path**: Type - String, Setting the path to kube config (which isn't in standard path)
 
 ## Advanced Usage
 
 ### Namespace-Restricted Mode
 
-Some users have permission to only manage resources in one particular namespace assigned to them
+Some users have permission to only manage resources in one particular namespace assigned to them.
 By default `mizu tap` creates a new namespace `mizu` for all of its Kubernetes resources. In order to instead install
-Mizu in an existing namespace, set the `mizu-resources-namespace` config option
+Mizu in an existing namespace, set the `mizu-resources-namespace` config option.
 
 If `mizu-resources-namespace` is set to a value other than the default `mizu`, Mizu will operate in a
 Namespace-Restricted mode. It will only tap pods in `mizu-resources-namespace`. This way Mizu only requires permissions
 to the namespace set by `mizu-resources-namespace`. The user must set the tapped namespace to the same namespace by
-using the `--namespace` flag or by setting `tap.namespaces` in the config file
+using the `--namespace` flag or by setting `tap.namespaces` in the config file.
 
-Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
-
-### User agent filtering
-
-User-agent filtering (like health checks) - can be configured:
-
-Any request that contains one of those values in the user-agent header will not be captured
-
-```bash
-$ mizu tap "^ca.*" --set ignored-user-agents=kube-probe --set ignored-user-agents=prometheus
-+carts-66c77f5fbb-fq65r
-+catalogue-5f4cb7cf5-7zrmn
-Web interface is now available at http://localhost:8899
-^C
-```
+Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior.

agent/go.sum

@@ -540,9 +540,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=
 gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw=
 gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=

agent/main.go

@@ -149,13 +149,15 @@ func getTrafficFilteringOptions() *shared.TrafficFilteringOptions {
 	return &filteringOptions
 }
 
+var userAgentsToFilter = []string{"kube-probe", "prometheus"}
+
 func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) {
 	for message := range inChannel {
 		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
 			continue
 		}
 		// TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441
-		if isHealthCheckByUserAgent(message, filterOptions.HealthChecksUserAgentHeaders) {
+		if filterOptions.HideHealthChecks && isHealthCheckByUserAgent(message) {
 			continue
 		}
 
@@ -167,11 +169,11 @@ func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *ta
 	}
 }
 
-func isHealthCheckByUserAgent(message *tap.OutputChannelItem, userAgentsToIgnore []string) bool {
+func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool {
 	for _, header := range message.HarEntry.Request.Headers {
 		if strings.ToLower(header.Name) == "user-agent" {
-			for _, userAgent := range userAgentsToIgnore {
-				if strings.Contains(strings.ToLower(header.Value), strings.ToLower(userAgent)) {
+			for _, userAgent := range userAgentsToFilter {
+				if strings.Contains(strings.ToLower(header.Value), userAgent) {
 					return true
 				}
 			}

agent/pkg/models/models.go

@@ -56,14 +56,12 @@ type BaseEntryDetails struct {
 type ApplicableRules struct {
 	Latency int64 `json:"latency,omitempty"`
 	Status  bool  `json:"status,omitempty"`
-	NumberOfRules int `json:"numberOfRules,omitempty"`
 }
 
-func NewApplicableRules(status bool, latency int64, number int) ApplicableRules {
+func NewApplicableRules(status bool, latency int64) ApplicableRules {
 	ar := ApplicableRules{}
 	ar.Status = status
 	ar.Latency = latency
-	ar.NumberOfRules = number
 	return ar
 }
 
@@ -220,7 +218,7 @@ func (fewp *FullEntryWithPolicy) UnmarshalData(entry *MizuEntry) error {
 
 func RunValidationRulesState(harEntry har.Entry, service string) ApplicableRules {
 	numberOfRules, resultPolicyToSend := rules.MatchRequestPolicy(harEntry, service)
-	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend, numberOfRules)
-	ar := NewApplicableRules(statusPolicyToSend, latency, numberOfRules)
+	statusPolicyToSend, latency := rules.PassedValidationRules(resultPolicyToSend, numberOfRules)
+	ar := NewApplicableRules(statusPolicyToSend, latency)
 	return ar
 }

agent/pkg/rules/models.go

@@ -92,19 +92,19 @@ func MatchRequestPolicy(harEntry har.Entry, service string) (int, []RulesMatched
 	return len(enforcePolicy.Rules), resultPolicyToSend
 }
 
-func PassedValidationRules(rulesMatched []RulesMatched, numberOfRules int) (bool, int64, int) {
+func PassedValidationRules(rulesMatched []RulesMatched, numberOfRules int) (bool, int64) {
 	if len(rulesMatched) == 0 {
-		return false, 0, 0
+		return false, 0
 	}
 	for _, rule := range rulesMatched {
 		if rule.Matched == false {
-			return false, -1, len(rulesMatched)
+			return false, -1
 		}
 	}
 	for _, rule := range rulesMatched {
 		if strings.ToLower(rule.Rule.Type) == "latency" {
-			return true, rule.Rule.Latency, len(rulesMatched)
+			return true, rule.Rule.Latency
 		}
 	}
-	return true, -1, len(rulesMatched)
+	return true, -1
 }

agent/start.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+./mizuagent -i any -hardump -targets ${TAPPED_ADDRESSES}

cli/Makefile

@@ -39,6 +39,3 @@ build-all: ## Build for all supported platforms.
 clean: ## Clean all build artifacts.
 	go clean
 	rm -rf ./bin/*
-
-test: ## Run cli tests.
-	@go test ./... -race -coverprofile=coverage.out -covermode=atomic

cli/cmd/tap.go

@@ -60,10 +60,11 @@ func init() {
 	defaults.Set(&defaultTapConfig)
 
 	tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
-	tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")
+	tapCmd.Flags().StringArrayP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")
 	tapCmd.Flags().Bool(configStructs.AnalysisTapName, defaultTapConfig.Analysis, "Uploads traffic to UP9 for further analysis (Beta)")
 	tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
-	tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
+	tapCmd.Flags().StringArrayP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
+	tapCmd.Flags().Bool(configStructs.HideHealthChecksTapName, defaultTapConfig.HideHealthChecks, "Hides requests with kube-probe or prometheus user-agent headers")
 	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
 	tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any")

cli/cmd/tapRunner.go

@@ -70,7 +70,7 @@ func RunMizuTap() {
 	targetNamespaces := getNamespaces(kubernetesProvider)
 
 	var namespacesStr string
-	if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
+	if targetNamespaces[0] != mizu.K8sAllNamespaces {
 		namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(targetNamespaces, "\", \""))
 	} else {
 		namespacesStr = "all namespaces"
@@ -85,7 +85,7 @@ func RunMizuTap() {
 
 	if len(state.currentlyTappedPods) == 0 {
 		var suggestionStr string
-		if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
+		if targetNamespaces[0] != mizu.K8sAllNamespaces {
 			suggestionStr = ". Select a different namespace with -n or tap all namespaces with -A"
 		}
 		mizu.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr))
@@ -207,11 +207,7 @@ func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
 		}
 	}
 
-	return &shared.TrafficFilteringOptions{
-		PlainTextMaskingRegexes:      compiledRegexSlice,
-		HealthChecksUserAgentHeaders: mizu.Config.Tap.HealthChecksUserAgentHeaders,
-		DisableRedaction:             mizu.Config.Tap.DisableRedaction,
-	}, nil
+	return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: mizu.Config.Tap.HideHealthChecks, DisableRedaction: mizu.Config.Tap.DisableRedaction}, nil
 }
 
 func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error {

cli/go.sum

@@ -217,6 +217,7 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -410,9 +411,10 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=

cli/kubernetes/provider.go

@@ -577,6 +577,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 		"./mizuagent",
 		"-i", "any",
 		"--tap",
+		"--hardump",
 		"--api-server-address", fmt.Sprintf("ws://%s/wsTapper", apiServerPodIp),
 	}
 	if tapOutgoing {

cli/mizu/config.go

@@ -21,8 +21,6 @@ import (
 const (
 	Separator      = "="
 	SetCommandName = "set"
-	FieldNameTag   = "yaml"
-	ReadonlyTag    = "readonly"
 )
 
 var allowedSetFlags = []string{
@@ -33,7 +31,6 @@ var allowedSetFlags = []string{
 	KubeConfigPathName,
 	configStructs.AnalysisDestinationTapName,
 	configStructs.SleepIntervalSecTapName,
-	configStructs.IgnoredUserAgentsTapName,
 }
 
 var Config = ConfigStruct{}
@@ -73,8 +70,8 @@ func GetConfigWithDefaults() (string, error) {
 		return "", err
 	}
 
-	configElem := reflect.ValueOf(&defaultConf).Elem()
-	setZeroForReadonlyFields(configElem)
+	// TODO: change to generic solution
+	defaultConf.AgentImage = ""
 
 	return uiUtils.PrettyYaml(defaultConf)
 }
@@ -112,44 +109,34 @@ func initFlag(f *pflag.Flag) {
 	}
 
 	if f.Name == SetCommandName {
-		mergeSetFlag(configElem, sliceValue.GetSlice())
+		mergeSetFlag(sliceValue.GetSlice())
 		return
 	}
 
 	mergeFlagValues(configElem, f.Name, sliceValue.GetSlice())
 }
 
-func mergeSetFlag(configElem reflect.Value, setValues []string) {
-	setMap := map[string][]string{}
+func mergeSetFlag(setValues []string) {
+	configElem := reflect.ValueOf(&Config).Elem()
 
 	for _, setValue := range setValues {
 		if !strings.Contains(setValue, Separator) {
 			Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument %s (set argument format: <flag name>=<flag value>)", setValue))
-			continue
 		}
 
 		split := strings.SplitN(setValue, Separator, 2)
 		if len(split) != 2 {
 			Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument %s (set argument format: <flag name>=<flag value>)", setValue))
-			continue
 		}
 
 		argumentKey, argumentValue := split[0], split[1]
 
-		setMap[argumentKey] = append(setMap[argumentKey], argumentValue)
-	}
-
-	for argumentKey, argumentValues := range setMap {
 		if !Contains(allowedSetFlags, argumentKey) {
-			Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument name \"%s\", flag name must be one of the following: \"%s\"", argumentKey, strings.Join(allowedSetFlags, "\", \"")))
+			Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument %s, flag name must be one of the following: \"%s\"", setValue, strings.Join(allowedSetFlags, "\", \"")))
 			continue
 		}
 
-		if len(argumentValues) > 1 {
-			mergeFlagValues(configElem, argumentKey, argumentValues)
-		} else {
-			mergeFlagValue(configElem, argumentKey, argumentValues[0])
-		}
+		mergeFlagValue(configElem, argumentKey, argumentValue)
 	}
 }
 
@@ -157,25 +144,21 @@ func mergeFlagValue(currentElem reflect.Value, flagKey string, flagValue string)
 	for i := 0; i < currentElem.NumField(); i++ {
 		currentField := currentElem.Type().Field(i)
 		currentFieldByName := currentElem.FieldByName(currentField.Name)
-		currentFieldKind := currentField.Type.Kind()
 
-		if currentFieldKind == reflect.Struct {
+		if currentField.Type.Kind() == reflect.Struct {
 			mergeFlagValue(currentFieldByName, flagKey, flagValue)
 			continue
 		}
 
-		if getFieldNameByTag(currentField) != flagKey {
+		if currentField.Tag.Get("yaml") != flagKey {
 			continue
 		}
 
-		if currentFieldKind == reflect.Slice {
-			mergeFlagValues(currentElem, flagKey, []string{flagValue})
-			return
-		}
+		flagValueKind := currentField.Type.Kind()
 
-		parsedValue, err := getParsedValue(currentFieldKind, flagValue)
+		parsedValue, err := getParsedValue(flagValueKind, flagValue)
 		if err != nil {
-			Log.Warningf(uiUtils.Warning, fmt.Sprintf("Invalid value %s for flag name %s, expected %s", flagValue, flagKey, currentFieldKind))
+			Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for flag name %s, expected %s", flagValue, flagKey, flagValueKind))
 			return
 		}
 
@@ -187,29 +170,23 @@ func mergeFlagValues(currentElem reflect.Value, flagKey string, flagValues []str
 	for i := 0; i < currentElem.NumField(); i++ {
 		currentField := currentElem.Type().Field(i)
 		currentFieldByName := currentElem.FieldByName(currentField.Name)
-		currentFieldKind := currentField.Type.Kind()
 
-		if currentFieldKind == reflect.Struct {
+		if currentField.Type.Kind() == reflect.Struct {
 			mergeFlagValues(currentFieldByName, flagKey, flagValues)
 			continue
 		}
 
-		if getFieldNameByTag(currentField) != flagKey {
+		if currentField.Tag.Get("yaml") != flagKey {
 			continue
 		}
 
-		if currentFieldKind != reflect.Slice {
-			Log.Warningf(uiUtils.Warning, fmt.Sprintf("Invalid values %s for flag name %s, expected %s", strings.Join(flagValues, ","), flagKey, currentFieldKind))
-			return
-		}
-
 		flagValueKind := currentField.Type.Elem().Kind()
 
 		parsedValues := reflect.MakeSlice(reflect.SliceOf(currentField.Type.Elem()), 0, 0)
 		for _, flagValue := range flagValues {
 			parsedValue, err := getParsedValue(flagValueKind, flagValue)
 			if err != nil {
-				Log.Warningf(uiUtils.Warning, fmt.Sprintf("Invalid value %s for flag name %s, expected %s", flagValue, flagKey, flagValueKind))
+				Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for flag name %s, expected %s", flagValue, flagKey, flagValueKind))
 				return
 			}
 
@@ -220,10 +197,6 @@ func mergeFlagValues(currentElem reflect.Value, flagKey string, flagValues []str
 	}
 }
 
-func getFieldNameByTag(field reflect.StructField) string {
-	return strings.Split(field.Tag.Get(FieldNameTag), ",")[0]
-}
-
 func getParsedValue(kind reflect.Kind, value string) (reflect.Value, error) {
 	switch kind {
 	case reflect.String:
@@ -309,19 +282,3 @@ func getParsedValue(kind reflect.Kind, value string) (reflect.Value, error) {
 
 	return reflect.ValueOf(nil), errors.New("value to parse does not match type")
 }
-
-func setZeroForReadonlyFields(currentElem reflect.Value) {
-	for i := 0; i < currentElem.NumField(); i++ {
-		currentField := currentElem.Type().Field(i)
-		currentFieldByName := currentElem.FieldByName(currentField.Name)
-
-		if currentField.Type.Kind() == reflect.Struct {
-			setZeroForReadonlyFields(currentFieldByName)
-			continue
-		}
-
-		if _, ok := currentField.Tag.Lookup(ReadonlyTag); ok {
-			currentFieldByName.Set(reflect.Zero(currentField.Type))
-		}
-	}
-}

cli/mizu/configStruct.go

@@ -19,7 +19,7 @@ type ConfigStruct struct {
 	Fetch                  configStructs.FetchConfig   `yaml:"fetch"`
 	Version                configStructs.VersionConfig `yaml:"version"`
 	View                   configStructs.ViewConfig    `yaml:"view"`
-	AgentImage             string                      `yaml:"agent-image,omitempty" readonly:""`
+	AgentImage             string                      `yaml:"agent-image,omitempty"`
 	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
 	Telemetry              bool                        `yaml:"telemetry" default:"true"`
 	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`

cli/mizu/configStructs/tapConfig.go

@@ -17,8 +17,8 @@ const (
 	AnalysisTapName               = "analysis"
 	AllNamespacesTapName          = "all-namespaces"
 	PlainTextFilterRegexesTapName = "regex-masking"
+	HideHealthChecksTapName       = "hide-healthchecks"
 	DisableRedactionTapName       = "no-redact"
-	IgnoredUserAgentsTapName      = "ignored-user-agents"
 	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
 	DirectionTapName              = "direction"
 	DryRunTapName                 = "dry-run"
@@ -26,20 +26,20 @@ const (
 )
 
 type TapConfig struct {
-	AnalysisDestination          string   `yaml:"dest" default:"up9.app"`
-	SleepIntervalSec             int      `yaml:"upload-interval" default:"10"`
-	PodRegexStr                  string   `yaml:"regex" default:".*"`
-	GuiPort                      uint16   `yaml:"gui-port" default:"8899"`
-	Namespaces                   []string `yaml:"namespaces"`
-	Analysis                     bool     `yaml:"analysis" default:"false"`
-	AllNamespaces                bool     `yaml:"all-namespaces" default:"false"`
-	PlainTextFilterRegexes       []string `yaml:"regex-masking"`
-	HealthChecksUserAgentHeaders []string `yaml:"ignored-user-agents"`
-	DisableRedaction             bool     `yaml:"no-redact" default:"false"`
-	HumanMaxEntriesDBSize        string   `yaml:"max-entries-db-size" default:"200MB"`
-	Direction                    string   `yaml:"direction" default:"in"`
-	DryRun                       bool     `yaml:"dry-run" default:"false"`
-	EnforcePolicyFile            string   `yaml:"test-rules"`
+	AnalysisDestination    string   `yaml:"dest" default:"up9.app"`
+	SleepIntervalSec       int      `yaml:"upload-interval" default:"10"`
+	PodRegexStr            string   `yaml:"regex" default:".*"`
+	GuiPort                uint16   `yaml:"gui-port" default:"8899"`
+	Namespaces             []string `yaml:"namespaces"`
+	Analysis               bool     `yaml:"analysis" default:"false"`
+	AllNamespaces          bool     `yaml:"all-namespaces" default:"false"`
+	PlainTextFilterRegexes []string `yaml:"regex-masking"`
+	HideHealthChecks       bool     `yaml:"hide-healthchecks" default:"false"`
+	DisableRedaction       bool     `yaml:"no-redact" default:"false"`
+	HumanMaxEntriesDBSize  string   `yaml:"max-entries-db-size" default:"200MB"`
+	Direction              string   `yaml:"direction" default:"in"`
+	DryRun                 bool     `yaml:"dry-run" default:"false"`
+	EnforcePolicyFile      string   `yaml:"test-rules"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {

cli/mizu/config_test.go

@@ -1,39 +0,0 @@
-package mizu_test
-
-import (
-	"github.com/up9inc/mizu/cli/mizu"
-	"reflect"
-	"strings"
-	"testing"
-)
-
-func TestConfigWriteIgnoresReadonlyFields(t *testing.T) {
-	var readonlyFields []string
-
-	configElem := reflect.ValueOf(&mizu.ConfigStruct{}).Elem()
-	getFieldsWithReadonlyTag(configElem, &readonlyFields)
-
-	config, _ := mizu.GetConfigWithDefaults()
-	for _, readonlyField := range readonlyFields {
-		if strings.Contains(config, readonlyField) {
-			t.Errorf("unexpected result - readonly field: %v, config: %v", readonlyField, config)
-		}
-	}
-}
-
-func getFieldsWithReadonlyTag(currentElem reflect.Value, readonlyFields *[]string) {
-	for i := 0; i < currentElem.NumField(); i++ {
-		currentField := currentElem.Type().Field(i)
-		currentFieldByName := currentElem.FieldByName(currentField.Name)
-
-		if currentField.Type.Kind() == reflect.Struct {
-			getFieldsWithReadonlyTag(currentFieldByName, readonlyFields)
-			continue
-		}
-
-		if _, ok := currentField.Tag.Lookup(mizu.ReadonlyTag); ok {
-			fieldNameByTag := strings.Split(currentField.Tag.Get(mizu.FieldNameTag), ",")[0]
-			*readonlyFields = append(*readonlyFields, fieldNameByTag)
-		}
-	}
-}

cli/mizu/sliceUtils_test.go

@@ -1,82 +0,0 @@
-package mizu_test
-
-import (
-	"github.com/up9inc/mizu/cli/mizu"
-	"testing"
-)
-
-func TestContainsExists(t *testing.T) {
-	tests := []struct {
-		slice         []string
-		containsValue string
-		expected      bool
-	}{
-		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "apple", expected: true},
-		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "orange", expected: true},
-		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "banana", expected: true},
-		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "grapes", expected: true},
-	}
-
-	for _, test := range tests {
-		actual := mizu.Contains(test.slice, test.containsValue)
-		if actual != test.expected {
-			t.Errorf("unexpected result - expected: %v, actual: %v", test.expected, actual)
-		}
-	}
-}
-
-func TestContainsNotExists(t *testing.T) {
-	tests := []struct {
-		slice         []string
-		containsValue string
-		expected      bool
-	}{
-		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "cat", expected: false},
-		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "dog", expected: false},
-		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "apples", expected: false},
-		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "rapes", expected: false},
-	}
-
-	for _, test := range tests {
-		actual := mizu.Contains(test.slice, test.containsValue)
-		if actual != test.expected {
-			t.Errorf("unexpected result - expected: %v, actual: %v", test.expected, actual)
-		}
-	}
-}
-
-func TestContainsEmptySlice(t *testing.T) {
-	tests := []struct {
-		slice         []string
-		containsValue string
-		expected      bool
-	}{
-		{slice: []string{}, containsValue: "cat", expected: false},
-		{slice: []string{}, containsValue: "dog", expected: false},
-	}
-
-	for _, test := range tests {
-		actual := mizu.Contains(test.slice, test.containsValue)
-		if actual != test.expected {
-			t.Errorf("unexpected result - expected: %v, actual: %v", test.expected, actual)
-		}
-	}
-}
-
-func TestContainsNilSlice(t *testing.T) {
-	tests := []struct {
-		slice         []string
-		containsValue string
-		expected      bool
-	}{
-		{slice: nil, containsValue: "cat", expected: false},
-		{slice: nil, containsValue: "dog", expected: false},
-	}
-
-	for _, test := range tests {
-		actual := mizu.Contains(test.slice, test.containsValue)
-		if actual != test.expected {
-			t.Errorf("unexpected result - expected: %v, actual: %v", test.expected, actual)
-		}
-	}
-}

cli/mizu/telemetry.go

@@ -15,10 +15,6 @@ func ReportRun(cmd string, args interface{}) {
 		return
 	}
 
-	if Branch != "main" && Branch != "develop" {
-		Log.Debugf("not reporting telemetry on private branches")
-	}
-
 	argsBytes, _ := json.Marshal(args)
 	argsMap := map[string]string{
 		"telemetry_type": "execution",

shared/go.mod

@@ -3,7 +3,8 @@ module github.com/up9inc/mizu/shared
 go 1.16
 
 require (
-	github.com/docker/go-units v0.4.0
+	github.com/google/martian v2.1.0+incompatible // indirect
 	github.com/gorilla/websocket v1.4.2
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect
+	github.com/docker/go-units v0.4.0
 )

shared/go.sum

@@ -1,8 +1,8 @@
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=

shared/models.go

@@ -75,9 +75,9 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc
 }
 
 type TrafficFilteringOptions struct {
-	HealthChecksUserAgentHeaders []string
-	PlainTextMaskingRegexes      []*SerializableRegexp
-	DisableRedaction             bool
+	PlainTextMaskingRegexes []*SerializableRegexp
+	HideHealthChecks        bool
+	DisableRedaction        bool
 }
 
 type VersionResponse struct {

tap/passive_tapper.go

@@ -84,6 +84,7 @@ var staleTimeoutSeconds = flag.Int("staletimout", 120, "Max time in seconds to k
 var memprofile = flag.String("memprofile", "", "Write memory profile")
 
 // output
+var dumpToHar = flag.Bool("hardump", false, "Dump traffic to har files")
 var HarOutputDir = flag.String("hardir", "", "Directory in which to store output har files")
 var harEntriesPerFile = flag.Int("harentriesperfile", 200, "Number of max number of har entries to store in each file")
 
@@ -185,12 +186,19 @@ func (c *Context) GetCaptureInfo() gopacket.CaptureInfo {
 func StartPassiveTapper(opts *TapOpts) (<-chan *OutputChannelItem, <-chan *OutboundLink) {
 	hostMode = opts.HostMode
 
-	harWriter := NewHarWriter(*HarOutputDir, *harEntriesPerFile)
+	var harWriter *HarWriter
+	if *dumpToHar {
+		harWriter = NewHarWriter(*HarOutputDir, *harEntriesPerFile)
+	}
 	outboundLinkWriter := NewOutboundLinkWriter()
 
 	go startPassiveTapper(harWriter, outboundLinkWriter)
 
-	return harWriter.OutChan, outboundLinkWriter.OutChan
+	if harWriter != nil {
+		return harWriter.OutChan, outboundLinkWriter.OutChan
+	}
+
+	return nil, outboundLinkWriter.OutChan
 }
 
 func startMemoryProfiler() {
@@ -313,8 +321,10 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 		}
 	}
 
-	harWriter.Start()
-	defer harWriter.Stop()
+	if *dumpToHar {
+		harWriter.Start()
+		defer harWriter.Stop()
+	}
 	defer outboundLinkWriter.Stop()
 
 	var dec gopacket.Decoder

ui/src/components/HarEntry.tsx

@@ -25,8 +25,7 @@ interface HAREntry {
 
 interface Rules {
     status: boolean;
-    latency: number;
-    numberOfRules: number;
+    latency: number
 }
 
 interface HAREntryProps {
@@ -37,7 +36,6 @@ interface HAREntryProps {
 
 export const HarEntry: React.FC<HAREntryProps> = ({entry, setFocusedEntryId, isSelected}) => {
     const classification = getClassification(entry.statusCode)
-    const numberOfRules = entry.rules.numberOfRules
     let ingoingIcon;
     let outgoingIcon;
     switch(classification) {
@@ -57,36 +55,16 @@ export const HarEntry: React.FC<HAREntryProps> = ({entry, setFocusedEntryId, isS
             break;
         }
     }
-    let additionalRulesProperties = "";
-    let ruleSuccess: boolean;
-    let rule = 'latency' in entry.rules
-    if (rule) {
+    let backgroundColor = "";
+    if ('latency' in entry.rules) {
         if (entry.rules.latency !== -1) {
-            if (entry.rules.latency >= entry.latency) {
-                additionalRulesProperties = styles.ruleSuccessRow
-                ruleSuccess = true
-            } else {
-                additionalRulesProperties = styles.ruleFailureRow
-                ruleSuccess = false
-            }
-            if (isSelected) {
-                additionalRulesProperties += ` ${entry.rules.latency >= entry.latency ? styles.ruleSuccessRowSelected : styles.ruleFailureRowSelected}`
-            }
+            backgroundColor = entry.rules.latency >= entry.latency ? styles.ruleSuccessRow : styles.ruleFailureRow
         } else {
-            if (entry.rules.status) {
-                additionalRulesProperties = styles.ruleSuccessRow
-                ruleSuccess = true
-            } else {
-                additionalRulesProperties = styles.ruleFailureRow
-                ruleSuccess = false
-            }
-            if (isSelected) {
-                additionalRulesProperties += ` ${entry.rules.status ? styles.ruleSuccessRowSelected : styles.ruleFailureRowSelected}`
-            }
+            backgroundColor = entry.rules.status ? styles.ruleSuccessRow : styles.ruleFailureRow
         }
     }
     return <>
-        <div id={entry.id} className={`${styles.row} ${isSelected && !rule ? styles.rowSelected : additionalRulesProperties}`} onClick={() => setFocusedEntryId(entry.id)}>
+        <div id={entry.id} className={`${styles.row} ${isSelected ? styles.rowSelected : backgroundColor}`} onClick={() => setFocusedEntryId(entry.id)}>
             {entry.statusCode && <div>
                 <StatusCode statusCode={entry.statusCode}/>
             </div>}
@@ -96,13 +74,6 @@ export const HarEntry: React.FC<HAREntryProps> = ({entry, setFocusedEntryId, isS
                     {entry.service}
                 </div>
             </div>
-            {
-                rule ?
-                    <div className={`${ruleSuccess ? styles.ruleNumberTextSuccess : styles.ruleNumberTextFailure}`}>
-                        {`Rules (${numberOfRules})`}
-                    </div>
-                : ""
-            }
             <div className={styles.directionContainer}>
                 {entry.isOutgoing ?
                     <img src={outgoingIcon} alt="outgoing traffic" title="outgoing"/>

ui/src/components/HarEntryDetailed.tsx

@@ -43,6 +43,7 @@ const HarEntryTitle: React.FC<any> = ({har}) => {
         <div style={{margin: "0 18px", opacity: 0.5}}>{formatSize(bodySize)}</div>
         <div style={{marginRight: 18, opacity: 0.5}}>{status} {statusText}</div>
         <div style={{marginRight: 18, opacity: 0.5}}>{Math.round(receive)}ms</div>
+        <div style={{opacity: 0.5}}>{'rulesMatched' in entries[0] ? entries[0].rulesMatched?.length : '0'} Rules Applied</div>
     </div>;
 };
 

ui/src/components/HarEntryViewer/HAREntrySections.module.sass

@@ -92,6 +92,3 @@
         tr td:first-child
             white-space: nowrap
             padding-right: .5rem
-
-.noRules
-    padding: 0 1rem 1rem

ui/src/components/HarEntryViewer/HAREntrySections.tsx

@@ -260,7 +260,7 @@ export const HAREntryTablePolicySection: React.FC<HAREntryPolicySectionProps> =
                     </table>
                 </HAREntrySectionContainer>
                                             
-                </> : <span className={styles.noRules}>No rules could be applied to this request.</span>
+                </> : <span/>
         }
     </React.Fragment>
 }

ui/src/components/style/HarEntry.module.sass

@@ -24,40 +24,12 @@
   margin-right: 3px
 
 .ruleSuccessRow
-  background: #E8FFF1
-
-.ruleSuccessRowSelected
-  border: 1px #6FCF97 solid
-  border-left: 5px #6FCF97 solid
-  margin-left: 10px
-  margin-right: 3px
+  border: 1px $success-color solid
+  border-left: 5px $success-color solid
 
 .ruleFailureRow
-  background: #FFE9EF
-
-.ruleFailureRowSelected
   border: 1px $failure-color solid
   border-left: 5px $failure-color solid
-  margin-left: 10px
-  margin-right: 3px
-
-.ruleNumberTextFailure
-  color: #DB2156
-  font-family: Source Sans Pro;
-  font-style: normal;
-  font-weight: 600;
-  font-size: 12px;
-  line-height: 15px;
-  padding-right: 12px
-
-.ruleNumberTextSuccess
-  color: #219653
-  font-family: Source Sans Pro;
-  font-style: normal;
-  font-weight: 600;
-  font-size: 12px;
-  line-height: 15px;
-  padding-right: 12px
 
 .service
   text-overflow: ellipsis