codecov yml for tests threshold (#214)

* no message

* change local api path

* generic entry list item + rename files and vars

* entry detailed generic

* fix api file

* clean warnings

* switch

* empty lines

* fix scroll to end feature

Co-authored-by: Roee Gadot <roee.gadot@up9.com>

.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# This is a comment.
+# Each line is a file pattern followed by one or more owners.
+
+/ui/ @frontend

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Run mizu <command> '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Logs**
+Upload logs:
+1. Run the mizu command with `--set dump-logs=true` (e.g `mizu tap --set dump-logs=true`) 
+2. Try to reproduce the issue
+3. CNTRL+C on terminal tab which runs mizu
+4. Upload the logs zip file from ~/.mizu/mizu_logs_**.zip
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome]
+
+**Additional context**
+Add any other context about the problem here.

.github/workflows/validations.yaml

@@ -0,0 +1,84 @@
+name: validations
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+  push:
+    branches:
+      - 'develop'
+      - 'main'
+jobs:
+  build-cli:
+    name: Build CLI
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Build CLI
+        run: make cli
+
+  build-agent:
+    name: Build Agent
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - shell: bash
+        run: |
+          sudo apt-get install libpcap-dev
+
+      - name: Build Agent
+        run: make agent
+
+  run-tests-cli:
+    name: Run CLI tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: make test-cli
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2
+
+  run-tests-agent:
+    name: Run Agent tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - shell: bash
+        run: |
+          sudo apt-get install libpcap-dev
+
+      - name: Test
+        run: make test-agent
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2

CONTRIBUTE.md

@@ -0,0 +1,18 @@
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+# CONTRIBUTE
+We welcome code contributions from the community.
+Please read and follow the guidelines below.
+
+## Communication
+* Before starting work on a major feature, please reach out to us via [GitHub](https://github.com/up9inc/mizu), [Slack](https://join.slack.com/share/zt-u6bbs3pg-X1zhQOXOH0yEoqILgH~csw), [email](mailto:mizu@up9.com), etc. We will make sure no one else is already working on it. A _major feature_ is defined as any change that is > 100 LOC altered (not including tests), or changes any user-facing behavior
+* Small patches and bug fixes don't need prior communication.
+
+## Contribution requirements
+* Code style - most of the code is written in Go, please follow [these guidelines](https://golang.org/doc/effective_go)
+* Go-tools compatible (`go get`, `go test`, etc)
+* Unit-test coverage can’t go down ..
+* Code must be usefully commented. Not only for developers on the project, but also for external users of these packages
+* When reviewing PRs, you are encouraged to use Golang's [code review comments page](https://github.com/golang/go/wiki/CodeReviewComments)
+
+
+

Dockerfile

@@ -48,8 +48,6 @@ WORKDIR /app
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 
-COPY agent/start.sh .
-
 # gin-gonic runs in debug mode without this
 ENV GIN_MODE=release
 

Makefile

@@ -19,34 +19,30 @@ help: ## This help.
 TS_SUFFIX="$(shell date '+%s')"
 GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
 BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
+export SEM_VER?=0.0.0
 
-ui: ## build UI
+ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
 	@ls -l ui/build  
 
-cli: # build CLI
+cli: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build
 
-agent: ## build mizuagent server
+agent: ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
 	@ls -l agent/build
 
-#tap: ## build tap binary
-#	@(cd tap; go build -o build/tap ./src)
-#	@ls -l tap/build
+docker: ## Build and publish agent docker image.
+	$(MAKE) push-docker
 
-docker: ## build Docker image 
-	@(echo "building docker image" )
-	./build-push-featurebranch.sh
+push: push-docker push-cli ## Build and publish agent docker image & CLI.
 
-push: push-docker push-cli ## build and publish Mizu docker image & CLI
-
-push-docker: 
+push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
 	./build-push-featurebranch.sh
 
-push-cli:
+push-cli: ## Build and publish CLI.
 	@echo "publishing CLI .. "
 	@cd cli; $(MAKE) build-all
 	@echo "publishing file ${OUTPUT_FILE} .."
@@ -55,17 +51,22 @@ push-cli:
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
 
-clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts
+clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
 
-clean-ui: 
+clean-ui: ## Clean UI.
 	@(rm -rf ui/build ; echo "UI cleanup done" )
 
-clean-agent: 
+clean-agent: ## Clean agent.
 	@(rm -rf agent/build ; echo "agent cleanup done" )
 
-clean-cli: 
+clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
-clean-docker: 
+clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
+test-cli: ## Run tests.
+	@echo "running cli tests"; cd cli && $(MAKE) test
+
+test-agent: ## Run tests.
+	@echo "running agent tests"; cd agent && $(MAKE) test

PERMISSIONS.md

@@ -0,0 +1,328 @@
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+# Kubernetes permissions for MIZU  
+
+This document describes in details all permissions required for full and correct operation of Mizu
+
+We broke down this list into few categories:
+- Required - what is needed for `mizu` to run properly on your k8s cluster
+- Optional - permissions needed for proper name resolving for service & pod IPs 
+   - addition required for policy validation 
+ 
+
+
+# Required permissions
+
+Mizu needs following permissions on your Kubernetes cluster to run properly
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+```
+
+## Permissions required for service / pod name resolving (opt)
+
+Optionally, for proper resolving of IP addresses to Kubernetes service name, Mizu needs below permissions:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterroles
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterrolebindings
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  - apps
+  - extensions
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+```
+
+## Permissions for Policy rules validation feature (opt)
+
+Optionally, in order to use the policy rules validation feature, Mizu requires the following additional permissions:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - create
+  - delete
+```
+
+- - -
+
+## Namespace-Restricted mode
+
+Alternatively, in order to restrict Mizu to one namespace only (by setting `agent.namespace` in the config file), Mizu needs the following permissions in that namespace:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+```
+
+### Name resolving in Namespace-Restricted mode (opt)
+
+To restrict Mizu to one namespace while also resolving IPs, Mizu needs the following permissions in that namespace:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  - apps
+  - extensions
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+```

README.md

@@ -1,22 +1,23 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+
 # The API Traffic Viewer for Kubernetes
 
-A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined.
+A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined
 
 ![Simple UI](assets/mizu-ui.png)
 
 ## Features
 
 - Simple and powerful CLI
-- Real time view of all HTTP requests, REST and gRPC API calls
+- Real-time view of all HTTP requests, REST and gRPC API calls
 - No installation or code instrumentation
-- Works completely on premises (on-prem)
+- Works completely on premises
 
 ## Download
 
-Download `mizu` for your platform and operating system
+Download Mizu for your platform and operating system
 
-### Latest stable release
+### Latest Stable Release
 
 * for MacOS - Intel 
 ```
@@ -32,140 +33,50 @@ https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 \
 && chmod 755 mizu
 ``` 
 
-SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page.
+SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page
 
-### Development (unstable) build
-Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page.
+### Development (unstable) Build
+Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page
 
 ## Prerequisites
-1. Set `KUBECONFIG` environment variable to your kubernetes configuration. If this is not set, mizu assumes that configuration is at `${HOME}/.kube/config`
-2. mizu needs following permissions on your kubernetes cluster to run
+1. Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
+2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination)
 
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - list
-  - watch
-  - create
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - create
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - create
-  - patch
-- apiGroups:
-  - ""
-  resources:
-  - namespaces
-  verbs:
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services/proxy
-  verbs:
-  - get
-```
-3. Optionally, for resolving traffic ip to kubernetes service name, mizu needs below permissions
+For detailed list of k8s permissions see [PERMISSIONS](PERMISSIONS.md) document
 
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  - apps
-  - extensions
-  resources:
-  - endpoints
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - serviceaccounts
-  verbs:
-  - get
-  - create
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - clusterroles
-  verbs:
-  - list
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - clusterrolebindings
-  verbs:
-  - list
-  - create
-  - delete
-```
 
-See `examples/roles` for example `clusterroles`. 
-
-## How to run
+## How to Run
 
 1. Find pods you'd like to tap to in your Kubernetes cluster
-2. Run `mizu tap PODNAME` or `mizu tap REGEX` 
-3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI .. 
-4. Watch the API traffic flowing ..
+2. Run `mizu tap` or `mizu tap PODNAME`  
+3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI
+4. Watch the API traffic flowing
 5. Type ^C to stop
 
 ## Examples
 
 Run `mizu help` for usage options
 
+To tap all pods in current namespace - 
+``` 
+ $ kubectl get pods 
+ NAME                            READY   STATUS    RESTARTS   AGE
+ carts-66c77f5fbb-fq65r          2/2     Running   0          20m
+ catalogue-5f4cb7cf5-7zrmn       2/2     Running   0          20m
+ front-end-649fc5fd6-kqbtn       2/2     Running   0          20m
+ ..
+
+ $ mizu tap
+ +carts-66c77f5fbb-fq65r
+ +catalogue-5f4cb7cf5-7zrmn
+ +front-end-649fc5fd6-kqbtn
+ Web interface is now available at http://localhost:8899
+ ^C
+```
+
 
 To tap specific pod - 
-``` 
+```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
  front-end-649fc5fd6-kqbtn       2/2     Running   0          7m
@@ -178,7 +89,7 @@ To tap specific pod -
 ```
 
 To tap multiple pods using regex - 
-``` 
+```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
  carts-66c77f5fbb-fq65r          2/2     Running   0          20m
@@ -193,3 +104,67 @@ To tap multiple pods using regex -
  ^C
 ```
 
+## Configuration
+
+Mizu can work with config file which should be stored in ${HOME}/.mizu/config.yaml (macOS: ~/.mizu/config.yaml) <br />
+In case no config file found, defaults will be used <br />
+In case of partial configuration defined, all other fields will be used with defaults <br />
+You can always override the defaults or config file with CLI flags
+
+To get the default config params run `mizu config` <br />
+To generate a new config file with default values use `mizu config -r`
+
+### Telemetry
+
+By default, mizu reports usage telemetry. It can be disabled by adding a line of `telemetry: false` in the `${HOME}/.mizu/config.yaml` file
+
+
+## Advanced Usage
+
+### Namespace-Restricted Mode
+
+Some users have permission to only manage resources in one particular namespace assigned to them
+By default `mizu tap` creates a new namespace `mizu` for all of its Kubernetes resources. In order to instead install
+Mizu in an existing namespace, set the `mizu-resources-namespace` config option
+
+If `mizu-resources-namespace` is set to a value other than the default `mizu`, Mizu will operate in a
+Namespace-Restricted mode. It will only tap pods in `mizu-resources-namespace`. This way Mizu only requires permissions
+to the namespace set by `mizu-resources-namespace`. The user must set the tapped namespace to the same namespace by
+using the `--namespace` flag or by setting `tap.namespaces` in the config file
+
+Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
+
+### User agent filtering
+
+User-agent filtering (like health checks) - can be configured using command-line options:
+
+```shell
+$ mizu tap "^ca.*" --set ignored-user-agents=kube-probe --set ignored-user-agents=prometheus
++carts-66c77f5fbb-fq65r
++catalogue-5f4cb7cf5-7zrmn
+Web interface is now available at http://localhost:8899
+^C
+
+```
+
+Any request that contains `User-Agent` header with one of the specified values (`kube-probe` or `prometheus`) will not be captured
+
+### API Rules validation
+
+This feature allows you to define set of simple rules, and test the API against them.
+Such validation may test response for specific JSON fields, headers, etc.
+
+Please see [API RULES](docs/POLICY_RULES.md) page for more details and syntax.
+
+
+## How to Run local UI
+
+- run from mizu/agent `go run main.go --hars-read --hars-dir <folder>`
+
+- copy Har files into the folder from last command
+
+- change `MizuWebsocketURL` and `apiURL` in `api.js` file
+
+- run from mizu/ui - `npm run start`
+
+- open browser on `localhost:3000`

TESTING.md

@@ -0,0 +1,15 @@
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+# TESTING
+Testing guidelines for Mizu project 
+
+## Unit-tests
+* TBD 
+* TBD
+* TBD
+
+
+
+## System tests
+* TBD 
+* TBD
+* TBD

agent/Makefile

@@ -0,0 +1,2 @@
+test: ## Run agent tests.
+	@go test ./... -race -coverprofile=coverage.out -covermode=atomic

agent/go.mod

@@ -13,9 +13,12 @@ require (
 	github.com/go-playground/validator/v10 v10.5.0
 	github.com/google/martian v2.1.0+incompatible
 	github.com/gorilla/websocket v1.4.2
+	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
+	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
 	go.mongodb.org/mongo-driver v1.5.1
 	gorm.io/driver/sqlite v1.1.4
 	gorm.io/gorm v1.21.8

agent/go.sum

@@ -45,6 +45,8 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l
 github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
 github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
 github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
+github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M=
+github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -245,6 +247,8 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -283,6 +287,8 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -534,8 +540,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=
 gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw=
 gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=

agent/main.go

@@ -4,6 +4,12 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"github.com/gin-contrib/static"
+	"github.com/gin-gonic/gin"
+	"github.com/gorilla/websocket"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap"
 	"mizuserver/pkg/api"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/routes"
@@ -13,39 +19,38 @@ import (
 	"os"
 	"os/signal"
 	"strings"
-
-	"github.com/gin-contrib/static"
-	"github.com/gin-gonic/gin"
-	"github.com/gorilla/websocket"
-	"github.com/romana/rlog"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap"
 )
 
-var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API")
-var demo = flag.Bool("demo", false, "Run in Demo mode with API")
-var apiServer = flag.Bool("api-server", false, "Run in API server mode with API")
-var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode")
+var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API")
+var apiServerMode = flag.Bool("api-server", false, "Run in API server mode with API")
+var standaloneMode = flag.Bool("standalone", false, "Run in standalone tapper and API mode")
 var apiServerAddress = flag.String("api-server-address", "", "Address of mizu API server")
+var namespace = flag.String("namespace", "", "Resolve IPs if they belong to resources in this namespace (default is all)")
+var harsReaderMode = flag.Bool("hars-read", false, "Run in hars-read mode")
+var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
 
 func main() {
 	flag.Parse()
 	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
 	tapOpts := &tap.TapOpts{HostMode: hostMode}
 
-	if !*shouldTap && !*apiServer && !*standalone {
-		panic("One of the flags --tap, --api or --standalone must be provided")
+
+	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode{
+		panic("One of the flags --tap, --api or --standalone or --hars-read must be provided")
 	}
-	if *standalone {
+
+	if *standaloneMode {
+		api.StartResolving(*namespace)
+
 		harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts)
 		filteredHarChannel := make(chan *tap.OutputChannelItem)
 
 		go filterHarItems(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions())
-		go api.StartReadingEntries(filteredHarChannel, nil, false)
+		go api.StartReadingEntries(filteredHarChannel, nil)
 		go api.StartReadingOutbound(outboundLinkOutputChannel)
 
 		hostApi(nil)
-	} else if *shouldTap {
+	} else if *tapperMode {
 		if *apiServerAddress == "" {
 			panic("API server address must be provided with --api-server-address when using --tap")
 		}
@@ -63,21 +68,25 @@ func main() {
 			panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
 		}
 
-		go pipeChannelToSocket(socketConnection, harOutputChannel)
-		go api.StartReadingOutbound(outboundLinkOutputChannel)
-	} else if *apiServer {
+		go pipeTapChannelToSocket(socketConnection, harOutputChannel)
+		go pipeOutboundLinksChannelToSocket(socketConnection, outboundLinkOutputChannel)
+	} else if *apiServerMode {
+		api.StartResolving(*namespace)
+
 		socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000)
 		filteredHarChannel := make(chan *tap.OutputChannelItem)
 
 		go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions())
-		if *demo {
-			workdir := "./hars"
-			go api.StartReadingEntries(filteredHarChannel, &workdir, true)
-		} else {
-			go api.StartReadingEntries(filteredHarChannel, nil, false)
-		}
+		go api.StartReadingEntries(filteredHarChannel, nil)
 
 		hostApi(socketHarOutChannel)
+	} else if *harsReaderMode {
+		socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000)
+		filteredHarChannel := make(chan *tap.OutputChannelItem)
+
+		go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions())
+		go api.StartReadingEntries(filteredHarChannel, harsDir)
+		hostApi(nil)
 	}
 
 	signalChan := make(chan os.Signal, 1)
@@ -91,7 +100,7 @@ func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
 	app := gin.Default()
 
 	app.GET("/echo", func(c *gin.Context) {
-		c.String(http.StatusOK, "Hello, World 👋!")
+		c.String(http.StatusOK, "Here is Mizu agent")
 	})
 
 	eventHandlers := api.RoutesEventHandlers{
@@ -101,9 +110,10 @@ func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
 	app.Use(static.ServeRoot("/", "./site"))
 	app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
 
-	routes.WebSocketRoutes(app, &eventHandlers)
+	api.WebSocketRoutes(app, &eventHandlers)
 	routes.EntriesRoutes(app)
 	routes.MetadataRoutes(app)
+	routes.StatusRoutes(app)
 	routes.NotFoundRoute(app)
 
 	utils.StartServer(app)
@@ -149,15 +159,13 @@ func getTrafficFilteringOptions() *shared.TrafficFilteringOptions {
 	return &filteringOptions
 }
 
-var userAgentsToFilter = []string{"kube-probe", "prometheus"}
-
 func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) {
 	for message := range inChannel {
 		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
 			continue
 		}
 		// TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441
-		if filterOptions.HideHealthChecks && isHealthCheckByUserAgent(message) {
+		if isHealthCheckByUserAgent(message, filterOptions.HealthChecksUserAgentHeaders) {
 			continue
 		}
 
@@ -169,11 +177,11 @@ func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *ta
 	}
 }
 
-func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool {
+func isHealthCheckByUserAgent(message *tap.OutputChannelItem, userAgentsToIgnore []string) bool {
 	for _, header := range message.HarEntry.Request.Headers {
 		if strings.ToLower(header.Name) == "user-agent" {
-			for _, userAgent := range userAgentsToFilter {
-				if strings.Contains(strings.ToLower(header.Value), userAgent) {
+			for _, userAgent := range userAgentsToIgnore {
+				if strings.Contains(strings.ToLower(header.Value), strings.ToLower(userAgent)) {
 					return true
 				}
 			}
@@ -183,7 +191,7 @@ func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool {
 	return false
 }
 
-func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) {
+func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) {
 	if connection == nil {
 		panic("Websocket connection is nil")
 	}
@@ -206,3 +214,21 @@ func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *
 		}
 	}
 }
+
+func pipeOutboundLinksChannelToSocket(connection *websocket.Conn, outboundLinkChannel <-chan *tap.OutboundLink) {
+	for outboundLink := range outboundLinkChannel {
+		if outboundLink.SuggestedProtocol == tap.TLSProtocol {
+			marshaledData, err := models.CreateWebsocketOutboundLinkMessage(outboundLink)
+			if err != nil {
+				rlog.Infof("Error converting outbound link to json %s, (%v,%+v)", err, err, err)
+				continue
+			}
+
+			err = connection.WriteMessage(websocket.TextMessage, marshaledData)
+			if err != nil {
+				rlog.Infof("error sending outbound link message through socket server %s, (%v,%+v)", err, err, err)
+				continue
+			}
+		}
+	}
+}

agent/pkg/api/main.go

@@ -5,8 +5,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"math/rand"
 	"mizuserver/pkg/holder"
+	"mizuserver/pkg/providers"
 	"net/url"
 	"os"
 	"path"
@@ -27,9 +27,9 @@ import (
 
 var k8sResolver *resolver.Resolver
 
-func init() {
+func StartResolving(namespace string) {
 	errOut := make(chan error, 100)
-	res, err := resolver.NewFromInCluster(errOut)
+	res, err := resolver.NewFromInCluster(errOut, namespace)
 	if err != nil {
 		rlog.Infof("error creating k8s resolver %s", err)
 		return
@@ -49,15 +49,15 @@ func init() {
 	holder.SetResolver(res)
 }
 
-func StartReadingEntries(harChannel <-chan *tap.OutputChannelItem, workingDir *string, demo bool) {
+func StartReadingEntries(harChannel <-chan *tap.OutputChannelItem, workingDir *string) {
 	if workingDir != nil && *workingDir != "" {
-		startReadingFiles(*workingDir, demo)
+		startReadingFiles(*workingDir)
 	} else {
 		startReadingChannel(harChannel)
 	}
 }
 
-func startReadingFiles(workingDir string, infiniteLoad bool) {
+func startReadingFiles(workingDir string) {
 	err := os.MkdirAll(workingDir, os.ModePerm)
 	utils.CheckErr(err)
 
@@ -88,10 +88,7 @@ func startReadingFiles(workingDir string, infiniteLoad bool) {
 		utils.CheckErr(decErr)
 
 		for _, entry := range inputHar.Log.Entries {
-			if infiniteLoad {
-				entry.StartedDateTime = time.Now().Add(20 * time.Millisecond)
-			}
-			time.Sleep(time.Millisecond * time.Duration(rand.Intn(300)))
+			time.Sleep(time.Millisecond * 250)
 			connectionInfo := &tap.ConnectionInfo{
 				ClientIP:   fileInfo.Name(),
 				ClientPort: "",
@@ -101,10 +98,8 @@ func startReadingFiles(workingDir string, infiniteLoad bool) {
 			}
 			saveHarToDb(entry, connectionInfo)
 		}
-		if !infiniteLoad {
-			rmErr := os.Remove(inputFilePath)
-			utils.CheckErr(rmErr)
-		}
+		rmErr := os.Remove(inputFilePath)
+		utils.CheckErr(rmErr)
 	}
 }
 
@@ -114,6 +109,7 @@ func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) {
 	}
 
 	for item := range outputItems {
+		providers.EntryAdded()
 		saveHarToDb(item.HarEntry, item.ConnectionInfo)
 	}
 }
@@ -173,8 +169,10 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
 	if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil {
 		return
 	}
+	baseEntry.Rules = models.RunValidationRulesState(*entry, serviceName)
+	baseEntry.Latency = entry.Timings.Receive
 	baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry)
-	broadcastToBrowserClients(baseEntryBytes)
+	BroadcastToBrowserClients(baseEntryBytes)
 }
 
 func getServiceNameFromUrl(inputUrl string) (string, string) {

agent/pkg/api/socket_routes.go

@@ -1,10 +1,10 @@
-package routes
+package api
 
 import (
 	"errors"
-	"fmt"
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
+	"github.com/romana/rlog"
 	"github.com/up9inc/mizu/shared/debounce"
 	"net/http"
 	"sync"
@@ -18,10 +18,10 @@ type EventHandlers interface {
 }
 
 type SocketConnection struct {
-	connection *websocket.Conn
-	lock *sync.Mutex
+	connection    *websocket.Conn
+	lock          *sync.Mutex
 	eventHandlers EventHandlers
-	isTapper bool
+	isTapper      bool
 }
 
 var websocketUpgrader = websocket.Upgrader{
@@ -50,7 +50,7 @@ func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
 func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) {
 	conn, err := websocketUpgrader.Upgrade(w, r, nil)
 	if err != nil {
-		fmt.Println("Failed to set websocket upgrade: %+v", err)
+		rlog.Errorf("Failed to set websocket upgrade: %v", err)
 		return
 	}
 
@@ -71,7 +71,7 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 	for {
 		_, msg, err := conn.ReadMessage()
 		if err != nil {
-			fmt.Printf("Conn err: %v\n", err)
+			rlog.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
 			break
 		}
 		eventHandlers.WebSocketMessage(socketId, msg)
@@ -81,7 +81,7 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 func socketCleanup(socketId int, socketConnection *SocketConnection) {
 	err := socketConnection.connection.Close()
 	if err != nil {
-		fmt.Printf("Error closing socket connection for socket id %d: %v\n", socketId, err)
+		rlog.Errorf("Error closing socket connection for socket id %d: %v\n", socketId, err)
 	}
 
 	websocketIdsLock.Lock()
@@ -91,8 +91,8 @@ func socketCleanup(socketId int, socketConnection *SocketConnection) {
 	socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
 }
 
-var db = debounce.NewDebouncer(time.Second * 5, func() {
-	fmt.Println("Successfully sent to socket")
+var db = debounce.NewDebouncer(time.Second*5, func() {
+	rlog.Error("Successfully sent to socket")
 })
 
 func SendToSocket(socketId int, message []byte) error {
@@ -102,9 +102,9 @@ func SendToSocket(socketId int, message []byte) error {
 	}
 
 	var sent = false
-	time.AfterFunc(time.Second * 5, func() {
+	time.AfterFunc(time.Second*5, func() {
 		if !sent {
-			fmt.Println("Socket timed out")
+			rlog.Error("Socket timed out")
 			socketCleanup(socketId, socketObj)
 		}
 	})

agent/pkg/api/socket_server_handlers.go

@@ -3,26 +3,26 @@ package api
 import (
 	"encoding/json"
 	"fmt"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/up9"
+	"sync"
+
 	"github.com/romana/rlog"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/routes"
-	"mizuserver/pkg/up9"
-	"sync"
 )
 
 var browserClientSocketUUIDs = make([]int, 0)
 var socketListLock = sync.Mutex{}
 
 type RoutesEventHandlers struct {
-	routes.EventHandlers
+	EventHandlers
 	SocketHarOutChannel chan<- *tap.OutputChannelItem
 }
 
 func init() {
-	go up9.UpdateAnalyzeStatus(broadcastToBrowserClients)
+	go up9.UpdateAnalyzeStatus(BroadcastToBrowserClients)
 }
 
 func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
@@ -47,15 +47,14 @@ func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 	}
 }
 
-func broadcastToBrowserClients(message []byte) {
+func BroadcastToBrowserClients(message []byte) {
 	for _, socketId := range browserClientSocketUUIDs {
 		go func(socketId int) {
-			err := routes.SendToSocket(socketId, message)
+			err := SendToSocket(socketId, message)
 			if err != nil {
-				fmt.Printf("error sending message to socket ID %d: %v", socketId, err)
+				rlog.Errorf("error sending message to socket ID %d: %v", socketId, err)
 			}
 		}(socketId)
-
 	}
 }
 
@@ -80,8 +79,16 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 			if err != nil {
 				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
 			} else {
-				controllers.TapStatus = statusMessage.TappingStatus
-				broadcastToBrowserClients(message)
+				providers.TapStatus.Pods = statusMessage.TappingStatus.Pods
+				BroadcastToBrowserClients(message)
+			}
+		case shared.WebsocketMessageTypeOutboundLink:
+			var outboundLinkMessage models.WebsocketOutboundLinkMessage
+			err := json.Unmarshal(message, &outboundLinkMessage)
+			if err != nil {
+				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+			} else {
+				handleTLSLink(outboundLinkMessage)
 			}
 		default:
 			rlog.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
@@ -89,6 +96,29 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 	}
 }
 
+func handleTLSLink(outboundLinkMessage models.WebsocketOutboundLinkMessage) {
+	resolvedName := k8sResolver.Resolve(outboundLinkMessage.Data.DstIP)
+	if resolvedName != "" {
+		outboundLinkMessage.Data.DstIP = resolvedName
+	} else if outboundLinkMessage.Data.SuggestedResolvedName != "" {
+		outboundLinkMessage.Data.DstIP = outboundLinkMessage.Data.SuggestedResolvedName
+	}
+	cacheKey := fmt.Sprintf("%s -> %s:%d", outboundLinkMessage.Data.Src, outboundLinkMessage.Data.DstIP, outboundLinkMessage.Data.DstPort)
+	_, isInCache := providers.RecentTLSLinks.Get(cacheKey)
+	if isInCache {
+		return
+	} else {
+		providers.RecentTLSLinks.SetDefault(cacheKey, outboundLinkMessage.Data)
+	}
+	marshaledMessage, err := json.Marshal(outboundLinkMessage)
+	if err != nil {
+		rlog.Errorf("Error marshaling outbound link message for broadcasting: %v", err)
+	} else {
+		rlog.Errorf("Broadcasting outboundlink message %s", string(marshaledMessage))
+		BroadcastToBrowserClients(marshaledMessage)
+	}
+}
+
 func removeSocketUUIDFromBrowserSlice(uuidToRemove int) {
 	newUUIDSlice := make([]int, 0, len(browserClientSocketUUIDs))
 	for _, uuid := range browserClientSocketUUIDs {

agent/pkg/controllers/entries_controller.go

@@ -3,17 +3,19 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/gin-gonic/gin"
-	"github.com/google/martian/har"
-	"github.com/romana/rlog"
 	"mizuserver/pkg/database"
 	"mizuserver/pkg/models"
+	"mizuserver/pkg/providers"
 	"mizuserver/pkg/up9"
 	"mizuserver/pkg/utils"
 	"mizuserver/pkg/validation"
 	"net/http"
 	"strings"
 	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/google/martian/har"
+	"github.com/romana/rlog"
 )
 
 func GetEntries(c *gin.Context) {
@@ -55,7 +57,7 @@ func GetEntries(c *gin.Context) {
 }
 
 func GetHARs(c *gin.Context) {
-	entriesFilter := &models.HarFetchRequestBody{}
+	entriesFilter := &models.HarFetchRequestQuery{}
 	order := database.OrderDesc
 	if err := c.BindQuery(entriesFilter); err != nil {
 		c.JSON(http.StatusBadRequest, err)
@@ -144,12 +146,12 @@ func GetHARs(c *gin.Context) {
 func UploadEntries(c *gin.Context) {
 	rlog.Infof("Upload entries - started\n")
 
-	uploadRequestBody := &models.UploadEntriesRequestBody{}
-	if err := c.BindQuery(uploadRequestBody); err != nil {
+	uploadParams := &models.UploadEntriesRequestQuery{}
+	if err := c.BindQuery(uploadParams); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
-	if err := validation.Validate(uploadRequestBody); err != nil {
+	if err := validation.Validate(uploadParams); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
@@ -158,19 +160,19 @@ func UploadEntries(c *gin.Context) {
 		return
 	}
 
-	rlog.Infof("Upload entries - creating token. dest %s\n", uploadRequestBody.Dest)
-	token, err := up9.CreateAnonymousToken(uploadRequestBody.Dest)
+	rlog.Infof("Upload entries - creating token. dest %s\n", uploadParams.Dest)
+	token, err := up9.CreateAnonymousToken(uploadParams.Dest)
 	if err != nil {
 		c.String(http.StatusServiceUnavailable, "Cannot analyze, mizu is already analyzing")
 		return
 	}
 	rlog.Infof("Upload entries - uploading. token: %s model: %s\n", token.Token, token.Model)
-	go up9.UploadEntriesImpl(token.Token, token.Model, uploadRequestBody.Dest, uploadRequestBody.SleepIntervalSec)
+	go up9.UploadEntriesImpl(token.Token, token.Model, uploadParams.Dest, uploadParams.SleepIntervalSec)
 	c.String(http.StatusOK, "OK")
 }
 
 func GetFullEntries(c *gin.Context) {
-	entriesFilter := &models.HarFetchRequestBody{}
+	entriesFilter := &models.HarFetchRequestQuery{}
 	if err := c.BindQuery(entriesFilter); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 	}
@@ -217,7 +219,14 @@ func GetEntry(c *gin.Context) {
 			"msg":   "Can't get entry details",
 		})
 	}
-	c.JSON(http.StatusOK, fullEntry)
+	fullEntryWithPolicy := models.FullEntryWithPolicy{}
+	if err := models.GetEntry(&entryData, &fullEntryWithPolicy); err != nil {
+		c.JSON(http.StatusInternalServerError, map[string]interface{}{
+			"error": true,
+			"msg":   "Can't get entry details",
+		})
+	}
+	c.JSON(http.StatusOK, fullEntryWithPolicy)
 }
 
 func DeleteAllEntries(c *gin.Context) {
@@ -232,12 +241,17 @@ func DeleteAllEntries(c *gin.Context) {
 }
 
 func GetGeneralStats(c *gin.Context) {
-	sqlQuery := "SELECT count(*) as count, min(timestamp) as min, max(timestamp) as max from mizu_entries"
-	var result struct {
-		Count int
-		Min   int
-		Max   int
-	}
-	database.GetEntriesTable().Raw(sqlQuery).Scan(&result)
-	c.JSON(http.StatusOK, result)
+	c.JSON(http.StatusOK, providers.GetGeneralStats())
+}
+
+func GetTappingStatus(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.TapStatus)
+}
+
+func AnalyzeInformation(c *gin.Context) {
+	c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
+}
+
+func GetRecentTLSLinks(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
 }

agent/pkg/controllers/status_controller.go

@@ -1,18 +1,32 @@
 package controllers
 
 import (
+	"encoding/json"
 	"github.com/gin-gonic/gin"
+	"github.com/romana/rlog"
 	"github.com/up9inc/mizu/shared"
-	"mizuserver/pkg/up9"
+	"mizuserver/pkg/api"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/validation"
 	"net/http"
 )
 
-var TapStatus shared.TapStatus
-
-func GetTappingStatus(c *gin.Context) {
-	c.JSON(http.StatusOK, TapStatus)
-}
-
-func AnalyzeInformation(c *gin.Context) {
-	c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
+func PostTappedPods(c *gin.Context) {
+	tapStatus := &shared.TapStatus{}
+	if err := c.Bind(tapStatus); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+	if err := validation.Validate(tapStatus); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+	rlog.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods))
+	providers.TapStatus.Pods = tapStatus.Pods
+	message := shared.CreateWebSocketStatusMessage(*tapStatus)
+	if jsonBytes, err := json.Marshal(message); err != nil {
+		rlog.Errorf("Could not Marshal message %v\n", err)
+	} else {
+		api.BroadcastToBrowserClients(jsonBytes)
+	}
 }

agent/pkg/database/size_enforcer.go

@@ -1,8 +1,8 @@
 package database
 
 import (
-	"fmt"
 	"github.com/fsnotify/fsnotify"
+	"github.com/romana/rlog"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/units"
@@ -47,7 +47,7 @@ func StartEnforcingDatabaseSize() {
 				if !ok {
 					return // closed channel
 				}
-				fmt.Printf("filesystem watcher encountered error:%v\n", err)
+				rlog.Errorf("filesystem watcher encountered error:%v", err)
 			}
 		}
 	}()
@@ -72,7 +72,7 @@ func getMaxEntriesDBByteSize() (int64, error) {
 func checkFileSize(maxSizeBytes int64) {
 	fileStat, err := os.Stat(DBPath)
 	if err != nil {
-		fmt.Printf("Error checking %s file size: %v\n", DBPath, err)
+		rlog.Errorf("Error checking %s file size: %v", DBPath, err)
 	} else {
 		if fileStat.Size() > maxSizeBytes {
 			pruneOldEntries(fileStat.Size())
@@ -83,13 +83,13 @@ func checkFileSize(maxSizeBytes int64) {
 func pruneOldEntries(currentFileSize int64) {
 	// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
 	IsDBLocked = true
-	defer func() {IsDBLocked = false}()
+	defer func() { IsDBLocked = false }()
 
 	amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
 
 	rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()
 	if err != nil {
-		fmt.Printf("Error getting 10000 first db rows: %v\n", err)
+		rlog.Errorf("Error getting 10000 first db rows: %v", err)
 		return
 	}
 
@@ -102,7 +102,7 @@ func pruneOldEntries(currentFileSize int64) {
 		var entry models.MizuEntry
 		err = DB.ScanRows(rows, &entry)
 		if err != nil {
-			fmt.Printf("Error scanning db row: %v\n", err)
+			rlog.Errorf("Error scanning db row: %v", err)
 			continue
 		}
 
@@ -114,8 +114,8 @@ func pruneOldEntries(currentFileSize int64) {
 		GetEntriesTable().Where(entryIdsToRemove).Delete(models.MizuEntry{})
 		// VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this
 		DB.Exec("VACUUM")
-		fmt.Printf("Removed %d rows and cleared %s\n", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
+		rlog.Errorf("Removed %d rows and cleared %s", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
 	} else {
-		fmt.Println("Found no rows to remove when pruning")
+		rlog.Error("Found no rows to remove when pruning")
 	}
 }

agent/pkg/models/models.go

@@ -2,11 +2,14 @@ package models
 
 import (
 	"encoding/json"
+
+	"mizuserver/pkg/rules"
+	"mizuserver/pkg/utils"
+	"time"
+
 	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
-	"mizuserver/pkg/utils"
-	"time"
 )
 
 type DataUnmarshaler interface {
@@ -33,19 +36,35 @@ type MizuEntry struct {
 	ResolvedSource      string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
 	ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
 	IsOutgoing          bool   `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"`
-	EstimatedSizeBytes           int `json:"-" gorm:"column:estimatedSizeBytes"`
+	EstimatedSizeBytes  int    `json:"-" gorm:"column:estimatedSizeBytes"`
 }
 
 type BaseEntryDetails struct {
-	Id              string `json:"id,omitempty"`
-	Url             string `json:"url,omitempty"`
-	RequestSenderIp string `json:"requestSenderIp,omitempty"`
-	Service         string `json:"service,omitempty"`
-	Path            string `json:"path,omitempty"`
-	StatusCode      int    `json:"statusCode,omitempty"`
-	Method          string `json:"method,omitempty"`
-	Timestamp       int64  `json:"timestamp,omitempty"`
-	IsOutgoing      bool   `json:"isOutgoing,omitempty"`
+	Id              string          `json:"id,omitempty"`
+	Url             string          `json:"url,omitempty"`
+	RequestSenderIp string          `json:"requestSenderIp,omitempty"`
+	Service         string          `json:"service,omitempty"`
+	Path            string          `json:"path,omitempty"`
+	StatusCode      int             `json:"statusCode,omitempty"`
+	Method          string          `json:"method,omitempty"`
+	Timestamp       int64           `json:"timestamp,omitempty"`
+	IsOutgoing      bool            `json:"isOutgoing,omitempty"`
+	Latency         int64           `json:"latency,omitempty"`
+	Rules           ApplicableRules `json:"rules,omitempty"`
+}
+
+type ApplicableRules struct {
+	Latency int64 `json:"latency,omitempty"`
+	Status  bool  `json:"status,omitempty"`
+	NumberOfRules int `json:"numberOfRules,omitempty"`
+}
+
+func NewApplicableRules(status bool, latency int64, number int) ApplicableRules {
+	ar := ApplicableRules{}
+	ar.Status = status
+	ar.Latency = latency
+	ar.NumberOfRules = number
+	return ar
 }
 
 type FullEntryDetails struct {
@@ -101,25 +120,20 @@ func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *MizuEntry) error {
 	return nil
 }
 
-type EntryData struct {
-	Entry               string `json:"entry,omitempty"`
-	ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
-}
-
 type EntriesFilter struct {
-	Limit     int    `query:"limit" validate:"required,min=1,max=200"`
-	Operator  string `query:"operator" validate:"required,oneof='lt' 'gt'"`
-	Timestamp int64  `query:"timestamp" validate:"required,min=1"`
+	Limit     int    `form:"limit" validate:"required,min=1,max=200"`
+	Operator  string `form:"operator" validate:"required,oneof='lt' 'gt'"`
+	Timestamp int64  `form:"timestamp" validate:"required,min=1"`
 }
 
-type UploadEntriesRequestBody struct {
+type UploadEntriesRequestQuery struct {
 	Dest             string `form:"dest"`
 	SleepIntervalSec int    `form:"interval"`
 }
 
-type HarFetchRequestBody struct {
-	From int64 `query:"from"`
-	To   int64 `query:"to"`
+type HarFetchRequestQuery struct {
+	From int64 `form:"from"`
+	To   int64 `form:"to"`
 }
 
 type WebSocketEntryMessage struct {
@@ -132,6 +146,11 @@ type WebSocketTappedEntryMessage struct {
 	Data *tap.OutputChannelItem
 }
 
+type WebsocketOutboundLinkMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *tap.OutboundLink
+}
+
 func CreateBaseEntryWebSocketMessage(base *BaseEntryDetails) ([]byte, error) {
 	message := &WebSocketEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
@@ -152,6 +171,16 @@ func CreateWebsocketTappedEntryMessage(base *tap.OutputChannelItem) ([]byte, err
 	return json.Marshal(message)
 }
 
+func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error) {
+	message := &WebsocketOutboundLinkMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebsocketMessageTypeOutboundLink,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
 // ExtendedHAR is the top level object of a HAR log.
 type ExtendedHAR struct {
 	Log *ExtendedLog `json:"log"`
@@ -171,3 +200,27 @@ type ExtendedCreator struct {
 	*har.Creator
 	Source *string `json:"_source"`
 }
+
+type FullEntryWithPolicy struct {
+	RulesMatched []rules.RulesMatched `json:"rulesMatched,omitempty"`
+	Entry        har.Entry            `json:"entry"`
+	Service      string               `json:"service"`
+}
+
+func (fewp *FullEntryWithPolicy) UnmarshalData(entry *MizuEntry) error {
+	if err := json.Unmarshal([]byte(entry.Entry), &fewp.Entry); err != nil {
+		return err
+	}
+
+	_, resultPolicyToSend := rules.MatchRequestPolicy(fewp.Entry, entry.Service)
+	fewp.RulesMatched = resultPolicyToSend
+	fewp.Service = entry.Service
+	return nil
+}
+
+func RunValidationRulesState(harEntry har.Entry, service string) ApplicableRules {
+	numberOfRules, resultPolicyToSend := rules.MatchRequestPolicy(harEntry, service)
+	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend, numberOfRules)
+	ar := NewApplicableRules(statusPolicyToSend, latency, numberOfRules)
+	return ar
+}

agent/pkg/providers/stats_provider.go

@@ -0,0 +1,36 @@
+package providers
+
+import (
+	"reflect"
+	"time"
+)
+
+type GeneralStats struct {
+	EntriesCount        int
+	FirstEntryTimestamp int
+	LastEntryTimestamp  int
+}
+
+var generalStats = GeneralStats{}
+
+func ResetGeneralStats() {
+	generalStats = GeneralStats{}
+}
+
+func GetGeneralStats() GeneralStats {
+	return generalStats
+}
+
+func EntryAdded() {
+	generalStats.EntriesCount++
+
+	currentTimestamp := int(time.Now().Unix())
+
+	if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) {
+		generalStats.FirstEntryTimestamp = currentTimestamp
+	}
+
+	generalStats.LastEntryTimestamp = currentTimestamp
+}
+
+

agent/pkg/providers/stats_provider_test.go

@@ -0,0 +1,35 @@
+package providers_test
+
+import (
+	"fmt"
+	"mizuserver/pkg/providers"
+	"testing"
+)
+
+func TestNoEntryAddedCount(t *testing.T) {
+	entriesStats := providers.GetGeneralStats()
+
+	if entriesStats.EntriesCount != 0 {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesCount)
+	}
+}
+
+func TestEntryAddedCount(t *testing.T) {
+	tests := []int{1, 5, 10, 100, 500, 1000}
+
+	for _, entriesCount := range tests {
+		t.Run(fmt.Sprintf("EntriesCount%v", entriesCount), func(t *testing.T) {
+			t.Cleanup(providers.ResetGeneralStats)
+
+			for i := 0; i < entriesCount; i++ {
+				providers.EntryAdded()
+			}
+
+			entriesStats := providers.GetGeneralStats()
+
+			if entriesStats.EntriesCount != entriesCount {
+				t.Errorf("unexpected result - expected: %v, actual: %v", entriesCount, entriesStats.EntriesCount)
+			}
+		})
+	}
+}

agent/pkg/providers/status_provider.go

@@ -0,0 +1,28 @@
+package providers
+
+import (
+	"github.com/patrickmn/go-cache"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap"
+	"time"
+)
+
+const tlsLinkRetainmentTime = time.Minute * 15
+
+var (
+	TapStatus shared.TapStatus
+	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
+)
+
+func GetAllRecentTLSAddresses() []string {
+	recentTLSLinks := make([]string, 0)
+
+	for _, outboundLinkItem := range RecentTLSLinks.Items() {
+		outboundLink, castOk := outboundLinkItem.Object.(*tap.OutboundLink)
+		if castOk {
+			recentTLSLinks = append(recentTLSLinks, outboundLink.DstIP)
+		}
+	}
+
+	return recentTLSLinks
+}

agent/pkg/resolver/README.md

@@ -32,7 +32,7 @@ Now you will be able to import `github.com/up9inc/mizu/resolver` in any `.go` fi
 errOut := make(chan error, 100)
 k8sResolver, err := resolver.NewFromOutOfCluster("", errOut)
 if err != nil {
-    fmt.Printf("error creating k8s resolver %s", err)
+    rlog.Errorf("error creating k8s resolver %s", err)
 }
 
 ctx, cancel := context.WithCancel(context.Background())
@@ -40,15 +40,15 @@ k8sResolver.Start(ctx)
 
 resolvedName := k8sResolver.Resolve("10.107.251.91") // will always return `nil` in real scenarios as the internal map takes a moment to populate after `Start` is called
 if resolvedName != nil {
-    fmt.Printf("resolved 10.107.251.91=%s", *resolvedName)
+    rlog.Errorf("resolved 10.107.251.91=%s", *resolvedName)
 } else {
-    fmt.Printf("Could not find a resolved name for 10.107.251.91")
+    rlog.Error("Could not find a resolved name for 10.107.251.91")
 }
 
 for {
     select {
         case err := <- errOut:
-            fmt.Printf("name resolving error %s", err)
+            rlog.Errorf("name resolving error %s", err)
     }
 }
 ```

agent/pkg/resolver/loader.go

@@ -1,6 +1,7 @@
 package resolver
 
 import (
+	cmap "github.com/orcaman/concurrent-map"
 	"k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@@ -9,7 +10,7 @@ import (
 	restclient "k8s.io/client-go/rest"
 )
 
-func NewFromInCluster(errOut chan error) (*Resolver, error) {
+func NewFromInCluster(errOut chan error, namesapce string) (*Resolver, error) {
 	config, err := restclient.InClusterConfig()
 	if err != nil {
 		return nil, err
@@ -18,5 +19,5 @@ func NewFromInCluster(errOut chan error) (*Resolver, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &Resolver{clientConfig: config, clientSet: clientset, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}, nil
+	return &Resolver{clientConfig: config, clientSet: clientset, nameMap: cmap.New(), serviceMap: cmap.New(), errOut: errOut, namespace: namesapce}, nil
 }

agent/pkg/resolver/resolver.go

@@ -7,28 +7,32 @@ import (
 	"github.com/romana/rlog"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 
+	"github.com/orcaman/concurrent-map"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 )
+
 const (
 	kubClientNullString = "None"
 )
 
 type Resolver struct {
-	clientConfig  *restclient.Config
-	clientSet *kubernetes.Clientset
-	nameMap map[string]string
-	serviceMap map[string]string
-	isStarted bool
-	errOut chan error
+	clientConfig *restclient.Config
+	clientSet    *kubernetes.Clientset
+	nameMap      cmap.ConcurrentMap
+	serviceMap   cmap.ConcurrentMap
+	isStarted    bool
+	errOut       chan error
+	namespace    string
 }
 
 func (resolver *Resolver) Start(ctx context.Context) {
 	if !resolver.isStarted {
 		resolver.isStarted = true
+
 		go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchServices)
 		go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchEndpoints)
 		go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchPods)
@@ -36,97 +40,97 @@ func (resolver *Resolver) Start(ctx context.Context) {
 }
 
 func (resolver *Resolver) Resolve(name string) string {
-	resolvedName, isFound := resolver.nameMap[name]
+	resolvedName, isFound := resolver.nameMap.Get(name)
 	if !isFound {
 		return ""
 	}
-	return resolvedName
+	return resolvedName.(string)
 }
 
-func (resolver *Resolver) GetMap() map[string]string {
+func (resolver *Resolver) GetMap() cmap.ConcurrentMap {
 	return resolver.nameMap
 }
 
 func (resolver *Resolver) CheckIsServiceIP(address string) bool {
-	_, isFound := resolver.serviceMap[address]
+	_, isFound := resolver.serviceMap.Get(address)
 	return isFound
 }
 
 func (resolver *Resolver) watchPods(ctx context.Context) error {
 	// empty namespace makes the client watch all namespaces
-	watcher, err := resolver.clientSet.CoreV1().Pods("").Watch(ctx, metav1.ListOptions{Watch: true})
+	watcher, err := resolver.clientSet.CoreV1().Pods(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true})
 	if err != nil {
 		return err
 	}
 	for {
 		select {
-			case event := <- watcher.ResultChan():
-				if event.Object == nil {
-					return errors.New("error in kubectl pod watch")
-				}
-				if event.Type == watch.Deleted {
-					pod := event.Object.(*corev1.Pod)
-					resolver.saveResolvedName(pod.Status.PodIP, "", event.Type)
-				}
-			case <- ctx.Done():
-				watcher.Stop()
-				return nil
+		case event := <-watcher.ResultChan():
+			if event.Object == nil {
+				return errors.New("error in kubectl pod watch")
+			}
+			if event.Type == watch.Deleted {
+				pod := event.Object.(*corev1.Pod)
+				resolver.saveResolvedName(pod.Status.PodIP, "", event.Type)
+			}
+		case <-ctx.Done():
+			watcher.Stop()
+			return nil
 		}
 	}
 }
 
 func (resolver *Resolver) watchEndpoints(ctx context.Context) error {
 	// empty namespace makes the client watch all namespaces
-	watcher, err := resolver.clientSet.CoreV1().Endpoints("").Watch(ctx, metav1.ListOptions{Watch: true})
+	watcher, err := resolver.clientSet.CoreV1().Endpoints(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true})
 	if err != nil {
 		return err
 	}
 	for {
 		select {
-			case event := <- watcher.ResultChan():
-				if event.Object == nil {
-					return errors.New("error in kubectl endpoint watch")
-				}
-				endpoint := event.Object.(*corev1.Endpoints)
-				serviceHostname := fmt.Sprintf("%s.%s", endpoint.Name, endpoint.Namespace)
-				if endpoint.Subsets != nil {
-					for _, subset := range endpoint.Subsets {
-						var ports []int32
-						if subset.Ports != nil {
-							for _, portMapping := range subset.Ports {
-								if portMapping.Port > 0 {
-									ports = append(ports, portMapping.Port)
-								}
+		case event := <-watcher.ResultChan():
+			if event.Object == nil {
+				return errors.New("error in kubectl endpoint watch")
+			}
+			endpoint := event.Object.(*corev1.Endpoints)
+			serviceHostname := fmt.Sprintf("%s.%s", endpoint.Name, endpoint.Namespace)
+			if endpoint.Subsets != nil {
+				for _, subset := range endpoint.Subsets {
+					var ports []int32
+					if subset.Ports != nil {
+						for _, portMapping := range subset.Ports {
+							if portMapping.Port > 0 {
+								ports = append(ports, portMapping.Port)
 							}
 						}
-						if subset.Addresses != nil {
-							for _, address := range subset.Addresses {
-								resolver.saveResolvedName(address.IP, serviceHostname, event.Type)
-								for _, port := range ports {
-									ipWithPort := fmt.Sprintf("%s:%d", address.IP, port)
-									resolver.saveResolvedName(ipWithPort, serviceHostname, event.Type)
-								}
-							}
-						}
-
 					}
+					if subset.Addresses != nil {
+						for _, address := range subset.Addresses {
+							resolver.saveResolvedName(address.IP, serviceHostname, event.Type)
+							for _, port := range ports {
+								ipWithPort := fmt.Sprintf("%s:%d", address.IP, port)
+								resolver.saveResolvedName(ipWithPort, serviceHostname, event.Type)
+							}
+						}
+					}
+
 				}
-			case <- ctx.Done():
-				watcher.Stop()
-				return nil
+			}
+		case <-ctx.Done():
+			watcher.Stop()
+			return nil
 		}
 	}
 }
 
 func (resolver *Resolver) watchServices(ctx context.Context) error {
 	// empty namespace makes the client watch all namespaces
-	watcher, err := resolver.clientSet.CoreV1().Services("").Watch(ctx, metav1.ListOptions{Watch: true})
+	watcher, err := resolver.clientSet.CoreV1().Services(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true})
 	if err != nil {
 		return err
 	}
 	for {
 		select {
-		case event := <- watcher.ResultChan():
+		case event := <-watcher.ResultChan():
 			if event.Object == nil {
 				return errors.New("error in kubectl service watch")
 			}
@@ -142,7 +146,7 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
 					resolver.saveResolvedName(ingress.IP, serviceHostname, event.Type)
 				}
 			}
-		case <- ctx.Done():
+		case <-ctx.Done():
 			watcher.Stop()
 			return nil
 		}
@@ -151,19 +155,19 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
 
 func (resolver *Resolver) saveResolvedName(key string, resolved string, eventType watch.EventType) {
 	if eventType == watch.Deleted {
-		delete(resolver.nameMap, key)
+		resolver.nameMap.Remove(key)
 		rlog.Infof("setting %s=nil\n", key)
 	} else {
-		resolver.nameMap[key] = resolved
+		resolver.nameMap.Set(key, resolved)
 		rlog.Infof("setting %s=%s\n", key, resolved)
 	}
 }
 
 func (resolver *Resolver) saveServiceIP(key string, resolved string, eventType watch.EventType) {
 	if eventType == watch.Deleted {
-		delete(resolver.serviceMap, key)
+		resolver.serviceMap.Remove(key)
 	} else {
-		resolver.serviceMap[key] = resolved
+		resolver.serviceMap.Set(key, resolved)
 	}
 }
 
@@ -186,4 +190,3 @@ func (resolver *Resolver) infiniteErrorHandleRetryFunc(ctx context.Context, fun
 		}
 	}
 }
-

agent/pkg/routes/entries_routes.go

@@ -22,4 +22,5 @@ func EntriesRoutes(ginApp *gin.Engine) {
 
 	routeGroup.GET("/tapStatus", controllers.GetTappingStatus) // get tapping status
 	routeGroup.GET("/analyzeStatus", controllers.AnalyzeInformation)
+	routeGroup.GET("/recentTLSLinks", controllers.GetRecentTLSLinks)
 }

agent/pkg/routes/status_routes.go

@@ -0,0 +1,12 @@
+package routes
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/controllers"
+)
+
+func StatusRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/status")
+
+	routeGroup.POST("/tappedPods", controllers.PostTappedPods)
+}

agent/pkg/rules/models.go

@@ -0,0 +1,110 @@
+package rules
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"regexp"
+	"strings"
+
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared"
+	jsonpath "github.com/yalp/jsonpath"
+)
+
+type RulesMatched struct {
+	Matched bool              `json:"matched"`
+	Rule    shared.RulePolicy `json:"rule"`
+}
+
+func appendRulesMatched(rulesMatched []RulesMatched, matched bool, rule shared.RulePolicy) []RulesMatched {
+	return append(rulesMatched, RulesMatched{Matched: matched, Rule: rule})
+}
+
+func ValidatePath(URLFromRule string, URL string) bool {
+	if URLFromRule != "" {
+		matchPath, err := regexp.MatchString(URLFromRule, URL)
+		if err != nil || !matchPath {
+			return false
+		}
+	}
+	return true
+}
+
+func ValidateService(serviceFromRule string, service string) bool {
+	if serviceFromRule != "" {
+		matchService, err := regexp.MatchString(serviceFromRule, service)
+		if err != nil || !matchService {
+			return false
+		}
+	}
+	return true
+}
+
+func MatchRequestPolicy(harEntry har.Entry, service string) (int, []RulesMatched) {
+	enforcePolicy, _ := shared.DecodeEnforcePolicy(fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.RulePolicyFileName))
+	var resultPolicyToSend []RulesMatched
+	for _, rule := range enforcePolicy.Rules {
+		if !ValidatePath(rule.Path, harEntry.Request.URL) || !ValidateService(rule.Service, service) {
+			continue
+		}
+		if rule.Type == "json" {
+			var bodyJsonMap interface{}
+			if err := json.Unmarshal(harEntry.Response.Content.Text, &bodyJsonMap); err != nil {
+				continue
+			}
+			out, err := jsonpath.Read(bodyJsonMap, rule.Key)
+			if err != nil || out == nil {
+				continue
+			}
+			var matchValue bool
+			if reflect.TypeOf(out).Kind() == reflect.String {
+				matchValue, err = regexp.MatchString(rule.Value, out.(string))
+				if err != nil {
+					continue
+				}
+			} else {
+				val := fmt.Sprint(out)
+				matchValue, err = regexp.MatchString(rule.Value, val)
+				if err != nil {
+					continue
+				}
+			}
+			resultPolicyToSend = appendRulesMatched(resultPolicyToSend, matchValue, rule)
+		} else if rule.Type == "header" {
+			for j := range harEntry.Response.Headers {
+				matchKey, err := regexp.MatchString(rule.Key, harEntry.Response.Headers[j].Name)
+				if err != nil {
+					continue
+				}
+				if matchKey {
+					matchValue, err := regexp.MatchString(rule.Value, harEntry.Response.Headers[j].Value)
+					if err != nil {
+						continue
+					}
+					resultPolicyToSend = appendRulesMatched(resultPolicyToSend, matchValue, rule)
+				}
+			}
+		} else {
+			resultPolicyToSend = appendRulesMatched(resultPolicyToSend, true, rule)
+		}
+	}
+	return len(enforcePolicy.Rules), resultPolicyToSend
+}
+
+func PassedValidationRules(rulesMatched []RulesMatched, numberOfRules int) (bool, int64, int) {
+	if len(rulesMatched) == 0 {
+		return false, 0, 0
+	}
+	for _, rule := range rulesMatched {
+		if rule.Matched == false {
+			return false, -1, len(rulesMatched)
+		}
+	}
+	for _, rule := range rulesMatched {
+		if strings.ToLower(rule.Rule.Type) == "latency" {
+			return true, rule.Rule.Latency, len(rulesMatched)
+		}
+	}
+	return true, -1, len(rulesMatched)
+}

agent/pkg/utils/truncating_logger.go

@@ -3,6 +3,7 @@ package utils
 import (
 	"context"
 	"fmt"
+	"github.com/romana/rlog"
 	"gorm.io/gorm/logger"
 	"gorm.io/gorm/utils"
 	"time"
@@ -10,7 +11,7 @@ import (
 
 // TruncatingLogger implements the gorm logger.Interface interface. Its purpose is to act as gorm's logger while truncating logs to a max of 50 characters to minimise the performance impact
 type TruncatingLogger struct {
-	LogLevel logger.LogLevel
+	LogLevel      logger.LogLevel
 	SlowThreshold time.Duration
 }
 
@@ -23,21 +24,21 @@ func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string
 	if truncatingLogger.LogLevel < logger.Info {
 		return
 	}
-	fmt.Printf("gorm info: %.150s\n", message)
+	rlog.Errorf("gorm info: %.150s", message)
 }
 
 func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) {
 	if truncatingLogger.LogLevel < logger.Warn {
 		return
 	}
-	fmt.Printf("gorm warning: %.150s\n", message)
+	rlog.Errorf("gorm warning: %.150s", message)
 }
 
 func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) {
 	if truncatingLogger.LogLevel < logger.Error {
 		return
 	}
-	fmt.Printf("gorm error: %.150s\n", message)
+	rlog.Errorf("gorm error: %.150s", message)
 }
 
 func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) {

agent/start.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-./mizuagent -i any -hardump -targets ${TAPPED_ADDRESSES}

build-push-featurebranch.sh

@@ -5,7 +5,9 @@ SERVER_NAME=mizu
 GCP_PROJECT=up9-docker-hub
 REPOSITORY=gcr.io/$GCP_PROJECT
 GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
-DOCKER_TAGGED_BUILD=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH:latest
+SEM_VER=${SEM_VER=0.0.0}
+DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
+DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
 
 if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
 then
@@ -13,8 +15,12 @@ then
   exit 1
 fi
 
-echo "building $DOCKER_TAGGED_BUILD"
-docker build -t "$DOCKER_TAGGED_BUILD" --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
+echo "building ${DOCKER_TAGGED_BUILDS[@]}"
+DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
+docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
 
-echo pushing to "$REPOSITORY"
-docker push "$DOCKER_TAGGED_BUILD"
+for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
+do
+        echo pushing "$DOCKER_TAG"
+        docker push "$DOCKER_TAG"
+done

cli/Makefile

@@ -3,6 +3,7 @@ COMMIT_HASH=$(shell git rev-parse HEAD)
 GIT_BRANCH=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
 GIT_VERSION=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
 BUILD_TIMESTAMP=$(shell date +%s)
+export SEM_VER?=0.0.0
 
 .PHONY: help
 .DEFAULT_GOAL := help
@@ -13,7 +14,7 @@ help: ## This help.
 install:
 	go install mizu.go
 
-build:	## build mizu CLI binary (select platform via GOOS / GOARCH env variables)
+build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
 	go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
@@ -21,7 +22,7 @@ build:	## build mizu CLI binary (select platform via GOOS / GOARCH env variables
 					   -o bin/mizu_$(SUFFIX) mizu.go 
 	(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
 
-build-all:  ## build for all supported platforms
+build-all: ## Build for all supported platforms.
 	@echo "Compiling for every OS and Platform"
 	@mkdir -p bin && echo "SHA256 checksums available for compiled binaries \n\nRun \`shasum -a 256 -c mizu_OS_ARCH.sha256\` to verify\n\n" >  bin/README.md
 	@$(MAKE) build GOOS=darwin GOARCH=amd64
@@ -35,6 +36,9 @@ build-all:  ## build for all supported platforms
 	@echo "---------"
 	@find ./bin -ls
 
-clean: ## clean all build artifacts
+clean: ## Clean all build artifacts.
 	go clean
 	rm -rf ./bin/*
+
+test: ## Run cli tests.
+	@go test ./... -race -coverprofile=coverage.out -covermode=atomic

cli/README.md

@@ -1,26 +0,0 @@
-# mizu CLI
-## Usage
-`./mizu {pod_name_regex}`
-
-### Optional Flags
-
-| flag                 | default          | purpose                                                                                                      |
-|----------------------|------------------|--------------------------------------------------------------------------------------------------------------|
-| `--no-gui`           | `false`          | Don't host the web interface (not applicable at the moment)                                                      |
-| `--gui-port`         | `8899`           | local port that web interface will be forwarded to                                                               |
-| `--namespace`        |                  | use namespace different than the one found in kubeconfig                                                     |
-| `--kubeconfig`       |                  | Path to custom kubeconfig file                                                                               |
-
-There are some extra flags defined in code that will show up in `./mizu --help`, these are non functional stubs for now
-
-## Installation
-Make sure your go version is at least 1.11
-1. cd to `mizu/cli`
-2. Run `go mod download` (may take a moment)
-3. Run `go build mizu.go`
-
-Alternatively, you can build+run directly using `go run mizu.go {pod_name_regex}`
-
-
-## Known issues
-* mid-flight port forwarding failures are not detected and no indication will be shown when this occurs

cli/cmd/config.go

@@ -3,24 +3,35 @@ package cmd
 import (
 	"fmt"
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"io/ioutil"
 )
 
-var outputFileName string
+var regenerateFile bool
 
 var configCmd = &cobra.Command{
 	Use:   "config",
-	Short: "Generate example config file to stdout",
+	Short: "Generate config with default values",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		template := mizu.GetTemplateConfig()
-		if outputFileName != "" {
+		go telemetry.ReportRun("config", config.Config)
+
+		template, err := config.GetConfigWithDefaults()
+		if err != nil {
+			logger.Log.Errorf("Failed generating config with defaults %v", err)
+			return nil
+		}
+		if regenerateFile {
 			data := []byte(template)
-			_ = ioutil.WriteFile(outputFileName, data, 0644)
-			mizu.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, outputFileName)))
+			if err := ioutil.WriteFile(config.GetConfigFilePath(), data, 0644); err != nil {
+				logger.Log.Errorf("Failed writing config %v", err)
+				return nil
+			}
+			logger.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, config.GetConfigFilePath())))
 		} else {
-			mizu.Log.Debugf("Writing template config.\n%v", template)
+			logger.Log.Debugf("Writing template config.\n%v", template)
 			fmt.Printf("%v", template)
 		}
 		return nil
@@ -29,6 +40,5 @@ var configCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(configCmd)
-
-	configCmd.Flags().StringVarP(&outputFileName, "file", "f", "", "Save content to local file")
+	configCmd.Flags().BoolVarP(&regenerateFile, "regenerate", "r", false, fmt.Sprintf("Regenerate the config file with default values %s", config.GetConfigFilePath()))
 }

cli/cmd/demo.go

@@ -1,32 +0,0 @@
-package cmd
-
-import (
-	"github.com/spf13/cobra"
-)
-
-type MizuDemoOptions struct {
-	GuiPort            uint16
-	Analyze            bool
-	AnalyzeDestination string
-}
-
-var mizuDemoOptions = &MizuDemoOptions{}
-
-var demoCmd = &cobra.Command{
-	Use:   "demo",
-	Short: "Record ingoing traffic of a kubernetes pod",
-	Long: `Record the ingoing traffic of a kubernetes pod.
-Supported protocols are HTTP and gRPC.`,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		RunMizuTapDemo(mizuDemoOptions)
-		return nil
-	},
-}
-
-func init() {
-	rootCmd.AddCommand(demoCmd)
-
-	demoCmd.Flags().Uint16VarP(&mizuDemoOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
-	demoCmd.Flags().BoolVar(&mizuDemoOptions.Analyze, "analyze", false, "Uploads traffic to UP9 cloud for further analysis (Beta)")
-	demoCmd.Flags().StringVar(&mizuDemoOptions.AnalyzeDestination, "dest", "up9.app", "Destination environment")
-}

cli/cmd/demoRunner.go

@@ -1,185 +0,0 @@
-package cmd
-
-import (
-	"archive/zip"
-	"bytes"
-	"context"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"os"
-	"os/exec"
-	"os/signal"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/up9inc/mizu/cli/uiUtils"
-)
-
-func RunMizuTapDemo(demoOptions *MizuDemoOptions) {
-	dir, _ := os.Getwd()
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	downloadMizuDemo(dir)
-
-	go callMizuDemo(ctx, cancel, dir, demoOptions)
-	if demoOptions.Analyze {
-		go analyze(demoOptions)
-		fmt.Printf(uiUtils.Purple, "mizu tap \"catalogue-.*|carts-[0-9].*|payment.*|shipping.*|user-[0-9].*\" -n sock-shop --analyze\n")
-	} else {
-		fmt.Printf(uiUtils.Purple, "mizu tap \"catalogue-.*|carts-[0-9].*|payment.*|shipping.*|user-[0-9].*\" -n sock-shop\n")
-	}
-	fmt.Println("Mizu will be available on http://localhost:8899 in a few seconds")
-	sigChan := make(chan os.Signal, 1)
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
-
-	select {
-	case <-ctx.Done():
-		break
-	case <-sigChan:
-		cleanUpDemoResources(dir)
-		cancel()
-	}
-}
-
-func cleanUpDemoResources(dir string) {
-	removeFile(fmt.Sprintf("%s/site.zip", dir))
-	removeFile(fmt.Sprintf("%s/site", dir))
-	removeFile(fmt.Sprintf("%s/apiserver.zip", dir))
-	removeFile(fmt.Sprintf("%s/apiserver", dir))
-	removeFile(fmt.Sprintf("%s/entries.db", dir))
-	removeFile(fmt.Sprintf("%s/hars", dir))
-	removeFile(fmt.Sprintf("%s/hars.zip", dir))
-}
-
-func removeFile(file string) {
-	err := os.RemoveAll(file)
-	if err != nil {
-		log.Fatal(err)
-	}
-}
-
-func downloadMizuDemo(dir string) {
-	if runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
-		panic("Platform not supported")
-	}
-	mizuApiURL := fmt.Sprintf("https://storage.googleapis.com/up9-mizu-demo-mode/apiserver-%s.zip", runtime.GOOS)
-	siteFileURL := "https://storage.googleapis.com/up9-mizu-demo-mode/site.zip"
-	harsURL := "https://storage.googleapis.com/up9-mizu-demo-mode/hars.zip"
-
-	dirApi := fmt.Sprintf("%s/apiserver.zip", dir)
-	dirSite := fmt.Sprintf("%s/site.zip", dir)
-	dirHars := fmt.Sprintf("%s/hars.zip", dir)
-
-	DownloadFile(dirApi, mizuApiURL)
-	DownloadFile(dirSite, siteFileURL)
-	DownloadFile(dirHars, harsURL)
-
-	UnzipSite(dirSite, fmt.Sprintf("%s/", dir))
-	UnzipSite(dirApi, fmt.Sprintf("%s/", dir))
-	UnzipSite(dirHars, fmt.Sprintf("%s/", dir))
-	allowExecutable(fmt.Sprintf("%s/apiserver", dir))
-}
-
-func DownloadFile(filepath string, url string) error {
-
-	resp, err := http.Get(url)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	out, err := os.Create(filepath)
-	if err != nil {
-		return err
-	}
-	defer out.Close()
-	_, err = io.Copy(out, resp.Body)
-	return err
-}
-
-func UnzipSite(src string, dest string) ([]string, error) {
-	var filenames []string
-
-	r, err := zip.OpenReader(src)
-	if err != nil {
-		return filenames, err
-	}
-	defer r.Close()
-
-	for _, f := range r.File {
-
-		fpath := filepath.Join(dest, f.Name)
-		if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) {
-			return filenames, fmt.Errorf("%s: illegal file path", fpath)
-		}
-
-		filenames = append(filenames, fpath)
-
-		if f.FileInfo().IsDir() {
-			os.MkdirAll(fpath, os.ModePerm)
-			continue
-		}
-		if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil {
-			return filenames, err
-		}
-
-		outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
-		if err != nil {
-			return filenames, err
-		}
-
-		rc, err := f.Open()
-		if err != nil {
-			return filenames, err
-		}
-
-		_, err = io.Copy(outFile, rc)
-		outFile.Close()
-		rc.Close()
-
-		if err != nil {
-			return filenames, err
-		}
-	}
-	return filenames, nil
-}
-
-func allowExecutable(dir string) {
-	if err := os.Chmod(dir, 0755); err != nil {
-		log.Fatalln(err)
-	}
-}
-
-func callMizuDemo(ctx context.Context, cancel context.CancelFunc, dir string, demoOptions *MizuDemoOptions) {
-	cmd := exec.Command(fmt.Sprintf("%s/apiserver", dir), "--aggregator", "--demo")
-	var out bytes.Buffer
-
-	// set the output to our variable
-	cmd.Stdout = &out
-	err := cmd.Run()
-	if err != nil {
-		log.Println(err)
-	}
-}
-
-func analyze(demoOptions *MizuDemoOptions) {
-	mizuProxiedUrl := getMizuCollectorProxiedHostAndPath(demoOptions.GuiPort)
-	for {
-		response, err := http.Get(fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=10", mizuProxiedUrl, demoOptions.AnalyzeDestination))
-		if err != nil || response.StatusCode != 200 {
-			fmt.Printf(uiUtils.Red, "Mizu Not running, waiting 10 seconds before trying again\n")
-		} else {
-			fmt.Printf(uiUtils.Purple, "Traffic is uploading to UP9 cloud for further analsys\n")
-			break
-		}
-		time.Sleep(10 * time.Second)
-	}
-}
-
-func getMizuCollectorProxiedHostAndPath(mizuPort uint16) string {
-	return fmt.Sprintf("localhost:%d", mizuPort)
-}

cli/cmd/fetch.go

@@ -1,30 +1,26 @@
 package cmd
 
 import (
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/mizu/version"
+	"github.com/up9inc/mizu/cli/telemetry"
 )
 
-type MizuFetchOptions struct {
-	FromTimestamp int64
-	ToTimestamp   int64
-	Directory     string
-	MizuPort      uint16
-}
-
-var mizuFetchOptions = MizuFetchOptions{}
-
 var fetchCmd = &cobra.Command{
 	Use:   "fetch",
 	Short: "Download recorded traffic to files",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go mizu.ReportRun("fetch", mizuTapOptions)
-		if isCompatible, err := mizu.CheckVersionCompatibility(mizuFetchOptions.MizuPort); err != nil {
+		go telemetry.ReportRun("fetch", config.Config.Fetch)
+
+		if isCompatible, err := version.CheckVersionCompatibility(config.Config.Fetch.GuiPort); err != nil {
 			return err
 		} else if !isCompatible {
 			return nil
 		}
-		RunMizuFetch(&mizuFetchOptions)
+		RunMizuFetch()
 		return nil
 	},
 }
@@ -32,8 +28,11 @@ var fetchCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(fetchCmd)
 
-	fetchCmd.Flags().StringVarP(&mizuFetchOptions.Directory, "directory", "d", ".", "Provide a custom directory for fetched entries")
-	fetchCmd.Flags().Int64Var(&mizuFetchOptions.FromTimestamp, "from", 0, "Custom start timestamp for fetched entries")
-	fetchCmd.Flags().Int64Var(&mizuFetchOptions.ToTimestamp, "to", 0, "Custom end timestamp fetched entries")
-	fetchCmd.Flags().Uint16VarP(&mizuFetchOptions.MizuPort, "port", "p", 8899, "Custom port for mizu")
+	defaultFetchConfig := configStructs.FetchConfig{}
+	defaults.Set(&defaultFetchConfig)
+
+	fetchCmd.Flags().StringP(configStructs.DirectoryFetchName, "d", defaultFetchConfig.Directory, "Provide a custom directory for fetched entries")
+	fetchCmd.Flags().Int(configStructs.FromTimestampFetchName, defaultFetchConfig.FromTimestamp, "Custom start timestamp for fetched entries")
+	fetchCmd.Flags().Int(configStructs.ToTimestampFetchName, defaultFetchConfig.ToTimestamp, "Custom end timestamp fetched entries")
+	fetchCmd.Flags().Uint16P(configStructs.GuiPortFetchName, "p", defaultFetchConfig.GuiPort, "Provide a custom port for the web interface webserver")
 }

cli/cmd/fetchRunner.go

@@ -4,8 +4,9 @@ import (
 	"archive/zip"
 	"bytes"
 	"fmt"
+	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/kubernetes"
-	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/logger"
 	"io"
 	"io/ioutil"
 	"log"
@@ -15,9 +16,9 @@ import (
 	"strings"
 )
 
-func RunMizuFetch(fetch *MizuFetchOptions) {
-	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(fetch.MizuPort)
-	resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, fetch.FromTimestamp, fetch.ToTimestamp))
+func RunMizuFetch() {
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Fetch.GuiPort)
+	resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, config.Config.Fetch.FromTimestamp, config.Config.Fetch.ToTimestamp))
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -33,8 +34,8 @@ func RunMizuFetch(fetch *MizuFetchOptions) {
 	if err != nil {
 		log.Fatal(err)
 	}
-	_ = Unzip(zipReader, fetch.Directory)
 
+	_ = Unzip(zipReader, config.Config.Fetch.Directory)
 }
 
 func Unzip(reader *zip.Reader, dest string) error {
@@ -64,7 +65,7 @@ func Unzip(reader *zip.Reader, dest string) error {
 			_ = os.MkdirAll(path, f.Mode())
 		} else {
 			_ = os.MkdirAll(filepath.Dir(path), f.Mode())
-			mizu.Log.Infof("writing HAR file [ %v ]", path)
+			logger.Log.Infof("writing HAR file [ %v ]", path)
 			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
 			if err != nil {
 				return err
@@ -73,7 +74,7 @@ func Unzip(reader *zip.Reader, dest string) error {
 				if err := f.Close(); err != nil {
 					panic(err)
 				}
-				mizu.Log.Info(" done")
+				logger.Log.Info(" done")
 			}()
 
 			_, err = io.Copy(f, rc)

cli/cmd/logs.go

@@ -0,0 +1,50 @@
+package cmd
+
+import (
+	"context"
+	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu/fsUtils"
+	"github.com/up9inc/mizu/cli/telemetry"
+	"os"
+	"path"
+)
+
+var filePath string
+
+var logsCmd = &cobra.Command{
+	Use:   "logs",
+	Short: "Create a zip file with logs for Github issue or troubleshoot",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("logs", config.Config)
+
+		kubernetesProvider, err := kubernetes.NewProvider(config.Config.View.KubeConfigPath)
+		if err != nil {
+			return nil
+		}
+		ctx, _ := context.WithCancel(context.Background())
+
+		if filePath == "" {
+			pwd, err := os.Getwd()
+			if err != nil {
+				logger.Log.Errorf("Failed to get PWD, %v (try using `mizu logs -f <full path dest zip file>)`", err)
+				return nil
+			}
+			filePath = path.Join(pwd, "mizu_logs.zip")
+		}
+		logger.Log.Debugf("Using file path %s", filePath)
+
+		if err := fsUtils.DumpLogs(kubernetesProvider, ctx, filePath); err != nil {
+			logger.Log.Errorf("Failed dump logs %v", err)
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(logsCmd)
+	logsCmd.Flags().StringVarP(&filePath, "file", "f", "", "Path for zip file (default current <pwd>\\mizu_logs.zip)")
+}

cli/cmd/root.go

@@ -1,32 +1,34 @@
 package cmd
 
 import (
-	"errors"
 	"fmt"
 	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 )
 
-var commandLineFlags []string
-
 var rootCmd = &cobra.Command{
 	Use:   "mizu",
 	Short: "A web traffic viewer for kubernetes",
 	Long: `A web traffic viewer for kubernetes
 Further info is available at https://github.com/up9inc/mizu`,
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-		if err := mizu.InitConfig(commandLineFlags); err != nil {
-			mizu.Log.Errorf("Invalid config, Exit %s", err)
-			return errors.New(fmt.Sprintf("%v", err))
+		if err := fsUtils.EnsureDir(mizu.GetMizuFolderPath()); err != nil {
+			logger.Log.Errorf("Failed to use mizu folder, %v", err)
 		}
-		prettifiedConfig := mizu.GetConfigStr()
-		mizu.Log.Debugf("Final Config: %s", prettifiedConfig)
+		logger.InitLogger()
+		if err := config.InitConfig(cmd); err != nil {
+			logger.Log.Fatal(err)
+		}
+
 		return nil
 	},
 }
 
 func init() {
-	rootCmd.PersistentFlags().StringSliceVar(&commandLineFlags, "set", []string{}, "Override values using --set")
+	rootCmd.PersistentFlags().StringSlice(config.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", config.SetCommandName))
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.

cli/cmd/tap.go

@@ -2,39 +2,18 @@ package cmd
 
 import (
 	"errors"
-	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/units"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"os"
-	"regexp"
-	"strings"
+
+	"github.com/creasty/defaults"
+	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/errormessage"
+	"github.com/up9inc/mizu/cli/uiUtils"
 )
 
-type MizuTapOptions struct {
-	GuiPort                uint16
-	Namespace              string
-	AllNamespaces          bool
-	Analysis               bool
-	AnalysisDestination    string
-	KubeConfigPath         string
-	MizuImage              string
-	PlainTextFilterRegexes []string
-	TapOutgoing            bool
-	HideHealthChecks       bool
-	MaxEntriesDBSizeBytes  int64
-	SleepIntervalSec       uint16
-	DisableRedaction       bool
-}
-
-var mizuTapOptions = &MizuTapOptions{}
-var direction string
-var humanMaxEntriesDBSize string
-var regex *regexp.Regexp
-
-const maxEntriesDBSizeFlagName = "max-entries-db-size"
-
 const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic for further analysis and enriched presentation options.`
 
 var tapCmd = &cobra.Command{
@@ -43,53 +22,35 @@ var tapCmd = &cobra.Command{
 	Long: `Record the ingoing traffic of a kubernetes pod.
 Supported protocols are HTTP and gRPC.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go mizu.ReportRun("tap", mizuTapOptions)
-		RunMizuTap(regex, mizuTapOptions)
+		go telemetry.ReportRun("tap", config.Config.Tap)
+		RunMizuTap()
 		return nil
-
 	},
 	PreRunE: func(cmd *cobra.Command, args []string) error {
-		mizu.Log.Debugf("Getting params")
-		mizuTapOptions.AnalysisDestination = mizu.GetString(mizu.ConfigurationKeyAnalyzingDestination)
-		mizuTapOptions.SleepIntervalSec = uint16(mizu.GetInt(mizu.ConfigurationKeyUploadInterval))
-		mizuTapOptions.MizuImage = mizu.GetString(mizu.ConfigurationKeyMizuImage)
-		mizu.Log.Debugf(uiUtils.PrettyJson(mizuTapOptions))
-
-		if len(args) == 0 {
-			return errors.New("POD REGEX argument is required")
+		if len(args) == 1 {
+			config.Config.Tap.PodRegexStr = args[0]
 		} else if len(args) > 1 {
 			return errors.New("unexpected number of arguments")
 		}
 
-		var compileErr error
-		regex, compileErr = regexp.Compile(args[0])
-		if compileErr != nil {
-			return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], compileErr))
+		if err := config.Config.Validate(); err != nil {
+			return errormessage.FormatError(err)
 		}
 
-		var parseHumanDataSizeErr error
-		mizuTapOptions.MaxEntriesDBSizeBytes, parseHumanDataSizeErr = units.HumanReadableToBytes(humanMaxEntriesDBSize)
-		if parseHumanDataSizeErr != nil {
-			return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize))
-		}
-		mizu.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
-
-		directionLowerCase := strings.ToLower(direction)
-		if directionLowerCase == "any" {
-			mizuTapOptions.TapOutgoing = true
-		} else if directionLowerCase == "in" {
-			mizuTapOptions.TapOutgoing = false
-		} else {
-			return errors.New(fmt.Sprintf("%s is not a valid value for flag --direction. Acceptable values are in/any.", direction))
+		if err := config.Config.Tap.Validate(); err != nil {
+			return errormessage.FormatError(err)
 		}
 
-		if mizuTapOptions.Analysis {
-			mizu.Log.Infof(analysisMessageToConfirm)
+		logger.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", config.Config.Tap.HumanMaxEntriesDBSize)
+
+		if config.Config.Tap.Analysis {
+			logger.Log.Infof(analysisMessageToConfirm)
 			if !uiUtils.AskForConfirmation("Would you like to proceed [Y/n]: ") {
-				mizu.Log.Infof("You can always run mizu without analysis, aborting")
+				logger.Log.Infof("You can always run mizu without analysis, aborting")
 				os.Exit(0)
 			}
 		}
+
 		return nil
 	},
 }
@@ -97,14 +58,17 @@ Supported protocols are HTTP and gRPC.`,
 func init() {
 	rootCmd.AddCommand(tapCmd)
 
-	tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
-	tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector")
-	tapCmd.Flags().BoolVar(&mizuTapOptions.Analysis, "analysis", false, "Uploads traffic to UP9 for further analysis (Beta)")
-	tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces")
-	tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
-	tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "regex-masking", "r", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies")
-	tapCmd.Flags().StringVarP(&direction, "direction", "", "in", "Record traffic that goes in this direction (relative to the tapped pod): in/any")
-	tapCmd.Flags().BoolVar(&mizuTapOptions.HideHealthChecks, "hide-healthchecks", false, "hides requests with kube-probe or prometheus user-agent headers")
-	tapCmd.Flags().StringVarP(&humanMaxEntriesDBSize, maxEntriesDBSizeFlagName, "", "200MB", "override the default max entries db size of 200mb")
-	tapCmd.Flags().BoolVar(&mizuTapOptions.DisableRedaction, "no-redact", false, "Disables redaction of potentially sensitive request/response headers and body values")
+	defaultTapConfig := configStructs.TapConfig{}
+	defaults.Set(&defaultTapConfig)
+
+	tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
+	tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")
+	tapCmd.Flags().Bool(configStructs.AnalysisTapName, defaultTapConfig.Analysis, "Uploads traffic to UP9 for further analysis (Beta)")
+	tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
+	tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
+	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
+	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
+	tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any")
+	tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
+	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file with policy rules")
 }

cli/cmd/tapRunner.go

@@ -1,172 +1,227 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu/fsUtils"
+	"github.com/up9inc/mizu/cli/mizu/goUtils"
+	"github.com/up9inc/mizu/cli/mizu/version"
+	"github.com/up9inc/mizu/cli/telemetry"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"path"
+	"regexp"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/kubernetes"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
+	yaml "gopkg.in/yaml.v3"
 	core "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/tools/clientcmd"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"os/signal"
-	"regexp"
-	"syscall"
-	"time"
 )
 
-var mizuServiceAccountExists bool
-var apiServerService *core.Service
-
 const (
-	updateTappersDelay = 5 * time.Second
 	cleanupTimeout     = time.Minute
+	updateTappersDelay = 5 * time.Second
 )
 
-var currentlyTappedPods []core.Pod
+type tapState struct {
+	apiServerService         *core.Service
+	currentlyTappedPods      []core.Pod
+	mizuServiceAccountExists bool
+	doNotRemoveConfigMap     bool
+}
 
-func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
-	mizuApiFilteringOptions, err := getMizuApiFilteringOptions(tappingOptions)
+var state tapState
+
+func RunMizuTap() {
+	mizuApiFilteringOptions, err := getMizuApiFilteringOptions()
 	if err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error parsing regex-masking: %v", errormessage.FormatError(err)))
+		return
+	}
+	var mizuValidationRules string
+	if config.Config.Tap.EnforcePolicyFile != "" {
+		mizuValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile)
+		if err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading policy file: %v", errormessage.FormatError(err)))
+			return
+		}
+	}
+
+	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath)
+	if err != nil {
+		logger.Log.Error(err)
 		return
 	}
 
-	kubernetesProvider, err := kubernetes.NewProvider(tappingOptions.KubeConfigPath)
-	if err != nil {
-		if clientcmd.IsEmptyConfig(err) {
-			mizu.Log.Infof(uiUtils.Red, "Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'\n")
-			return
-		}
-		if clientcmd.IsConfigurationInvalid(err) {
-			mizu.Log.Infof(uiUtils.Red, "Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'\n")
-			return
-		}
-	}
-
-	defer cleanUpMizuResources(kubernetesProvider)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel() // cancel will be called when this function exits
 
-	targetNamespace := getNamespace(tappingOptions, kubernetesProvider)
-	if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegexQuery, targetNamespace); err != nil {
-		mizu.Log.Infof("Error listing pods: %v", err)
-		return
-	} else {
-		currentlyTappedPods = matchingPods
-	}
+	targetNamespaces := getNamespaces(kubernetesProvider)
 
 	var namespacesStr string
-	if targetNamespace != mizu.K8sAllNamespaces {
-		namespacesStr = fmt.Sprintf("namespace \"%s\"", targetNamespace)
+	if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
+		namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(targetNamespaces, "\", \""))
 	} else {
 		namespacesStr = "all namespaces"
 	}
-	mizu.Log.Infof("Tapping pods in %s", namespacesStr)
+	version.CheckNewerVersion()
+	logger.Log.Infof("Tapping pods in %s", namespacesStr)
 
-	if len(currentlyTappedPods) == 0 {
+	if err, _ := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespaces); err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error getting pods by regex: %v", errormessage.FormatError(err)))
+		return
+	}
+
+	if len(state.currentlyTappedPods) == 0 {
 		var suggestionStr string
-		if targetNamespace != mizu.K8sAllNamespaces {
-			suggestionStr = "\nSelect a different namespace with -n or tap all namespaces with -A"
+		if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
+			suggestionStr = ". Select a different namespace with -n or tap all namespaces with -A"
 		}
-		mizu.Log.Infof("Did not find any pods matching the regex argument%s", suggestionStr)
+		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr))
 	}
 
-	nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
-	if err != nil {
+	if config.Config.Tap.DryRun {
 		return
 	}
 
-	if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions, mizuApiFilteringOptions); err != nil {
+	nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
+
+	defer cleanUpMizu(kubernetesProvider)
+	if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
 		return
 	}
 
-	mizu.CheckNewerVersion()
-	go portForwardApiPod(ctx, kubernetesProvider, cancel, tappingOptions) // TODO convert this to job for built in pod ttl or have the running app handle this
-	go watchPodsForTapping(ctx, kubernetesProvider, cancel, podRegexQuery, tappingOptions)
-	go syncApiStatus(ctx, cancel, tappingOptions)
+	go goUtils.HandleExcWrapper(createProxyToApiServerPod, ctx, kubernetesProvider, cancel)
+	go goUtils.HandleExcWrapper(watchPodsForTapping, ctx, kubernetesProvider, targetNamespaces, cancel)
 
 	//block until exit signal or error
 	waitForFinish(ctx, cancel)
 }
 
-func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
-	if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
+func readValidationRules(file string) (string, error) {
+	rules, err := shared.DecodeEnforcePolicy(file)
+	if err != nil {
+		return "", err
+	}
+	newContent, _ := yaml.Marshal(&rules)
+	return string(newContent), nil
+}
+
+func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, mizuApiFilteringOptions *shared.TrafficFilteringOptions, mizuValidationRules string) error {
+	if !config.Config.IsNsRestrictedMode() {
+		if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
+			return err
+		}
+	}
+
+	if err := createMizuApiServer(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
 		return err
 	}
 
-	if err := createMizuApiServer(ctx, kubernetesProvider, tappingOptions, mizuApiFilteringOptions); err != nil {
+	if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
 		return err
 	}
 
-	if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
-		return err
+	if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules); err != nil {
+		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err)))
+		state.doNotRemoveConfigMap = true
+	} else if mizuValidationRules == "" {
+		state.doNotRemoveConfigMap = true
 	}
 
 	return nil
 }
 
-func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider) error {
-	_, err := kubernetesProvider.CreateNamespace(ctx, mizu.ResourcesNamespace)
-	if err != nil {
-		mizu.Log.Infof("Error creating Namespace %s: %v", mizu.ResourcesNamespace, err)
-	}
-
+func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, data string) error {
+	err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName, data)
 	return err
 }
 
-func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
+func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider) error {
+	_, err := kubernetesProvider.CreateNamespace(ctx, config.Config.MizuResourcesNamespace)
+	return err
+}
+
+func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
 	var err error
 
-	mizuServiceAccountExists = createRBACIfNecessary(ctx, kubernetesProvider)
+	state.mizuServiceAccountExists, err = createRBACIfNecessary(ctx, kubernetesProvider)
+	if err != nil {
+		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err)))
+	}
+
 	var serviceAccountName string
-	if mizuServiceAccountExists {
+	if state.mizuServiceAccountExists {
 		serviceAccountName = mizu.ServiceAccountName
 	} else {
 		serviceAccountName = ""
 	}
-	_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, tappingOptions.MizuImage, serviceAccountName, mizuApiFilteringOptions, tappingOptions.MaxEntriesDBSizeBytes)
-	if err != nil {
-		mizu.Log.Infof("Error creating mizu %s pod: %v", mizu.ApiServerPodName, err)
-		return err
-	}
 
-	apiServerService, err = kubernetesProvider.CreateService(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, mizu.ApiServerPodName)
+	opts := &kubernetes.ApiServerOptions{
+		Namespace:               config.Config.MizuResourcesNamespace,
+		PodName:                 mizu.ApiServerPodName,
+		PodImage:                config.Config.AgentImage,
+		ServiceAccountName:      serviceAccountName,
+		IsNamespaceRestricted:   config.Config.IsNsRestrictedMode(),
+		MizuApiFilteringOptions: mizuApiFilteringOptions,
+		MaxEntriesDBSizeBytes:   config.Config.Tap.MaxEntriesDBSizeBytes(),
+	}
+	_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, opts, config.Config.Tap.ApiServerResources)
 	if err != nil {
-		mizu.Log.Infof("Error creating mizu %s service: %v", mizu.ApiServerPodName, err)
 		return err
 	}
+	logger.Log.Debugf("Successfully created API server pod: %s", mizu.ApiServerPodName)
+
+	state.apiServerService, err = kubernetesProvider.CreateService(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName, mizu.ApiServerPodName)
+	if err != nil {
+		return err
+	}
+	logger.Log.Debugf("Successfully created service: %s", mizu.ApiServerPodName)
 
 	return nil
 }
 
-func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.TrafficFilteringOptions, error) {
+func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
 	var compiledRegexSlice []*shared.SerializableRegexp
 
-	if tappingOptions.PlainTextFilterRegexes != nil && len(tappingOptions.PlainTextFilterRegexes) > 0 {
+	if config.Config.Tap.PlainTextFilterRegexes != nil && len(config.Config.Tap.PlainTextFilterRegexes) > 0 {
 		compiledRegexSlice = make([]*shared.SerializableRegexp, 0)
-		for _, regexStr := range tappingOptions.PlainTextFilterRegexes {
+		for _, regexStr := range config.Config.Tap.PlainTextFilterRegexes {
 			compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr)
 			if err != nil {
-				mizu.Log.Infof("Regex %s is invalid: %v", regexStr, err)
 				return nil, err
 			}
 			compiledRegexSlice = append(compiledRegexSlice, compiledRegex)
 		}
 	}
 
-	return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: tappingOptions.HideHealthChecks, DisableRedaction: tappingOptions.DisableRedaction}, nil
+	return &shared.TrafficFilteringOptions{
+		PlainTextMaskingRegexes:      compiledRegexSlice,
+		HealthChecksUserAgentHeaders: config.Config.Tap.HealthChecksUserAgentHeaders,
+		DisableRedaction:             config.Config.Tap.DisableRedaction,
+	}, nil
 }
 
-func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error {
+func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error {
 	if len(nodeToTappedPodIPMap) > 0 {
 		var serviceAccountName string
-		if mizuServiceAccountExists {
+		if state.mizuServiceAccountExists {
 			serviceAccountName = mizu.ServiceAccountName
 		} else {
 			serviceAccountName = ""
@@ -174,21 +229,21 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
 
 		if err := kubernetesProvider.ApplyMizuTapperDaemonSet(
 			ctx,
-			mizu.ResourcesNamespace,
+			config.Config.MizuResourcesNamespace,
 			mizu.TapperDaemonSetName,
-			tappingOptions.MizuImage,
+			config.Config.AgentImage,
 			mizu.TapperPodName,
-			fmt.Sprintf("%s.%s.svc.cluster.local", apiServerService.Name, apiServerService.Namespace),
+			fmt.Sprintf("%s.%s.svc.cluster.local", state.apiServerService.Name, state.apiServerService.Namespace),
 			nodeToTappedPodIPMap,
 			serviceAccountName,
-			tappingOptions.TapOutgoing,
+			config.Config.Tap.TapOutgoing(),
+			config.Config.Tap.TapperResources,
 		); err != nil {
-			mizu.Log.Infof("Error creating mizu tapper daemonset: %v", err)
 			return err
 		}
+		logger.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap))
 	} else {
-		if err := kubernetesProvider.RemoveDaemonSet(ctx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
-			mizu.Log.Infof("Error deleting mizu tapper daemonset: %v", err)
+		if err := kubernetesProvider.RemoveDaemonSet(ctx, config.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
 			return err
 		}
 	}
@@ -196,62 +251,143 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
 	return nil
 }
 
-func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
-	mizu.Log.Infof("\nRemoving mizu resources\n")
+func cleanUpMizu(kubernetesProvider *kubernetes.Provider) {
+	telemetry.ReportAPICalls(config.Config.Tap.GuiPort)
+	cleanUpMizuResources(kubernetesProvider)
+}
 
+func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
 	removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
 	defer cancel()
 
-	if err := kubernetesProvider.RemoveNamespace(removalCtx, mizu.ResourcesNamespace); err != nil {
-		mizu.Log.Infof("Error removing Namespace %s: %s (%v,%+v)", mizu.ResourcesNamespace, err, err, err)
-		return
-	}
-
-	if mizuServiceAccountExists {
-		if err := kubernetesProvider.RemoveNonNamespacedResources(removalCtx, mizu.ClusterRoleName, mizu.ClusterRoleBindingName); err != nil {
-			mizu.Log.Infof("Error removing non-namespaced resources: %s (%v,%+v)", err, err, err)
-			return
+	if config.Config.DumpLogs {
+		mizuDir := mizu.GetMizuFolderPath()
+		filePath = path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
+		if err := fsUtils.DumpLogs(kubernetesProvider, removalCtx, filePath); err != nil {
+			logger.Log.Errorf("Failed dump logs %v", err)
 		}
 	}
 
+	logger.Log.Infof("\nRemoving mizu resources\n")
+
+	if !config.Config.IsNsRestrictedMode() {
+		if err := kubernetesProvider.RemoveNamespace(removalCtx, config.Config.MizuResourcesNamespace); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Namespace %s: %v", config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			return
+		}
+	} else {
+		if err := kubernetesProvider.RemovePod(removalCtx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Pod %s in namespace %s: %v", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+		}
+
+		if err := kubernetesProvider.RemoveService(removalCtx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service %s in namespace %s: %v", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+		}
+
+		if err := kubernetesProvider.RemoveDaemonSet(removalCtx, config.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing DaemonSet %s in namespace %s: %v", mizu.TapperDaemonSetName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+		}
+
+		if !state.doNotRemoveConfigMap {
+			if err := kubernetesProvider.RemoveConfigMap(removalCtx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing ConfigMap %s in namespace %s: %v", mizu.ConfigMapName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			}
+		}
+
+	}
+
+	if state.mizuServiceAccountExists {
+		if !config.Config.IsNsRestrictedMode() {
+			if err := kubernetesProvider.RemoveNonNamespacedResources(removalCtx, mizu.ClusterRoleName, mizu.ClusterRoleBindingName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing non-namespaced resources: %v", errormessage.FormatError(err)))
+				return
+			}
+		} else {
+			if err := kubernetesProvider.RemoveServicAccount(removalCtx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service Account %s in namespace %s: %v", mizu.ServiceAccountName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+				return
+			}
+
+			if err := kubernetesProvider.RemoveRole(removalCtx, config.Config.MizuResourcesNamespace, mizu.RoleName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Role %s in namespace %s: %v", mizu.RoleName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			}
+
+			if err := kubernetesProvider.RemoveRoleBinding(removalCtx, config.Config.MizuResourcesNamespace, mizu.RoleBindingName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing RoleBinding %s in namespace %s: %v", mizu.RoleBindingName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			}
+		}
+	}
+
+	if !config.Config.IsNsRestrictedMode() {
+		waitUntilNamespaceDeleted(removalCtx, cancel, kubernetesProvider)
+	}
+}
+
+func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
 	// Call cancel if a terminating signal was received. Allows user to skip the wait.
 	go func() {
-		waitForFinish(removalCtx, cancel)
+		waitForFinish(ctx, cancel)
 	}()
 
-	if err := kubernetesProvider.WaitUtilNamespaceDeleted(removalCtx, mizu.ResourcesNamespace); err != nil {
+	if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, config.Config.MizuResourcesNamespace); err != nil {
 		switch {
-		case removalCtx.Err() == context.Canceled:
+		case ctx.Err() == context.Canceled:
 			// Do nothing. User interrupted the wait.
 		case err == wait.ErrWaitTimeout:
-			mizu.Log.Infof("Timeout while removing Namespace %s", mizu.ResourcesNamespace)
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", config.Config.MizuResourcesNamespace))
 		default:
-			mizu.Log.Infof("Error while waiting for Namespace %s to be deleted: %s (%v,%+v)", mizu.ResourcesNamespace, err, err, err)
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error while waiting for Namespace %s to be deleted: %v", config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
 		}
 	}
 }
 
-func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, podRegex *regexp.Regexp, tappingOptions *MizuTapOptions) {
-	targetNamespace := getNamespace(tappingOptions, kubernetesProvider)
+func reportTappedPods() {
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort)
+	tappedPodsUrl := fmt.Sprintf("http://%s/status/tappedPods", mizuProxiedUrl)
 
-	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, targetNamespace), podRegex)
+	podInfos := make([]shared.PodInfo, 0)
+	for _, pod := range state.currentlyTappedPods {
+		podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
+	}
+	tapStatus := shared.TapStatus{Pods: podInfos}
+
+	if jsonValue, err := json.Marshal(tapStatus); err != nil {
+		logger.Log.Debugf("[ERROR] failed Marshal the tapped pods %v", err)
+	} else {
+		if response, err := http.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+			logger.Log.Debugf("[ERROR] failed sending to API server the tapped pods %v", err)
+		} else if response.StatusCode != 200 {
+			logger.Log.Debugf("[ERROR] failed sending to API server the tapped pods, response status code %v", response.StatusCode)
+		} else {
+			logger.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos))
+		}
+	}
+}
+
+func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, targetNamespaces []string, cancel context.CancelFunc) {
+	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, targetNamespaces, config.Config.Tap.PodRegex())
 
 	restartTappers := func() {
-		if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex, targetNamespace); err != nil {
-			mizu.Log.Infof("Error getting pods by regex: %s (%v,%+v)", err, err, err)
-			cancel()
-		} else {
-			currentlyTappedPods = matchingPods
-		}
-
-		nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
+		err, changeFound := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespaces)
 		if err != nil {
-			mizu.Log.Infof("Error building node to ips map: %s (%v,%+v)", err, err, err)
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Failed to update currently tapped pods: %v", err))
 			cancel()
 		}
 
-		if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
-			mizu.Log.Infof("Error updating daemonset: %s (%v,%+v)", err, err, err)
+		if !changeFound {
+			logger.Log.Debugf("Nothing changed update tappers not needed")
+			return
+		}
+
+		reportTappedPods()
+
+		nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
+		if err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error building node to ips map: %v", errormessage.FormatError(err)))
+			cancel()
+		}
+		if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating daemonset: %v", errormessage.FormatError(err)))
 			cancel()
 		}
 	}
@@ -259,109 +395,187 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 
 	for {
 		select {
-		case newTarget := <-added:
-			mizu.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", newTarget.Name))
-
-		case removedTarget := <-removed:
-			mizu.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedTarget.Name))
+		case pod := <-added:
+			logger.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
 			restartTappersDebouncer.SetOn()
-
-		case modifiedTarget := <-modified:
+		case pod := <-removed:
+			logger.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
+			restartTappersDebouncer.SetOn()
+		case pod := <-modified:
+			logger.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
 			// Act only if the modified pod has already obtained an IP address.
 			// After filtering for IPs, on a normal pod restart this includes the following events:
 			// - Pod deletion
 			// - Pod reaches start state
 			// - Pod reaches ready state
 			// Ready/unready transitions might also trigger this event.
-			if modifiedTarget.Status.PodIP != "" {
+			if pod.Status.PodIP != "" {
 				restartTappersDebouncer.SetOn()
 			}
 
-		case <-errorChan:
+		case err := <-errorChan:
+			logger.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err)
+			restartTappersDebouncer.Cancel()
 			// TODO: Does this also perform cleanup?
 			cancel()
 
 		case <-ctx.Done():
+			logger.Log.Debugf("Watching pods loop, context done, stopping `restart tappers debouncer`")
+			restartTappersDebouncer.Cancel()
 			return
 		}
 	}
 }
 
-func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
+func updateCurrentlyTappedPods(kubernetesProvider *kubernetes.Provider, ctx context.Context, targetNamespaces []string) (error, bool) {
+	changeFound := false
+	if matchingPods, err := kubernetesProvider.ListAllRunningPodsMatchingRegex(ctx, config.Config.Tap.PodRegex(), targetNamespaces); err != nil {
+		return err, false
+	} else {
+		podsToTap := excludeMizuPods(matchingPods)
+		addedPods, removedPods := getPodArrayDiff(state.currentlyTappedPods, podsToTap)
+		for _, addedPod := range addedPods {
+			changeFound = true
+			logger.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", addedPod.Name))
+		}
+		for _, removedPod := range removedPods {
+			changeFound = true
+			logger.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedPod.Name))
+		}
+		state.currentlyTappedPods = podsToTap
+	}
+
+	return nil, changeFound
+}
+
+func excludeMizuPods(pods []core.Pod) []core.Pod {
+	mizuPrefixRegex := regexp.MustCompile("^" + mizu.MizuResourcesPrefix)
+
+	nonMizuPods := make([]core.Pod, 0)
+	for _, pod := range pods {
+		if !mizuPrefixRegex.MatchString(pod.Name) {
+			nonMizuPods = append(nonMizuPods, pod)
+		}
+	}
+
+	return nonMizuPods
+}
+
+func getPodArrayDiff(oldPods []core.Pod, newPods []core.Pod) (added []core.Pod, removed []core.Pod) {
+	added = getMissingPods(newPods, oldPods)
+	removed = getMissingPods(oldPods, newPods)
+
+	return added, removed
+}
+
+//returns pods present in pods1 array and missing in pods2 array
+func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod {
+	missingPods := make([]core.Pod, 0)
+	for _, pod1 := range pods1 {
+		var found = false
+		for _, pod2 := range pods2 {
+			if pod1.UID == pod2.UID {
+				found = true
+				break
+			}
+		}
+		if !found {
+			missingPods = append(missingPods, pod1)
+		}
+	}
+	return missingPods
+}
+
+func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName))
-	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, mizu.ResourcesNamespace), podExactRegex)
+	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
 	isPodReady := false
 	timeAfter := time.After(25 * time.Second)
 	for {
 		select {
 		case <-ctx.Done():
+			logger.Log.Debugf("Watching API Server pod loop, ctx done")
 			return
-
 		case <-added:
+			logger.Log.Debugf("Watching API Server pod loop, added")
 			continue
 		case <-removed:
-			mizu.Log.Infof("%s removed", mizu.ApiServerPodName)
+			logger.Log.Infof("%s removed", mizu.ApiServerPodName)
 			cancel()
 			return
 		case modifiedPod := <-modified:
-			if modifiedPod.Status.Phase == "Running" && !isPodReady {
-				isPodReady = true
-				go func() {
-					err := kubernetes.StartProxy(kubernetesProvider, tappingOptions.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
-					if err != nil {
-						mizu.Log.Infof("Error occured while running k8s proxy %v", err)
-						cancel()
-					}
-				}()
-				mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(tappingOptions.GuiPort)
-				mizu.Log.Infof("Mizu is available at http://%s", mizuProxiedUrl)
-
-				time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
-				if tappingOptions.Analysis {
-					urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(tappingOptions.AnalysisDestination), tappingOptions.SleepIntervalSec)
-					u, err := url.ParseRequestURI(urlPath)
-
-					if err != nil {
-						log.Fatal(fmt.Sprintf("Failed parsing the URL %v\n", err))
-					}
-					mizu.Log.Debugf("Sending get request to %v", u.String())
-					if response, err := http.Get(u.String()); err != nil || response.StatusCode != 200 {
-						mizu.Log.Infof("error sending upload entries req, status code: %v, err: %v", response.StatusCode, err)
-					} else {
-						mizu.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis")
-					}
-				}
+			if modifiedPod == nil {
+				logger.Log.Debugf("Watching API Server pod loop, modifiedPod with nil")
+				continue
+			}
+			logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
+			if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
+				isPodReady = true
+				go startProxyReportErrorIfAny(kubernetesProvider, cancel)
+				logger.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort))
+				time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
+				requestForAnalysis()
+				reportTappedPods()
 			}
-
 		case <-timeAfter:
 			if !isPodReady {
-				mizu.Log.Infof("error: %s pod was not ready in time", mizu.ApiServerPodName)
+				logger.Log.Errorf(uiUtils.Error, "Mizu API server was not ready in time")
 				cancel()
 			}
-
 		case <-errorChan:
+			logger.Log.Debugf("[ERROR] Agent creation, watching %v namespace", config.Config.MizuResourcesNamespace)
 			cancel()
 		}
 	}
 }
 
-func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
-	mizuRBACExists, err := kubernetesProvider.DoesServiceAccountExist(ctx, mizu.ResourcesNamespace, mizu.ServiceAccountName)
+func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+	err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
 	if err != nil {
-		mizu.Log.Infof("warning: could not ensure mizu rbac resources exist %v", err)
-		return false
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+
+			"Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName))
+		cancel()
 	}
-	if !mizuRBACExists {
-		err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.ResourcesNamespace, mizu.ServiceAccountName, mizu.ClusterRoleName, mizu.ClusterRoleBindingName, mizu.RBACVersion)
-		if err != nil {
-			mizu.Log.Infof("warning: could not create mizu rbac resources %v", err)
-			return false
-		}
-	}
-	return true
 }
 
-func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) (map[string][]string, error) {
+func requestForAnalysis() {
+	if !config.Config.Tap.Analysis {
+		return
+	}
+
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort)
+	urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(config.Config.Tap.AnalysisDestination), config.Config.Tap.SleepIntervalSec)
+	u, parseErr := url.ParseRequestURI(urlPath)
+	if parseErr != nil {
+		logger.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr)
+	}
+
+	logger.Log.Debugf("Sending get request to %v", u.String())
+	if response, requestErr := http.Get(u.String()); requestErr != nil {
+		logger.Log.Errorf("Failed to notify agent for analysis, err: %v", requestErr)
+	} else if response.StatusCode != 200 {
+		logger.Log.Errorf("Failed to notify agent for analysis, status code: %v", response.StatusCode)
+	} else {
+		logger.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis")
+	}
+}
+
+func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) (bool, error) {
+	if !config.Config.IsNsRestrictedMode() {
+		err := kubernetesProvider.CreateMizuRBAC(ctx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.ClusterRoleName, mizu.ClusterRoleBindingName, mizu.RBACVersion)
+		if err != nil {
+			return false, err
+		}
+	} else {
+		err := kubernetesProvider.CreateMizuRBACNamespaceRestricted(ctx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.RoleName, mizu.RoleBindingName, mizu.RBACVersion)
+		if err != nil {
+			return false, err
+		}
+	}
+	return true, nil
+}
+
+func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) map[string][]string {
 	nodeToTappedPodIPMap := make(map[string][]string, 0)
 	for _, pod := range tappedPods {
 		existingList := nodeToTappedPodIPMap[pod.Spec.NodeName]
@@ -371,7 +585,7 @@ func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) (map[string][]string, e
 			nodeToTappedPodIPMap[pod.Spec.NodeName] = append(nodeToTappedPodIPMap[pod.Spec.NodeName], pod.Status.PodIP)
 		}
 	}
-	return nodeToTappedPodIPMap, nil
+	return nodeToTappedPodIPMap
 }
 
 func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
@@ -387,35 +601,12 @@ func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
 	}
 }
 
-func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
-	controlSocketStr := fmt.Sprintf("ws://%s/ws", kubernetes.GetMizuApiServerProxiedHostAndPath(tappingOptions.GuiPort))
-	controlSocket, err := mizu.CreateControlSocket(controlSocketStr)
-	if err != nil {
-		mizu.Log.Infof("error establishing control socket connection %s", err)
-		cancel()
-	}
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		default:
-			err = controlSocket.SendNewTappedPodsListMessage(currentlyTappedPods)
-			if err != nil {
-				mizu.Log.Debugf("error Sending message via control socket %v, error: %s", controlSocketStr, err)
-			}
-			time.Sleep(10 * time.Second)
-		}
-	}
-
-}
-
-func getNamespace(tappingOptions *MizuTapOptions, kubernetesProvider *kubernetes.Provider) string {
-	if tappingOptions.AllNamespaces {
-		return mizu.K8sAllNamespaces
-	} else if len(tappingOptions.Namespace) > 0 {
-		return tappingOptions.Namespace
+func getNamespaces(kubernetesProvider *kubernetes.Provider) []string {
+	if config.Config.Tap.AllNamespaces {
+		return []string{mizu.K8sAllNamespaces}
+	} else if len(config.Config.Tap.Namespaces) > 0 {
+		return config.Config.Tap.Namespaces
 	} else {
-		return kubernetesProvider.CurrentNamespace()
+		return []string{kubernetesProvider.CurrentNamespace()}
 	}
 }

cli/cmd/version.go

@@ -1,30 +1,31 @@
 package cmd
 
 import (
-	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"strconv"
 	"time"
+
+	"github.com/creasty/defaults"
+	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/mizu"
 )
 
-type MizuVersionOptions struct {
-	DebugInfo bool
-}
-
-var mizuVersionOptions = &MizuVersionOptions{}
-
 var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print version info",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go mizu.ReportRun("version", mizuVersionOptions)
-		if mizuVersionOptions.DebugInfo {
+		go telemetry.ReportRun("version", config.Config.Version)
+
+		if config.Config.Version.DebugInfo {
 			timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
-			mizu.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
-			mizu.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
+			logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
+			logger.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
 
 		} else {
-			mizu.Log.Infof("Version: %s (%s)", mizu.SemVer, mizu.Branch)
+			logger.Log.Infof("Version: %s (%s)", mizu.SemVer, mizu.Branch)
 		}
 		return nil
 	},
@@ -33,6 +34,9 @@ var versionCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(versionCmd)
 
-	versionCmd.Flags().BoolVarP(&mizuVersionOptions.DebugInfo, "debug", "d", false, "Provide all information about version")
+	defaultVersionConfig := configStructs.VersionConfig{}
+	defaults.Set(&defaultVersionConfig)
+
+	versionCmd.Flags().BoolP(configStructs.DebugInfoVersionName, "d", defaultVersionConfig.DebugInfo, "Provide all information about version")
 
 }

cli/cmd/view.go

@@ -1,28 +1,19 @@
 package cmd
 
 import (
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/telemetry"
 )
 
-type MizuViewOptions struct {
-	GuiPort        uint16
-	KubeConfigPath string
-}
-
-var mizuViewOptions = &MizuViewOptions{}
-
 var viewCmd = &cobra.Command{
 	Use:   "view",
 	Short: "Open GUI in browser",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go mizu.ReportRun("view", mizuViewOptions)
-		if isCompatible, err := mizu.CheckVersionCompatibility(mizuViewOptions.GuiPort); err != nil {
-			return err
-		} else if !isCompatible {
-			return nil
-		}
-		runMizuView(mizuViewOptions)
+		go telemetry.ReportRun("view", config.Config.View)
+		runMizuView()
 		return nil
 	},
 }
@@ -30,6 +21,9 @@ var viewCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(viewCmd)
 
-	viewCmd.Flags().Uint16VarP(&mizuViewOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
-	viewCmd.Flags().StringVarP(&mizuViewOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
+	defaultViewConfig := configStructs.ViewConfig{}
+	defaults.Set(&defaultViewConfig)
+
+	viewCmd.Flags().Uint16P(configStructs.GuiPortViewName, "p", defaultViewConfig.GuiPort, "Provide a custom port for the web interface webserver")
+	viewCmd.Flags().StringP(configStructs.KubeConfigPathViewName, "k", defaultViewConfig.KubeConfigPath, "Path to kube-config file")
 }

cli/cmd/viewRunner.go

@@ -3,49 +3,55 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/cli/uiUtils"
-	"k8s.io/client-go/tools/clientcmd"
+	"github.com/up9inc/mizu/cli/mizu/version"
 	"net/http"
 )
 
-func runMizuView(mizuViewOptions *MizuViewOptions) {
-	kubernetesProvider, err := kubernetes.NewProvider(mizuViewOptions.KubeConfigPath)
+func runMizuView() {
+	kubernetesProvider, err := kubernetes.NewProvider(config.Config.View.KubeConfigPath)
 	if err != nil {
-		if clientcmd.IsEmptyConfig(err) {
-			mizu.Log.Infof("Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'")
-			return
-		}
-		if clientcmd.IsConfigurationInvalid(err) {
-			mizu.Log.Infof(uiUtils.Red, "Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'")
-			return
-		}
+		logger.Log.Error(err)
+		return
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	exists, err := kubernetesProvider.DoesServicesExist(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName)
+	exists, err := kubernetesProvider.DoesServicesExist(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
 	if err != nil {
-		panic(err)
+		logger.Log.Errorf("Failed to found mizu service %v", err)
+		cancel()
+		return
 	}
 	if !exists {
-		mizu.Log.Infof("The %s service not found", mizu.ApiServerPodName)
+		logger.Log.Infof("%s service not found, you should run `mizu tap` command first", mizu.ApiServerPodName)
+		cancel()
 		return
 	}
 
-	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizuViewOptions.GuiPort)
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.View.GuiPort)
 	_, err = http.Get(fmt.Sprintf("http://%s/", mizuProxiedUrl))
 	if err == nil {
-		mizu.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, mizuViewOptions.GuiPort)
+		logger.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, config.Config.View.GuiPort)
 		return
 	}
-	mizu.Log.Infof("Found service %s, creating k8s proxy", mizu.ApiServerPodName)
+	logger.Log.Debugf("Found service %s, creating k8s proxy", mizu.ApiServerPodName)
 
-	mizu.Log.Infof("Mizu is available at  http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizuViewOptions.GuiPort))
-	err = kubernetes.StartProxy(kubernetesProvider, mizuViewOptions.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
-	if err != nil {
-		mizu.Log.Infof("Error occured while running k8s proxy %v", err)
+	go startProxyReportErrorIfAny(kubernetesProvider, cancel)
+
+	logger.Log.Infof("Mizu is available at  http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.View.GuiPort))
+	if isCompatible, err := version.CheckVersionCompatibility(config.Config.View.GuiPort); err != nil {
+		logger.Log.Errorf("Failed to check versions compatibility %v", err)
+		cancel()
+		return
+	} else if !isCompatible {
+		cancel()
+		return
 	}
+
+	waitForFinish(ctx, cancel)
 }

cli/config/config.go

@@ -0,0 +1,337 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu"
+	"io/ioutil"
+	"os"
+	"path"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/creasty/defaults"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"gopkg.in/yaml.v3"
+)
+
+const (
+	Separator      = "="
+	SetCommandName = "set"
+	FieldNameTag   = "yaml"
+	ReadonlyTag    = "readonly"
+)
+
+var (
+	Config = ConfigStruct{}
+	cmdName string
+)
+
+func (config *ConfigStruct) Validate() error {
+	if config.IsNsRestrictedMode() {
+		if config.Tap.AllNamespaces || len(config.Tap.Namespaces) != 1 || config.Tap.Namespaces[0] != config.MizuResourcesNamespace {
+			return fmt.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+
+				"You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, MizuResourcesNamespaceConfigName)
+		}
+	}
+
+	return nil
+}
+
+func InitConfig(cmd *cobra.Command) error {
+	cmdName = cmd.Name()
+
+	if err := defaults.Set(&Config); err != nil {
+		return err
+	}
+
+	if err := mergeConfigFile(); err != nil {
+		return fmt.Errorf("invalid config %w\n"+
+			"you can regenerate the file using `mizu config -r` or just remove it %v", err, GetConfigFilePath())
+	}
+
+	cmd.Flags().Visit(initFlag)
+
+	finalConfigPrettified, _ := uiUtils.PrettyJson(Config)
+	logger.Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified)
+
+	return nil
+}
+
+func GetConfigWithDefaults() (string, error) {
+	defaultConf := ConfigStruct{}
+	if err := defaults.Set(&defaultConf); err != nil {
+		return "", err
+	}
+
+	configElem := reflect.ValueOf(&defaultConf).Elem()
+	setZeroForReadonlyFields(configElem)
+
+	return uiUtils.PrettyYaml(defaultConf)
+}
+
+func GetConfigFilePath() string {
+	return path.Join(mizu.GetMizuFolderPath(), "config.yaml")
+}
+
+func mergeConfigFile() error {
+	reader, openErr := os.Open(GetConfigFilePath())
+	if openErr != nil {
+		return nil
+	}
+
+	buf, readErr := ioutil.ReadAll(reader)
+	if readErr != nil {
+		return readErr
+	}
+
+	if err := yaml.Unmarshal(buf, &Config); err != nil {
+		return err
+	}
+	logger.Log.Debugf("Found config file, merged to default options")
+
+	return nil
+}
+
+func initFlag(f *pflag.Flag) {
+	configElemValue := reflect.ValueOf(&Config).Elem()
+
+	flagPath := []string {cmdName, f.Name}
+
+	sliceValue, isSliceValue := f.Value.(pflag.SliceValue)
+	if !isSliceValue {
+		if err := mergeFlagValue(configElemValue, flagPath, strings.Join(flagPath, "."), f.Value.String()); err != nil {
+			logger.Log.Warningf(uiUtils.Warning, err)
+		}
+		return
+	}
+
+	if f.Name == SetCommandName {
+		if err := mergeSetFlag(configElemValue, sliceValue.GetSlice()); err != nil {
+			logger.Log.Warningf(uiUtils.Warning, err)
+		}
+		return
+	}
+
+	if err := mergeFlagValues(configElemValue, flagPath, strings.Join(flagPath, "."), sliceValue.GetSlice()); err != nil {
+		logger.Log.Warningf(uiUtils.Warning, err)
+	}
+}
+
+func mergeSetFlag(configElemValue reflect.Value, setValues []string) error {
+	var setErrors []string
+	setMap := map[string][]string{}
+
+	for _, setValue := range setValues {
+		if !strings.Contains(setValue, Separator) {
+			setErrors = append(setErrors, fmt.Sprintf("Ignoring set argument %s (set argument format: <flag name>=<flag value>)", setValue))
+			continue
+		}
+
+		split := strings.SplitN(setValue, Separator, 2)
+		argumentKey, argumentValue := split[0], split[1]
+
+		setMap[argumentKey] = append(setMap[argumentKey], argumentValue)
+	}
+
+	for argumentKey, argumentValues := range setMap {
+		flagPath := strings.Split(argumentKey, ".")
+
+		if len(argumentValues) > 1 {
+			if err := mergeFlagValues(configElemValue, flagPath, argumentKey, argumentValues); err != nil {
+				setErrors = append(setErrors, fmt.Sprintf("%v", err))
+			}
+		} else {
+			if err := mergeFlagValue(configElemValue, flagPath, argumentKey, argumentValues[0]); err != nil {
+				setErrors = append(setErrors, fmt.Sprintf("%v", err))
+			}
+		}
+	}
+
+	if len(setErrors) > 0 {
+		return fmt.Errorf(strings.Join(setErrors, "\n"))
+	}
+
+	return nil
+}
+
+func mergeFlagValue(configElemValue reflect.Value, flagPath []string, fullFlagName string, flagValue string) error {
+	mergeFunction := func(flagName string, currentFieldStruct reflect.StructField, currentFieldElemValue reflect.Value, currentElemValue reflect.Value) error {
+		currentFieldKind := currentFieldStruct.Type.Kind()
+
+		if currentFieldKind == reflect.Slice {
+			return mergeFlagValues(currentElemValue, []string{flagName}, fullFlagName, []string{flagValue})
+		}
+
+		parsedValue, err := getParsedValue(currentFieldKind, flagValue)
+		if err != nil {
+			return fmt.Errorf("invalid value %s for flag name %s, expected %s", flagValue, flagName, currentFieldKind)
+		}
+
+		currentFieldElemValue.Set(parsedValue)
+		return nil
+	}
+
+	return mergeFlag(configElemValue, flagPath, fullFlagName, mergeFunction)
+}
+
+func mergeFlagValues(configElemValue reflect.Value, flagPath []string, fullFlagName string, flagValues []string) error {
+	mergeFunction := func(flagName string, currentFieldStruct reflect.StructField, currentFieldElemValue reflect.Value, currentElemValue reflect.Value) error {
+		currentFieldKind := currentFieldStruct.Type.Kind()
+
+		if currentFieldKind != reflect.Slice {
+			return fmt.Errorf("invalid values %s for flag name %s, expected %s", strings.Join(flagValues, ","), flagName, currentFieldKind)
+		}
+
+		flagValueKind := currentFieldStruct.Type.Elem().Kind()
+
+		parsedValues := reflect.MakeSlice(reflect.SliceOf(currentFieldStruct.Type.Elem()), 0, 0)
+		for _, flagValue := range flagValues {
+			parsedValue, err := getParsedValue(flagValueKind, flagValue)
+			if err != nil {
+				return fmt.Errorf("invalid value %s for flag name %s, expected %s", flagValue, flagName, flagValueKind)
+			}
+
+			parsedValues = reflect.Append(parsedValues, parsedValue)
+		}
+
+		currentFieldElemValue.Set(parsedValues)
+		return nil
+	}
+
+	return mergeFlag(configElemValue, flagPath, fullFlagName, mergeFunction)
+}
+
+func mergeFlag(currentElemValue reflect.Value, currentFlagPath []string, fullFlagName string, mergeFunction func(flagName string, currentFieldStruct reflect.StructField, currentFieldElemValue reflect.Value, currentElemValue reflect.Value) error) error {
+	if len(currentFlagPath) == 0 {
+		return fmt.Errorf("flag \"%s\" not found", fullFlagName)
+	}
+
+	for i := 0; i < currentElemValue.NumField(); i++ {
+		currentFieldStruct := currentElemValue.Type().Field(i)
+		currentFieldElemValue := currentElemValue.FieldByName(currentFieldStruct.Name)
+
+		if currentFieldStruct.Type.Kind() == reflect.Struct && getFieldNameByTag(currentFieldStruct) == currentFlagPath[0] {
+			return mergeFlag(currentFieldElemValue, currentFlagPath[1:], fullFlagName, mergeFunction)
+		}
+
+		if len(currentFlagPath) > 1 || getFieldNameByTag(currentFieldStruct) != currentFlagPath[0] {
+			continue
+		}
+
+		return mergeFunction(currentFlagPath[0], currentFieldStruct, currentFieldElemValue, currentElemValue)
+	}
+
+	return fmt.Errorf("flag \"%s\" not found", fullFlagName)
+}
+
+func getFieldNameByTag(field reflect.StructField) string {
+	return strings.Split(field.Tag.Get(FieldNameTag), ",")[0]
+}
+
+func getParsedValue(kind reflect.Kind, value string) (reflect.Value, error) {
+	switch kind {
+	case reflect.String:
+		return reflect.ValueOf(value), nil
+	case reflect.Bool:
+		boolArgumentValue, err := strconv.ParseBool(value)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(boolArgumentValue), nil
+	case reflect.Int:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(int(intArgumentValue)), nil
+	case reflect.Int8:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 8)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(int8(intArgumentValue)), nil
+	case reflect.Int16:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 16)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(int16(intArgumentValue)), nil
+	case reflect.Int32:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 32)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(int32(intArgumentValue)), nil
+	case reflect.Int64:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(intArgumentValue), nil
+	case reflect.Uint:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uint(uintArgumentValue)), nil
+	case reflect.Uint8:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 8)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uint8(uintArgumentValue)), nil
+	case reflect.Uint16:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 16)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uint16(uintArgumentValue)), nil
+	case reflect.Uint32:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 32)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uint32(uintArgumentValue)), nil
+	case reflect.Uint64:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uintArgumentValue), nil
+	}
+
+	return reflect.ValueOf(nil), errors.New("value to parse does not match type")
+}
+
+func setZeroForReadonlyFields(currentElem reflect.Value) {
+	for i := 0; i < currentElem.NumField(); i++ {
+		currentField := currentElem.Type().Field(i)
+		currentFieldByName := currentElem.FieldByName(currentField.Name)
+
+		if currentField.Type.Kind() == reflect.Struct {
+			setZeroForReadonlyFields(currentFieldByName)
+			continue
+		}
+
+		if _, ok := currentField.Tag.Lookup(ReadonlyTag); ok {
+			currentFieldByName.Set(reflect.Zero(currentField.Type))
+		}
+	}
+}

cli/config/configStruct.go

@@ -0,0 +1,31 @@
+package config
+
+import (
+	"fmt"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/mizu"
+)
+
+const (
+	MizuResourcesNamespaceConfigName = "mizu-resources-namespace"
+)
+
+type ConfigStruct struct {
+	Tap                    configStructs.TapConfig     `yaml:"tap"`
+	Fetch                  configStructs.FetchConfig   `yaml:"fetch"`
+	Version                configStructs.VersionConfig `yaml:"version"`
+	View                   configStructs.ViewConfig    `yaml:"view"`
+	AgentImage             string                      `yaml:"agent-image,omitempty" readonly:""`
+	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
+	Telemetry              bool                        `yaml:"telemetry" default:"true"`
+	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`
+	KubeConfigPath         string                      `yaml:"kube-config-path" default:""`
+}
+
+func (config *ConfigStruct) SetDefaults() {
+	config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer)
+}
+
+func (config *ConfigStruct) IsNsRestrictedMode() bool {
+	return config.MizuResourcesNamespace != "mizu" // Notice "mizu" string must match the default MizuResourcesNamespace
+}

cli/config/configStructs/fetchConfig.go

@@ -0,0 +1,15 @@
+package configStructs
+
+const (
+	DirectoryFetchName     = "directory"
+	FromTimestampFetchName = "from"
+	ToTimestampFetchName   = "to"
+	GuiPortFetchName       = "gui-port"
+)
+
+type FetchConfig struct {
+	Directory     string `yaml:"directory" default:"."`
+	FromTimestamp int    `yaml:"from" default:"0"`
+	ToTimestamp   int    `yaml:"to" default:"0"`
+	GuiPort       uint16 `yaml:"gui-port" default:"8899"`
+}

cli/config/configStructs/tapConfig.go

@@ -0,0 +1,87 @@
+package configStructs
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/up9inc/mizu/shared/units"
+)
+
+const (
+	GuiPortTapName                = "gui-port"
+	NamespacesTapName             = "namespaces"
+	AnalysisTapName               = "analysis"
+	AllNamespacesTapName          = "all-namespaces"
+	PlainTextFilterRegexesTapName = "regex-masking"
+	DisableRedactionTapName       = "no-redact"
+	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
+	DirectionTapName              = "direction"
+	DryRunTapName                 = "dry-run"
+	EnforcePolicyFile             = "test-rules"
+)
+
+type TapConfig struct {
+	AnalysisDestination          string    `yaml:"dest" default:"up9.app"`
+	SleepIntervalSec             int       `yaml:"upload-interval" default:"10"`
+	PodRegexStr                  string    `yaml:"regex" default:".*"`
+	GuiPort                      uint16    `yaml:"gui-port" default:"8899"`
+	Namespaces                   []string  `yaml:"namespaces"`
+	Analysis                     bool      `yaml:"analysis" default:"false"`
+	AllNamespaces                bool      `yaml:"all-namespaces" default:"false"`
+	PlainTextFilterRegexes       []string  `yaml:"regex-masking"`
+	HealthChecksUserAgentHeaders []string  `yaml:"ignored-user-agents"`
+	DisableRedaction             bool      `yaml:"no-redact" default:"false"`
+	HumanMaxEntriesDBSize        string    `yaml:"max-entries-db-size" default:"200MB"`
+	Direction                    string    `yaml:"direction" default:"in"`
+	DryRun                       bool      `yaml:"dry-run" default:"false"`
+	EnforcePolicyFile            string    `yaml:"test-rules"`
+	ApiServerResources           Resources `yaml:"api-server-resources"`
+	TapperResources              Resources `yaml:"tapper-resources"`
+}
+
+type Resources struct {
+	CpuLimit       string `yaml:"cpu-limit" default:"750m"`
+	MemoryLimit    string `yaml:"memory-limit" default:"1Gi"`
+	CpuRequests    string `yaml:"cpu-requests" default:"50m"`
+	MemoryRequests string `yaml:"memory-requests" default:"50Mi"`
+}
+
+func (config *TapConfig) PodRegex() *regexp.Regexp {
+	podRegex, _ := regexp.Compile(config.PodRegexStr)
+	return podRegex
+}
+
+func (config *TapConfig) TapOutgoing() bool {
+	directionLowerCase := strings.ToLower(config.Direction)
+	if directionLowerCase == "any" {
+		return true
+	}
+
+	return false
+}
+
+func (config *TapConfig) MaxEntriesDBSizeBytes() int64 {
+	maxEntriesDBSizeBytes, _ := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
+	return maxEntriesDBSizeBytes
+}
+
+func (config *TapConfig) Validate() error {
+	_, compileErr := regexp.Compile(config.PodRegexStr)
+	if compileErr != nil {
+		return errors.New(fmt.Sprintf("%s is not a valid regex %s", config.PodRegexStr, compileErr))
+	}
+
+	_, parseHumanDataSizeErr := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
+	if parseHumanDataSizeErr != nil {
+		return errors.New(fmt.Sprintf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize))
+	}
+
+	directionLowerCase := strings.ToLower(config.Direction)
+	if directionLowerCase != "any" && directionLowerCase != "in" {
+		return errors.New(fmt.Sprintf("%s is not a valid value for flag --%s. Acceptable values are in/any.", config.Direction, DirectionTapName))
+	}
+
+	return nil
+}

cli/config/configStructs/versionConfig.go

@@ -0,0 +1,9 @@
+package configStructs
+
+const (
+	DebugInfoVersionName = "debug"
+)
+
+type VersionConfig struct {
+	DebugInfo bool `yaml:"debug" default:"false"`
+}

cli/config/configStructs/viewConfig.go

@@ -0,0 +1,11 @@
+package configStructs
+
+const (
+	GuiPortViewName        = "gui-port"
+	KubeConfigPathViewName = "kube-config"
+)
+
+type ViewConfig struct {
+	GuiPort        uint16 `yaml:"gui-port" default:"8899"`
+	KubeConfigPath string `yaml:"kube-config"`
+}

cli/config/config_internal_test.go

@@ -0,0 +1,340 @@
+package config
+
+import (
+	"reflect"
+	"testing"
+)
+
+type ConfigMock struct {
+	SectionMock      SectionMock `yaml:"section"`
+	Test             string      `yaml:"test"`
+	StringField      string      `yaml:"string-field"`
+	IntField         int         `yaml:"int-field"`
+	BoolField        bool        `yaml:"bool-field"`
+	UintField        uint        `yaml:"uint-field"`
+	StringSliceField []string    `yaml:"string-slice-field"`
+	IntSliceField    []int       `yaml:"int-slice-field"`
+	BoolSliceField   []bool      `yaml:"bool-slice-field"`
+	UintSliceField   []uint      `yaml:"uint-slice-field"`
+}
+
+type SectionMock struct {
+	Test string `yaml:"test"`
+}
+
+func TestMergeSetFlagNoSeparator(t *testing.T) {
+	tests := [][]string{{""}, {"t"}, {"", "t"}, {"t", "test", "test:true"}, {"test", "test:true", "testing!", "true"}}
+
+	for _, setValues := range tests {
+		configMock := ConfigMock{}
+		configMockElemValue := reflect.ValueOf(&configMock).Elem()
+
+		err := mergeSetFlag(configMockElemValue, setValues)
+
+		if err == nil {
+			t.Errorf("unexpected unhandled error - setValues: %v", setValues)
+			continue
+		}
+
+		for i := 0; i < configMockElemValue.NumField(); i++ {
+			currentField := configMockElemValue.Type().Field(i)
+			currentFieldByName := configMockElemValue.FieldByName(currentField.Name)
+
+			if !currentFieldByName.IsZero() {
+				t.Errorf("unexpected value with not default value - setValues: %v", setValues)
+			}
+		}
+	}
+}
+
+func TestMergeSetFlagInvalidFlagName(t *testing.T) {
+	tests := [][]string{{"invalid_flag=true"}, {"section.invalid_flag=test"}, {"section=test"}, {"=true"}, {"invalid_flag=true", "config.invalid_flag=test", "section=test", "=true"}}
+
+	for _, setValues := range tests {
+		configMock := ConfigMock{}
+		configMockElemValue := reflect.ValueOf(&configMock).Elem()
+
+		err := mergeSetFlag(configMockElemValue, setValues)
+
+		if err == nil {
+			t.Errorf("unexpected unhandled error - setValues: %v", setValues)
+			continue
+		}
+
+		for i := 0; i < configMockElemValue.NumField(); i++ {
+			currentField := configMockElemValue.Type().Field(i)
+			currentFieldByName := configMockElemValue.FieldByName(currentField.Name)
+
+			if !currentFieldByName.IsZero() {
+				t.Errorf("unexpected case - setValues: %v", setValues)
+			}
+		}
+	}
+}
+
+func TestMergeSetFlagInvalidFlagValue(t *testing.T) {
+	tests := [][]string{{"int-field=true"}, {"bool-field:5"}, {"uint-field=-1"}, {"int-slice-field=true"}, {"bool-slice-field=5"}, {"uint-slice-field=-1"}, {"int-field=6", "int-field=66"}}
+
+	for _, setValues := range tests {
+		configMock := ConfigMock{}
+		configMockElemValue := reflect.ValueOf(&configMock).Elem()
+
+		err := mergeSetFlag(configMockElemValue, setValues)
+
+		if err == nil {
+			t.Errorf("unexpected unhandled error - setValues: %v", setValues)
+			continue
+		}
+
+		for i := 0; i < configMockElemValue.NumField(); i++ {
+			currentField := configMockElemValue.Type().Field(i)
+			currentFieldByName := configMockElemValue.FieldByName(currentField.Name)
+
+			if !currentFieldByName.IsZero() {
+				t.Errorf("unexpected case - setValues: %v", setValues)
+			}
+		}
+	}
+}
+
+func TestMergeSetFlagNotSliceValues(t *testing.T) {
+	tests := [][]struct {
+		SetValue   string
+		FieldName  string
+		FieldValue interface{}
+	}{
+		{{SetValue: "string-field=test", FieldName: "StringField", FieldValue: "test"}},
+		{{SetValue: "int-field=6", FieldName: "IntField", FieldValue: 6}},
+		{{SetValue: "bool-field=true", FieldName: "BoolField", FieldValue: true}},
+		{{SetValue: "uint-field=6", FieldName: "UintField", FieldValue: uint(6)}},
+		{
+			{SetValue: "string-field=test", FieldName: "StringField", FieldValue: "test"},
+			{SetValue: "int-field=6", FieldName: "IntField", FieldValue: 6},
+			{SetValue: "bool-field=true", FieldName: "BoolField", FieldValue: true},
+			{SetValue: "uint-field=6", FieldName: "UintField", FieldValue: uint(6)},
+		},
+	}
+
+	for _, test := range tests {
+		configMock := ConfigMock{}
+		configMockElemValue := reflect.ValueOf(&configMock).Elem()
+
+		var setValues []string
+		for _, setValueInfo := range test {
+			setValues = append(setValues, setValueInfo.SetValue)
+		}
+
+		err := mergeSetFlag(configMockElemValue, setValues)
+
+		if err != nil {
+			t.Errorf("unexpected error result - err: %v", err)
+			continue
+		}
+
+		for _, setValueInfo := range test {
+			fieldValue := configMockElemValue.FieldByName(setValueInfo.FieldName).Interface()
+			if fieldValue != setValueInfo.FieldValue {
+				t.Errorf("unexpected result - expected: %v, actual: %v", setValueInfo.FieldValue, fieldValue)
+			}
+		}
+	}
+}
+
+func TestMergeSetFlagSliceValues(t *testing.T) {
+	tests := [][]struct {
+		SetValues  []string
+		FieldName  string
+		FieldValue interface{}
+	}{
+		{{SetValues: []string{"string-slice-field=test"}, FieldName: "StringSliceField", FieldValue: []string{"test"}}},
+		{{SetValues: []string{"int-slice-field=6"}, FieldName: "IntSliceField", FieldValue: []int{6}}},
+		{{SetValues: []string{"bool-slice-field=true"}, FieldName: "BoolSliceField", FieldValue: []bool{true}}},
+		{{SetValues: []string{"uint-slice-field=6"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6)}}},
+		{
+			{SetValues: []string{"string-slice-field=test"}, FieldName: "StringSliceField", FieldValue: []string{"test"}},
+			{SetValues: []string{"int-slice-field=6"}, FieldName: "IntSliceField", FieldValue: []int{6}},
+			{SetValues: []string{"bool-slice-field=true"}, FieldName: "BoolSliceField", FieldValue: []bool{true}},
+			{SetValues: []string{"uint-slice-field=6"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6)}},
+		},
+		{{SetValues: []string{"string-slice-field=test", "string-slice-field=test2"}, FieldName: "StringSliceField", FieldValue: []string{"test", "test2"}}},
+		{{SetValues: []string{"int-slice-field=6", "int-slice-field=66"}, FieldName: "IntSliceField", FieldValue: []int{6, 66}}},
+		{{SetValues: []string{"bool-slice-field=true", "bool-slice-field=false"}, FieldName: "BoolSliceField", FieldValue: []bool{true, false}}},
+		{{SetValues: []string{"uint-slice-field=6", "uint-slice-field=66"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6), uint(66)}}},
+		{
+			{SetValues: []string{"string-slice-field=test", "string-slice-field=test2"}, FieldName: "StringSliceField", FieldValue: []string{"test", "test2"}},
+			{SetValues: []string{"int-slice-field=6", "int-slice-field=66"}, FieldName: "IntSliceField", FieldValue: []int{6, 66}},
+			{SetValues: []string{"bool-slice-field=true", "bool-slice-field=false"}, FieldName: "BoolSliceField", FieldValue: []bool{true, false}},
+			{SetValues: []string{"uint-slice-field=6", "uint-slice-field=66"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6), uint(66)}},
+		},
+	}
+
+	for _, test := range tests {
+		configMock := ConfigMock{}
+		configMockElemValue := reflect.ValueOf(&configMock).Elem()
+
+		var setValues []string
+		for _, setValueInfo := range test {
+			for _, setValue := range setValueInfo.SetValues {
+				setValues = append(setValues, setValue)
+			}
+		}
+
+		err := mergeSetFlag(configMockElemValue, setValues)
+
+		if err != nil {
+			t.Errorf("unexpected error result - err: %v", err)
+			continue
+		}
+
+		for _, setValueInfo := range test {
+			fieldValue := configMockElemValue.FieldByName(setValueInfo.FieldName).Interface()
+			if !reflect.DeepEqual(fieldValue, setValueInfo.FieldValue) {
+				t.Errorf("unexpected result - expected: %v, actual: %v", setValueInfo.FieldValue, fieldValue)
+			}
+		}
+	}
+}
+
+func TestMergeSetFlagMixValues(t *testing.T) {
+	tests := [][]struct {
+		SetValues  []string
+		FieldName  string
+		FieldValue interface{}
+	}{
+		{
+			{SetValues: []string{"string-slice-field=test"}, FieldName: "StringSliceField", FieldValue: []string{"test"}},
+			{SetValues: []string{"int-slice-field=6"}, FieldName: "IntSliceField", FieldValue: []int{6}},
+			{SetValues: []string{"bool-slice-field=true"}, FieldName: "BoolSliceField", FieldValue: []bool{true}},
+			{SetValues: []string{"uint-slice-field=6"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6)}},
+			{SetValues: []string{"string-field=test"}, FieldName: "StringField", FieldValue: "test"},
+			{SetValues: []string{"int-field=6"}, FieldName: "IntField", FieldValue: 6},
+			{SetValues: []string{"bool-field=true"}, FieldName: "BoolField", FieldValue: true},
+			{SetValues: []string{"uint-field=6"}, FieldName: "UintField", FieldValue: uint(6)},
+		},
+		{
+			{SetValues: []string{"string-slice-field=test", "string-slice-field=test2"}, FieldName: "StringSliceField", FieldValue: []string{"test", "test2"}},
+			{SetValues: []string{"int-slice-field=6", "int-slice-field=66"}, FieldName: "IntSliceField", FieldValue: []int{6, 66}},
+			{SetValues: []string{"bool-slice-field=true", "bool-slice-field=false"}, FieldName: "BoolSliceField", FieldValue: []bool{true, false}},
+			{SetValues: []string{"uint-slice-field=6", "uint-slice-field=66"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6), uint(66)}},
+			{SetValues: []string{"string-field=test"}, FieldName: "StringField", FieldValue: "test"},
+			{SetValues: []string{"int-field=6"}, FieldName: "IntField", FieldValue: 6},
+			{SetValues: []string{"bool-field=true"}, FieldName: "BoolField", FieldValue: true},
+			{SetValues: []string{"uint-field=6"}, FieldName: "UintField", FieldValue: uint(6)},
+		},
+	}
+
+	for _, test := range tests {
+		configMock := ConfigMock{}
+		configMockElemValue := reflect.ValueOf(&configMock).Elem()
+
+		var setValues []string
+		for _, setValueInfo := range test {
+			for _, setValue := range setValueInfo.SetValues {
+				setValues = append(setValues, setValue)
+			}
+		}
+
+		err := mergeSetFlag(configMockElemValue, setValues)
+
+		if err != nil {
+			t.Errorf("unexpected error result - err: %v", err)
+			continue
+		}
+
+		for _, setValueInfo := range test {
+			fieldValue := configMockElemValue.FieldByName(setValueInfo.FieldName).Interface()
+			if !reflect.DeepEqual(fieldValue, setValueInfo.FieldValue) {
+				t.Errorf("unexpected result - expected: %v, actual: %v", setValueInfo.FieldValue, fieldValue)
+			}
+		}
+	}
+}
+
+func TestGetParsedValueValidValue(t *testing.T) {
+	tests := []struct {
+		StringValue string
+		Kind        reflect.Kind
+		ActualValue interface{}
+	}{
+		{StringValue: "test", Kind: reflect.String, ActualValue: "test"},
+		{StringValue: "123", Kind: reflect.String, ActualValue: "123"},
+		{StringValue: "true", Kind: reflect.Bool, ActualValue: true},
+		{StringValue: "false", Kind: reflect.Bool, ActualValue: false},
+		{StringValue: "6", Kind: reflect.Int, ActualValue: 6},
+		{StringValue: "-6", Kind: reflect.Int, ActualValue: -6},
+		{StringValue: "6", Kind: reflect.Int8, ActualValue: int8(6)},
+		{StringValue: "-6", Kind: reflect.Int8, ActualValue: int8(-6)},
+		{StringValue: "6", Kind: reflect.Int16, ActualValue: int16(6)},
+		{StringValue: "-6", Kind: reflect.Int16, ActualValue: int16(-6)},
+		{StringValue: "6", Kind: reflect.Int32, ActualValue: int32(6)},
+		{StringValue: "-6", Kind: reflect.Int32, ActualValue: int32(-6)},
+		{StringValue: "6", Kind: reflect.Int64, ActualValue: int64(6)},
+		{StringValue: "-6", Kind: reflect.Int64, ActualValue: int64(-6)},
+		{StringValue: "6", Kind: reflect.Uint, ActualValue: uint(6)},
+		{StringValue: "66", Kind: reflect.Uint, ActualValue: uint(66)},
+		{StringValue: "6", Kind: reflect.Uint8, ActualValue: uint8(6)},
+		{StringValue: "66", Kind: reflect.Uint8, ActualValue: uint8(66)},
+		{StringValue: "6", Kind: reflect.Uint16, ActualValue: uint16(6)},
+		{StringValue: "66", Kind: reflect.Uint16, ActualValue: uint16(66)},
+		{StringValue: "6", Kind: reflect.Uint32, ActualValue: uint32(6)},
+		{StringValue: "66", Kind: reflect.Uint32, ActualValue: uint32(66)},
+		{StringValue: "6", Kind: reflect.Uint64, ActualValue: uint64(6)},
+		{StringValue: "66", Kind: reflect.Uint64, ActualValue: uint64(66)},
+	}
+
+	for _, test := range tests {
+		parsedValue, err := getParsedValue(test.Kind, test.StringValue)
+
+		if err != nil {
+			t.Errorf("unexpected error result - err: %v", err)
+			continue
+		}
+
+		if parsedValue.Interface() != test.ActualValue {
+			t.Errorf("unexpected result - expected: %v, actual: %v", test.ActualValue, parsedValue)
+		}
+	}
+}
+
+func TestGetParsedValueInvalidValue(t *testing.T) {
+	tests := []struct {
+		StringValue string
+		Kind        reflect.Kind
+	}{
+		{StringValue: "test", Kind: reflect.Bool},
+		{StringValue: "123", Kind: reflect.Bool},
+		{StringValue: "test", Kind: reflect.Int},
+		{StringValue: "true", Kind: reflect.Int},
+		{StringValue: "test", Kind: reflect.Int8},
+		{StringValue: "true", Kind: reflect.Int8},
+		{StringValue: "test", Kind: reflect.Int16},
+		{StringValue: "true", Kind: reflect.Int16},
+		{StringValue: "test", Kind: reflect.Int32},
+		{StringValue: "true", Kind: reflect.Int32},
+		{StringValue: "test", Kind: reflect.Int64},
+		{StringValue: "true", Kind: reflect.Int64},
+		{StringValue: "test", Kind: reflect.Uint},
+		{StringValue: "-6", Kind: reflect.Uint},
+		{StringValue: "test", Kind: reflect.Uint8},
+		{StringValue: "-6", Kind: reflect.Uint8},
+		{StringValue: "test", Kind: reflect.Uint16},
+		{StringValue: "-6", Kind: reflect.Uint16},
+		{StringValue: "test", Kind: reflect.Uint32},
+		{StringValue: "-6", Kind: reflect.Uint32},
+		{StringValue: "test", Kind: reflect.Uint64},
+		{StringValue: "-6", Kind: reflect.Uint64},
+	}
+
+	for _, test := range tests {
+		parsedValue, err := getParsedValue(test.Kind, test.StringValue)
+
+		if err == nil {
+			t.Errorf("unexpected unhandled error - stringValue: %v, Kind: %v", test.StringValue, test.Kind)
+			continue
+		}
+
+		if parsedValue != reflect.ValueOf(nil) {
+			t.Errorf("unexpected parsed value - parsedValue: %v", parsedValue)
+		}
+	}
+}

cli/config/config_test.go

@@ -0,0 +1,39 @@
+package config_test
+
+import (
+	"github.com/up9inc/mizu/cli/config"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestConfigWriteIgnoresReadonlyFields(t *testing.T) {
+	var readonlyFields []string
+
+	configElem := reflect.ValueOf(&config.ConfigStruct{}).Elem()
+	getFieldsWithReadonlyTag(configElem, &readonlyFields)
+
+	configWithDefaults, _ := config.GetConfigWithDefaults()
+	for _, readonlyField := range readonlyFields {
+		if strings.Contains(configWithDefaults, readonlyField) {
+			t.Errorf("unexpected result - readonly field: %v, config: %v", readonlyField, configWithDefaults)
+		}
+	}
+}
+
+func getFieldsWithReadonlyTag(currentElem reflect.Value, readonlyFields *[]string) {
+	for i := 0; i < currentElem.NumField(); i++ {
+		currentField := currentElem.Type().Field(i)
+		currentFieldByName := currentElem.FieldByName(currentField.Name)
+
+		if currentField.Type.Kind() == reflect.Struct {
+			getFieldsWithReadonlyTag(currentFieldByName, readonlyFields)
+			continue
+		}
+
+		if _, ok := currentField.Tag.Lookup(config.ReadonlyTag); ok {
+			fieldNameByTag := strings.Split(currentField.Tag.Get(config.FieldNameTag), ",")[0]
+			*readonlyFields = append(*readonlyFields, fieldNameByTag)
+		}
+	}
+}

cli/errormessage/errormessage.go

@@ -0,0 +1,36 @@
+package errormessage
+
+import (
+	"errors"
+	"fmt"
+	"github.com/up9inc/mizu/cli/config"
+	regexpsyntax "regexp/syntax"
+
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+)
+
+// formatError wraps error with a detailed message that is meant for the user.
+// While the errors are meant to be displayed, they are not meant to be exported as classes outsite of CLI.
+func FormatError(err error) error {
+	var errorNew error
+	if k8serrors.IsForbidden(err) {
+		errorNew = fmt.Errorf("insufficient permissions: %w. "+
+			"supply the required permission or control Mizu's access to namespaces by setting %s "+
+			"in the config file or setting the tapped namespace with --%s %s=<NAMEPSACE>",
+			err,
+			config.MizuResourcesNamespaceConfigName,
+			config.SetCommandName,
+			config.MizuResourcesNamespaceConfigName)
+	} else if syntaxError, isSyntaxError := asRegexSyntaxError(err); isSyntaxError {
+		errorNew = fmt.Errorf("regex %s is invalid: %w", syntaxError.Expr, err)
+	} else {
+		errorNew = err
+	}
+
+	return errorNew
+}
+
+func asRegexSyntaxError(err error) (*regexpsyntax.Error, bool) {
+	var syntaxError *regexpsyntax.Error
+	return syntaxError, errors.As(err, &syntaxError)
+}

cli/go.mod

@@ -3,10 +3,12 @@ module github.com/up9inc/mizu/cli
 go 1.16
 
 require (
+	github.com/creasty/defaults v1.5.1
 	github.com/google/go-github/v37 v37.0.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.1.3
+	github.com/spf13/pflag v1.0.5
 	github.com/up9inc/mizu/shared v0.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.21.2

cli/go.sum

@@ -82,6 +82,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creasty/defaults v1.5.1 h1:j8WexcS3d/t4ZmllX4GEkl4wIB/trOr035ajcLHCISM=
+github.com/creasty/defaults v1.5.1/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -410,6 +412,7 @@ github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=

cli/kubernetes/provider.go

@@ -1,26 +1,28 @@
 package kubernetes
 
 import (
+	"bytes"
 	_ "bytes"
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/cli/mizu"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/client-go/tools/cache"
-	"k8s.io/client-go/util/homedir"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strconv"
 
+	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/shared"
+	"io"
 	core "k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	resource "k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/watch"
 	applyconfapp "k8s.io/client-go/applyconfigurations/apps/v1"
@@ -32,9 +34,11 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
 	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/clientcmd"
 	_ "k8s.io/client-go/tools/portforward"
 	watchtools "k8s.io/client-go/tools/watch"
+	"k8s.io/client-go/util/homedir"
 )
 
 type Provider struct {
@@ -52,7 +56,12 @@ func NewProvider(kubeConfigPath string) (*Provider, error) {
 	kubernetesConfig := loadKubernetesConfiguration(kubeConfigPath)
 	restClientConfig, err := kubernetesConfig.ClientConfig()
 	if err != nil {
-		return nil, err
+		if clientcmd.IsEmptyConfig(err) {
+			return nil, fmt.Errorf("Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'\n")
+		}
+		if clientcmd.IsConfigurationInvalid(err) {
+			return nil, fmt.Errorf("Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'\n")
+		}
 	}
 	clientSet := getClientSet(restClientConfig)
 
@@ -125,42 +134,67 @@ func (provider *Provider) CreateNamespace(ctx context.Context, name string) (*co
 	return provider.clientSet.CoreV1().Namespaces().Create(ctx, namespaceSpec, metav1.CreateOptions{})
 }
 
-func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, namespace string, podName string, podImage string, serviceAccountName string, mizuApiFilteringOptions *shared.TrafficFilteringOptions, maxEntriesDBSizeBytes int64) (*core.Pod, error) {
-	marshaledFilteringOptions, err := json.Marshal(mizuApiFilteringOptions)
+type ApiServerOptions struct {
+	Namespace               string
+	PodName                 string
+	PodImage                string
+	ServiceAccountName      string
+	IsNamespaceRestricted   bool
+	MizuApiFilteringOptions *shared.TrafficFilteringOptions
+	MaxEntriesDBSizeBytes   int64
+}
+
+func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiServerOptions, resources configStructs.Resources) (*core.Pod, error) {
+	marshaledFilteringOptions, err := json.Marshal(opts.MizuApiFilteringOptions)
 	if err != nil {
 		return nil, err
 	}
+	configMapVolumeName := &core.ConfigMapVolumeSource{}
+	configMapVolumeName.Name = mizu.ConfigMapName
+	configMapOptional := true
+	configMapVolumeName.Optional = &configMapOptional
 
-	cpuLimit, err := resource.ParseQuantity("750m")
+	cpuLimit, err := resource.ParseQuantity(resources.CpuLimit)
 	if err != nil {
-		return nil, errors.New(fmt.Sprintf("invalid cpu limit for %s container", podName))
+		return nil, errors.New(fmt.Sprintf("invalid cpu limit for %s container", opts.PodName))
 	}
-	memLimit, err := resource.ParseQuantity("512Mi")
+	memLimit, err := resource.ParseQuantity(resources.MemoryLimit)
 	if err != nil {
-		return nil, errors.New(fmt.Sprintf("invalid memory limit for %s container", podName))
+		return nil, errors.New(fmt.Sprintf("invalid memory limit for %s container", opts.PodName))
 	}
-	cpuRequests, err := resource.ParseQuantity("50m")
+	cpuRequests, err := resource.ParseQuantity(resources.CpuRequests)
 	if err != nil {
-		return nil, errors.New(fmt.Sprintf("invalid cpu request for %s container", podName))
+		return nil, errors.New(fmt.Sprintf("invalid cpu request for %s container", opts.PodName))
 	}
-	memRequests, err := resource.ParseQuantity("50Mi")
+	memRequests, err := resource.ParseQuantity(resources.MemoryRequests)
 	if err != nil {
-		return nil, errors.New(fmt.Sprintf("invalid memory request for %s container", podName))
+		return nil, errors.New(fmt.Sprintf("invalid memory request for %s container", opts.PodName))
+	}
+
+	command := []string{"./mizuagent", "--api-server"}
+	if opts.IsNamespaceRestricted {
+		command = append(command, "--namespace", opts.Namespace)
 	}
 
 	pod := &core.Pod{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      podName,
-			Namespace: namespace,
-			Labels:    map[string]string{"app": podName},
+			Name:      opts.PodName,
+			Namespace: opts.Namespace,
+			Labels:    map[string]string{"app": opts.PodName},
 		},
 		Spec: core.PodSpec{
 			Containers: []core.Container{
 				{
-					Name:            podName,
-					Image:           podImage,
+					Name:            opts.PodName,
+					Image:           opts.PodImage,
 					ImagePullPolicy: core.PullAlways,
-					Command:         []string{"./mizuagent", "--api-server"},
+					VolumeMounts: []core.VolumeMount{
+						{
+							Name:      mizu.ConfigMapName,
+							MountPath: shared.RulePolicyPath,
+						},
+					},
+					Command: command,
 					Env: []core.EnvVar{
 						{
 							Name:  shared.HostModeEnvVar,
@@ -172,7 +206,7 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, namespace
 						},
 						{
 							Name:  shared.MaxEntriesDBSizeBytesEnvVar,
-							Value: strconv.FormatInt(maxEntriesDBSizeBytes, 10),
+							Value: strconv.FormatInt(opts.MaxEntriesDBSizeBytes, 10),
 						},
 					},
 					Resources: core.ResourceRequirements{
@@ -187,15 +221,23 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, namespace
 					},
 				},
 			},
+			Volumes: []core.Volume{
+				{
+					Name: mizu.ConfigMapName,
+					VolumeSource: core.VolumeSource{
+						ConfigMap: configMapVolumeName,
+					},
+				},
+			},
 			DNSPolicy:                     core.DNSClusterFirstWithHostNet,
 			TerminationGracePeriodSeconds: new(int64),
 		},
 	}
 	//define the service account only when it exists to prevent pod crash
-	if serviceAccountName != "" {
-		pod.Spec.ServiceAccountName = serviceAccountName
+	if opts.ServiceAccountName != "" {
+		pod.Spec.ServiceAccountName = opts.ServiceAccountName
 	}
-	return provider.clientSet.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
+	return provider.clientSet.CoreV1().Pods(opts.Namespace).Create(ctx, pod, metav1.CreateOptions{})
 }
 
 func (provider *Provider) CreateService(ctx context.Context, namespace string, serviceName string, appLabelValue string) (*core.Service, error) {
@@ -215,7 +257,55 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s
 
 func (provider *Provider) DoesServiceAccountExist(ctx context.Context, namespace string, serviceAccountName string) (bool, error) {
 	serviceAccount, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{})
+	return provider.doesResourceExist(serviceAccount, err)
+}
 
+func (provider *Provider) DoesConfigMapExist(ctx context.Context, namespace string, name string) (bool, error) {
+	resource, err := provider.clientSet.CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) DoesServicesExist(ctx context.Context, namespace string, name string) (bool, error) {
+	resource, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) DoesNamespaceExist(ctx context.Context, name string) (bool, error) {
+	resource, err := provider.clientSet.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) DoesClusterRoleExist(ctx context.Context, name string) (bool, error) {
+	resource, err := provider.clientSet.RbacV1().ClusterRoles().Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) DoesClusterRoleBindingExist(ctx context.Context, name string) (bool, error) {
+	resource, err := provider.clientSet.RbacV1().ClusterRoleBindings().Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) DoesRoleExist(ctx context.Context, namespace string, name string) (bool, error) {
+	resource, err := provider.clientSet.RbacV1().Roles(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) DoesRoleBindingExist(ctx context.Context, namespace string, name string) (bool, error) {
+	resource, err := provider.clientSet.RbacV1().RoleBindings(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) DoesPodExist(ctx context.Context, namespace string, name string) (bool, error) {
+	resource, err := provider.clientSet.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) DoesDaemonSetExist(ctx context.Context, namespace string, name string) (bool, error) {
+	resource, err := provider.clientSet.AppsV1().DaemonSets(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) doesResourceExist(resource interface{}, err error) (bool, error) {
 	var statusError *k8serrors.StatusError
 	if errors.As(err, &statusError) {
 		// expected behavior when resource does not exist
@@ -226,22 +316,7 @@ func (provider *Provider) DoesServiceAccountExist(ctx context.Context, namespace
 	if err != nil {
 		return false, err
 	}
-	return serviceAccount != nil, nil
-}
-
-func (provider *Provider) DoesServicesExist(ctx context.Context, namespace string, serviceName string) (bool, error) {
-	service, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, serviceName, metav1.GetOptions{})
-
-	var statusError *k8serrors.StatusError
-	if errors.As(err, &statusError) {
-		if statusError.ErrStatus.Reason == metav1.StatusReasonNotFound {
-			return false, nil
-		}
-	}
-	if err != nil {
-		return false, err
-	}
-	return service != nil, nil
+	return resource != nil, nil
 }
 
 func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, serviceAccountName string, clusterRoleName string, clusterRoleBindingName string, version string) error {
@@ -284,22 +359,76 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
 		},
 	}
 	_, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Create(ctx, serviceAccount, metav1.CreateOptions{})
-	if err != nil {
+	if err != nil && !k8serrors.IsAlreadyExists(err) {
 		return err
 	}
 	_, err = provider.clientSet.RbacV1().ClusterRoles().Create(ctx, clusterRole, metav1.CreateOptions{})
-	if err != nil {
+	if err != nil && !k8serrors.IsAlreadyExists(err) {
 		return err
 	}
 	_, err = provider.clientSet.RbacV1().ClusterRoleBindings().Create(ctx, clusterRoleBinding, metav1.CreateOptions{})
-	if err != nil {
+	if err != nil && !k8serrors.IsAlreadyExists(err) {
+		return err
+	}
+	return nil
+}
+
+func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, namespace string, serviceAccountName string, roleName string, roleBindingName string, version string) error {
+	serviceAccount := &core.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      serviceAccountName,
+			Namespace: namespace,
+			Labels:    map[string]string{"mizu-cli-version": version},
+		},
+	}
+	role := &rbac.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   roleName,
+			Labels: map[string]string{"mizu-cli-version": version},
+		},
+		Rules: []rbac.PolicyRule{
+			{
+				APIGroups: []string{"", "extensions", "apps"},
+				Resources: []string{"pods", "services", "endpoints"},
+				Verbs:     []string{"list", "get", "watch"},
+			},
+		},
+	}
+	roleBinding := &rbac.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   roleBindingName,
+			Labels: map[string]string{"mizu-cli-version": version},
+		},
+		RoleRef: rbac.RoleRef{
+			Name:     roleName,
+			Kind:     "Role",
+			APIGroup: "rbac.authorization.k8s.io",
+		},
+		Subjects: []rbac.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      serviceAccountName,
+				Namespace: namespace,
+			},
+		},
+	}
+	_, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Create(ctx, serviceAccount, metav1.CreateOptions{})
+	if err != nil && !k8serrors.IsAlreadyExists(err) {
+		return err
+	}
+	_, err = provider.clientSet.RbacV1().Roles(namespace).Create(ctx, role, metav1.CreateOptions{})
+	if err != nil && !k8serrors.IsAlreadyExists(err) {
+		return err
+	}
+	_, err = provider.clientSet.RbacV1().RoleBindings(namespace).Create(ctx, roleBinding, metav1.CreateOptions{})
+	if err != nil && !k8serrors.IsAlreadyExists(err) {
 		return err
 	}
 	return nil
 }
 
 func (provider *Provider) RemoveNamespace(ctx context.Context, name string) error {
-	if isFound, err := provider.CheckNamespaceExists(ctx, name); err != nil {
+	if isFound, err := provider.DoesNamespaceExist(ctx, name); err != nil {
 		return err
 	} else if !isFound {
 		return nil
@@ -321,7 +450,7 @@ func (provider *Provider) RemoveNonNamespacedResources(ctx context.Context, clus
 }
 
 func (provider *Provider) RemoveClusterRole(ctx context.Context, name string) error {
-	if isFound, err := provider.CheckClusterRoleExists(ctx, name); err != nil {
+	if isFound, err := provider.DoesClusterRoleExist(ctx, name); err != nil {
 		return err
 	} else if !isFound {
 		return nil
@@ -331,7 +460,7 @@ func (provider *Provider) RemoveClusterRole(ctx context.Context, name string) er
 }
 
 func (provider *Provider) RemoveClusterRoleBinding(ctx context.Context, name string) error {
-	if isFound, err := provider.CheckClusterRoleBindingExists(ctx, name); err != nil {
+	if isFound, err := provider.DoesClusterRoleBindingExist(ctx, name); err != nil {
 		return err
 	} else if !isFound {
 		return nil
@@ -340,8 +469,38 @@ func (provider *Provider) RemoveClusterRoleBinding(ctx context.Context, name str
 	return provider.clientSet.RbacV1().ClusterRoleBindings().Delete(ctx, name, metav1.DeleteOptions{})
 }
 
+func (provider *Provider) RemoveRoleBinding(ctx context.Context, namespace string, name string) error {
+	if isFound, err := provider.DoesRoleBindingExist(ctx, namespace, name); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.RbacV1().RoleBindings(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+}
+
+func (provider *Provider) RemoveRole(ctx context.Context, namespace string, name string) error {
+	if isFound, err := provider.DoesRoleExist(ctx, namespace, name); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.RbacV1().Roles(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+}
+
+func (provider *Provider) RemoveServicAccount(ctx context.Context, namespace string, name string) error {
+	if isFound, err := provider.DoesServiceAccountExist(ctx, namespace, name); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+}
+
 func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) error {
-	if isFound, err := provider.CheckPodExists(ctx, namespace, podName); err != nil {
+	if isFound, err := provider.DoesPodExist(ctx, namespace, podName); err != nil {
 		return err
 	} else if !isFound {
 		return nil
@@ -350,8 +509,18 @@ func (provider *Provider) RemovePod(ctx context.Context, namespace string, podNa
 	return provider.clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{})
 }
 
+func (provider *Provider) RemoveConfigMap(ctx context.Context, namespace string, configMapName string) error {
+	if isFound, err := provider.DoesConfigMapExist(ctx, namespace, configMapName); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.CoreV1().ConfigMaps(namespace).Delete(ctx, configMapName, metav1.DeleteOptions{})
+}
+
 func (provider *Provider) RemoveService(ctx context.Context, namespace string, serviceName string) error {
-	if isFound, err := provider.CheckServiceExists(ctx, namespace, serviceName); err != nil {
+	if isFound, err := provider.DoesServicesExist(ctx, namespace, serviceName); err != nil {
 		return err
 	} else if !isFound {
 		return nil
@@ -361,7 +530,7 @@ func (provider *Provider) RemoveService(ctx context.Context, namespace string, s
 }
 
 func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, daemonSetName string) error {
-	if isFound, err := provider.CheckDaemonSetExists(ctx, namespace, daemonSetName); err != nil {
+	if isFound, err := provider.DoesDaemonSetExist(ctx, namespace, daemonSetName); err != nil {
 		return err
 	} else if !isFound {
 		return nil
@@ -370,109 +539,33 @@ func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string,
 	return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{})
 }
 
-func (provider *Provider) CheckNamespaceExists(ctx context.Context, name string) (bool, error) {
-	listOptions := metav1.ListOptions{
-		FieldSelector: fmt.Sprintf("metadata.name=%s", name),
-		Limit:         1,
-	}
-	resourceList, err := provider.clientSet.CoreV1().Namespaces().List(ctx, listOptions)
-	if err != nil {
-		return false, err
+func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, data string) error {
+	if data == "" {
+		return nil
 	}
 
-	if len(resourceList.Items) > 0 {
-		return true, nil
+	configMapData := make(map[string]string, 0)
+	configMapData[shared.RulePolicyFileName] = data
+	configMap := &core.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ConfigMap",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configMapName,
+			Namespace: namespace,
+		},
+		Data: configMapData,
 	}
-
-	return false, nil
+	if _, err := provider.clientSet.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{}); err != nil {
+		return err
+	}
+	return nil
 }
 
-func (provider *Provider) CheckClusterRoleExists(ctx context.Context, name string) (bool, error) {
-	listOptions := metav1.ListOptions{
-		FieldSelector: fmt.Sprintf("metadata.name=%s", name),
-		Limit:         1,
-	}
-	resourceList, err := provider.clientSet.RbacV1().ClusterRoles().List(ctx, listOptions)
-	if err != nil {
-		return false, err
-	}
+func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool, resources configStructs.Resources) error {
+	logger.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
 
-	if len(resourceList.Items) > 0 {
-		return true, nil
-	}
-
-	return false, nil
-}
-
-func (provider *Provider) CheckClusterRoleBindingExists(ctx context.Context, name string) (bool, error) {
-	listOptions := metav1.ListOptions{
-		FieldSelector: fmt.Sprintf("metadata.name=%s", name),
-		Limit:         1,
-	}
-	resourceList, err := provider.clientSet.RbacV1().ClusterRoleBindings().List(ctx, listOptions)
-	if err != nil {
-		return false, err
-	}
-
-	if len(resourceList.Items) > 0 {
-		return true, nil
-	}
-
-	return false, nil
-}
-
-func (provider *Provider) CheckPodExists(ctx context.Context, namespace string, name string) (bool, error) {
-	listOptions := metav1.ListOptions{
-		FieldSelector: fmt.Sprintf("metadata.name=%s", name),
-		Limit:         1,
-	}
-	resourceList, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, listOptions)
-	if err != nil {
-		return false, err
-	}
-
-	if len(resourceList.Items) > 0 {
-		return true, nil
-	}
-
-	return false, nil
-}
-
-func (provider *Provider) CheckServiceExists(ctx context.Context, namespace string, name string) (bool, error) {
-	listOptions := metav1.ListOptions{
-		FieldSelector: fmt.Sprintf("metadata.name=%s", name),
-		Limit:         1,
-	}
-	resourceList, err := provider.clientSet.CoreV1().Services(namespace).List(ctx, listOptions)
-	if err != nil {
-		return false, err
-	}
-
-	if len(resourceList.Items) > 0 {
-		return true, nil
-	}
-
-	return false, nil
-}
-
-func (provider *Provider) CheckDaemonSetExists(ctx context.Context, namespace string, name string) (bool, error) {
-	listOptions := metav1.ListOptions{
-		FieldSelector: fmt.Sprintf("metadata.name=%s", name),
-		Limit:         1,
-	}
-	resourceList, err := provider.clientSet.AppsV1().DaemonSets(namespace).List(ctx, listOptions)
-	if err != nil {
-		return false, err
-	}
-
-	if len(resourceList.Items) > 0 {
-		return true, nil
-	}
-
-	return false, nil
-}
-
-func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool) error {
 	if len(nodeToTappedPodIPMap) == 0 {
 		return fmt.Errorf("Daemon set %s must tap at least 1 pod", daemonSetName)
 	}
@@ -486,19 +579,17 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 		"./mizuagent",
 		"-i", "any",
 		"--tap",
-		"--hardump",
 		"--api-server-address", fmt.Sprintf("ws://%s/wsTapper", apiServerPodIp),
 	}
 	if tapOutgoing {
 		mizuCmd = append(mizuCmd, "--anydirection")
 	}
 
-	privileged := true
 	agentContainer := applyconfcore.Container()
 	agentContainer.WithName(tapperPodName)
 	agentContainer.WithImage(podImage)
 	agentContainer.WithImagePullPolicy(core.PullAlways)
-	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(privileged))
+	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true))
 	agentContainer.WithCommand(mizuCmd...)
 	agentContainer.WithEnv(
 		applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"),
@@ -511,19 +602,19 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 			),
 		),
 	)
-	cpuLimit, err := resource.ParseQuantity("500m")
+	cpuLimit, err := resource.ParseQuantity(resources.CpuLimit)
 	if err != nil {
 		return errors.New(fmt.Sprintf("invalid cpu limit for %s container", tapperPodName))
 	}
-	memLimit, err := resource.ParseQuantity("1Gi")
+	memLimit, err := resource.ParseQuantity(resources.MemoryLimit)
 	if err != nil {
 		return errors.New(fmt.Sprintf("invalid memory limit for %s container", tapperPodName))
 	}
-	cpuRequests, err := resource.ParseQuantity("50m")
+	cpuRequests, err := resource.ParseQuantity(resources.CpuRequests)
 	if err != nil {
 		return errors.New(fmt.Sprintf("invalid cpu request for %s container", tapperPodName))
 	}
-	memRequests, err := resource.ParseQuantity("50Mi")
+	memRequests, err := resource.ParseQuantity(resources.MemoryRequests)
 	if err != nil {
 		return errors.New(fmt.Sprintf("invalid memory request for %s container", tapperPodName))
 	}
@@ -587,18 +678,55 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	return err
 }
 
-func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespace string) ([]core.Pod, error) {
-	pods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
-	if err != nil {
-		return nil, err
+func (provider *Provider) ListAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespaces []string) ([]core.Pod, error) {
+	var pods []core.Pod
+	for _, namespace := range namespaces {
+		namespacePods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
+		if err != nil {
+			return nil, fmt.Errorf("failed to get pods in ns: [%s], %w", namespace, err)
+		}
+
+		pods = append(pods, namespacePods.Items...)
 	}
+
 	matchingPods := make([]core.Pod, 0)
-	for _, pod := range pods.Items {
+	for _, pod := range pods {
 		if regex.MatchString(pod.Name) {
 			matchingPods = append(matchingPods, pod)
 		}
 	}
-	return matchingPods, err
+	return matchingPods, nil
+}
+
+func (provider *Provider) ListAllRunningPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespaces []string) ([]core.Pod, error) {
+	pods, err := provider.ListAllPodsMatchingRegex(ctx, regex, namespaces)
+	if err != nil {
+		return nil, err
+	}
+
+	matchingPods := make([]core.Pod, 0)
+	for _, pod := range pods {
+		if isPodRunning(&pod) {
+			matchingPods = append(matchingPods, pod)
+		}
+	}
+	return matchingPods, nil
+}
+
+func (provider *Provider) GetPodLogs(namespace string, podName string, ctx context.Context) (string, error) {
+	podLogOpts := core.PodLogOptions{}
+	req := provider.clientSet.CoreV1().Pods(namespace).GetLogs(podName, &podLogOpts)
+	podLogs, err := req.Stream(ctx)
+	if err != nil {
+		return "", fmt.Errorf("error opening log stream on ns: %s, pod: %s, %w", namespace, podName, err)
+	}
+	defer podLogs.Close()
+	buf := new(bytes.Buffer)
+	if _, err = io.Copy(buf, podLogs); err != nil {
+		return "", fmt.Errorf("error copy information from podLogs to buf, ns: %s, pod: %s, %w", namespace, podName, err)
+	}
+	str := buf.String()
+	return str, nil
 }
 
 func getClientSet(config *restclient.Config) *kubernetes.Clientset {
@@ -619,7 +747,7 @@ func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
 		kubeConfigPath = filepath.Join(home, ".kube", "config")
 	}
 
-	mizu.Log.Debugf("Using kube config %s", kubeConfigPath)
+	logger.Log.Debugf("Using kube config %s", kubeConfigPath)
 	configPathList := filepath.SplitList(kubeConfigPath)
 	configLoadingRules := &clientcmd.ClientConfigLoadingRules{}
 	if len(configPathList) <= 1 {
@@ -635,3 +763,7 @@ func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
 		},
 	)
 }
+
+func isPodRunning(pod *core.Pod) bool {
+	return pod.Status.Phase == core.PodRunning
+}

cli/kubernetes/proxy.go

@@ -2,6 +2,7 @@ package kubernetes
 
 import (
 	"fmt"
+	"github.com/up9inc/mizu/cli/logger"
 	"k8s.io/kubectl/pkg/proxy"
 	"net"
 	"net/http"
@@ -13,6 +14,7 @@ const k8sProxyApiPrefix = "/"
 const mizuServicePort = 80
 
 func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace string, mizuServiceName string) error {
+	logger.Log.Debugf("Starting proxy. namespace: [%v], service name: [%s], port: [%v]", mizuNamespace, mizuServiceName, mizuPort)
 	filter := &proxy.FilterServer{
 		AcceptPaths:   proxy.MakeRegexpArrayOrDie(proxy.DefaultPathAcceptRE),
 		RejectPaths:   proxy.MakeRegexpArrayOrDie(proxy.DefaultPathRejectRE),

cli/kubernetes/watch.go

@@ -4,49 +4,64 @@ import (
 	"context"
 	"errors"
 	"regexp"
+	"sync"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/watch"
 )
 
-func FilteredWatch(ctx context.Context, watcher watch.Interface, podFilter *regexp.Regexp) (chan *corev1.Pod, chan *corev1.Pod, chan *corev1.Pod, chan error) {
+func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetNamespaces []string, podFilter *regexp.Regexp) (chan *corev1.Pod, chan *corev1.Pod, chan *corev1.Pod, chan error) {
 	addedChan := make(chan *corev1.Pod)
 	modifiedChan := make(chan *corev1.Pod)
 	removedChan := make(chan *corev1.Pod)
 	errorChan := make(chan error)
-	go func() {
-		for {
-			select {
-			case e := <-watcher.ResultChan():
 
-				if e.Object == nil {
-					errorChan <- errors.New("kubernetes pod watch failed")
+	var wg sync.WaitGroup
+
+	for _, targetNamespace := range targetNamespaces {
+		wg.Add(1)
+
+		go func(targetNamespace string) {
+			defer wg.Done()
+			watcher := kubernetesProvider.GetPodWatcher(ctx, targetNamespace)
+
+			for {
+				select {
+				case e := <-watcher.ResultChan():
+					if e.Object == nil {
+						errorChan <- errors.New("kubernetes pod watch failed")
+						return
+					}
+
+					pod := e.Object.(*corev1.Pod)
+
+					if !podFilter.MatchString(pod.Name) {
+						continue
+					}
+
+					switch e.Type {
+					case watch.Added:
+						addedChan <- pod
+					case watch.Modified:
+						modifiedChan <- pod
+					case watch.Deleted:
+						removedChan <- pod
+					}
+				case <-ctx.Done():
+					watcher.Stop()
 					return
 				}
-
-				pod := e.Object.(*corev1.Pod)
-
-				if !podFilter.MatchString(pod.Name) {
-					continue
-				}
-
-				switch e.Type {
-				case watch.Added:
-					addedChan <- pod
-				case watch.Modified:
-					modifiedChan <- pod
-				case watch.Deleted:
-					removedChan <- pod
-				}
-			case <-ctx.Done():
-				watcher.Stop()
-				close(addedChan)
-				close(modifiedChan)
-				close(removedChan)
-				close(errorChan)
-				return
 			}
-		}
+		}(targetNamespace)
+	}
+
+	go func() {
+		<-ctx.Done()
+		wg.Wait()
+		close(addedChan)
+		close(modifiedChan)
+		close(removedChan)
+		close(errorChan)
 	}()
 
 	return addedChan, modifiedChan, removedChan, errorChan

cli/logger/logger.go

@@ -1,8 +1,8 @@
-package mizu
+package logger
 
 import (
-	"fmt"
 	"github.com/op/go-logging"
+	"github.com/up9inc/mizu/cli/mizu"
 	"os"
 	"path"
 )
@@ -13,16 +13,15 @@ var format = logging.MustStringFormatter(
 	`%{time} %{level:.5s} ▶ %{pid} %{shortfile} %{shortfunc} ▶ %{message}`,
 )
 
+func GetLogFilePath() string {
+	return path.Join(mizu.GetMizuFolderPath(), "mizu_cli.log")
+}
+
 func InitLogger() {
-	homeDirPath, _ := os.UserHomeDir()
-	mizuDirPath := path.Join(homeDirPath, ".mizu")
-	if err := os.MkdirAll(mizuDirPath, os.ModePerm); err != nil {
-		panic(fmt.Sprintf("Failed creating .mizu dir: %v, err %v", mizuDirPath, err))
-	}
-	logPath := path.Join(mizuDirPath, "log.log")
+	logPath := GetLogFilePath()
 	f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
 	if err != nil {
-		panic(fmt.Sprintf("Failed mizu log file: %v, err %v", logPath, err))
+		Log.Infof("Failed to open mizu log file: %v, err %v", logPath, err)
 	}
 
 	fileLog := logging.NewLogBackend(f, "", 0)
@@ -35,5 +34,6 @@ func InitLogger() {
 
 	logging.SetBackend(backend1Leveled, backend2Formatter)
 
-	Log.Debugf("Running mizu version %v", SemVer)
+	Log.Debugf("\n\n\n")
+	Log.Debugf("Running mizu version %v", mizu.SemVer)
 }

cli/mizu.go

@@ -2,10 +2,9 @@ package main
 
 import (
 	"github.com/up9inc/mizu/cli/cmd"
-	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/goUtils"
 )
 
 func main() {
-	mizu.InitLogger()
-	cmd.Execute()
+	goUtils.HandleExcWrapper(cmd.Execute)
 }

cli/mizu/config.go

@@ -1,221 +0,0 @@
-package mizu
-
-import (
-	"errors"
-	"fmt"
-	"github.com/up9inc/mizu/cli/uiUtils"
-	"gopkg.in/yaml.v3"
-	"io/ioutil"
-	"os"
-	"path"
-	"reflect"
-	"strconv"
-	"strings"
-)
-
-const separator = "="
-
-var configObj = map[string]interface{}{}
-
-type CommandLineFlag struct {
-	CommandLineName   string
-	YamlHierarchyName string
-	DefaultValue      interface{}
-}
-
-const (
-	ConfigurationKeyAnalyzingDestination = "tap.dest"
-	ConfigurationKeyUploadInterval       = "tap.uploadInterval"
-	ConfigurationKeyMizuImage            = "mizuImage"
-	ConfigurationKeyTelemetry            = "telemetry"
-)
-
-var allowedSetFlags = []CommandLineFlag{
-	{
-		CommandLineName:   "dest",
-		YamlHierarchyName: ConfigurationKeyAnalyzingDestination,
-		DefaultValue:      "up9.app",
-		// TODO: maybe add short description that we can show
-	},
-	{
-		CommandLineName:   "uploadInterval",
-		YamlHierarchyName: ConfigurationKeyUploadInterval,
-		DefaultValue:      10,
-	},
-	{
-		CommandLineName:   "mizuImage",
-		YamlHierarchyName: ConfigurationKeyMizuImage,
-		DefaultValue:      fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", Branch, SemVer),
-	},
-	{
-		CommandLineName:   "telemetry",
-		YamlHierarchyName: ConfigurationKeyTelemetry,
-		DefaultValue:      true,
-	},
-}
-
-func GetString(key string) string {
-	return fmt.Sprintf("%v", getValueFromMergedConfig(key))
-}
-
-func GetBool(key string) bool {
-	stringVal := GetString(key)
-	Log.Debugf("Found string value %v", stringVal)
-
-	val, err := strconv.ParseBool(stringVal)
-	if err != nil {
-		Log.Warningf(uiUtils.Red, fmt.Sprintf( "Invalid value %v for key %s, expected bool", stringVal, key))
-		os.Exit(1)
-	}
-	return val
-}
-
-func GetInt(key string) int {
-	stringVal := GetString(key)
-	Log.Debugf("Found string value %v", stringVal)
-
-	val, err := strconv.Atoi(stringVal)
-	if err != nil {
-		Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for key %s, expected int", stringVal, key))
-		os.Exit(1)
-	}
-	return val
-}
-
-func InitConfig(commandLineValues []string) error {
-	Log.Debugf("Merging default values")
-	mergeDefaultValues()
-	Log.Debugf("Merging config file values")
-	if err1 := mergeConfigFile(); err1 != nil {
-		Log.Infof(fmt.Sprintf(uiUtils.Red, "Invalid config file\n"))
-		return err1
-	}
-	Log.Debugf("Merging command line values")
-	if err2 := mergeCommandLineFlags(commandLineValues); err2 != nil {
-		Log.Infof(fmt.Sprintf(uiUtils.Red, "Invalid commanad argument\n"))
-		return err2
-	}
-	finalConfigPrettified, _ := uiUtils.PrettyJson(configObj)
-	Log.Debugf("Merged all config successfully\n Final config: %v", finalConfigPrettified)
-	return nil
-}
-
-func GetTemplateConfig() string {
-	templateConfig := map[string]interface{}{}
-	for _, allowedFlag := range allowedSetFlags {
-		addToConfigObj(allowedFlag.YamlHierarchyName, allowedFlag.DefaultValue, templateConfig)
-	}
-	prettifiedConfig, _ := uiUtils.PrettyYaml(templateConfig)
-	return prettifiedConfig
-}
-
-func GetConfigStr() string {
-	val, _ := uiUtils.PrettyYaml(configObj)
-	return val
-}
-
-func getValueFromMergedConfig(key string) interface{} {
-	if a, ok := configObj[key]; ok {
-		return a
-	}
-	return nil
-}
-
-func mergeDefaultValues() {
-	for _, allowedFlag := range allowedSetFlags {
-		Log.Debugf("Setting %v to %v", allowedFlag.YamlHierarchyName, allowedFlag.DefaultValue)
-		configObj[allowedFlag.YamlHierarchyName] = allowedFlag.DefaultValue
-	}
-}
-
-func mergeConfigFile() error {
-	Log.Debugf("Merging mizu config file values")
-	home, homeDirErr := os.UserHomeDir()
-	if homeDirErr != nil {
-		return nil
-	}
-	reader, openErr := os.Open(path.Join(home, ".mizu", "config.yaml"))
-	if openErr != nil {
-		return nil
-	}
-	buf, readErr := ioutil.ReadAll(reader)
-	if readErr != nil {
-		return readErr
-	}
-	m := make(map[string]interface{})
-	if err := yaml.Unmarshal(buf, &m); err != nil {
-		return err
-	}
-	for k, v := range m {
-		addToConfig(k, v)
-	}
-	return nil
-}
-
-func addToConfig(prefix string, value interface{}) {
-	typ := reflect.TypeOf(value).Kind()
-	if typ == reflect.Map {
-		for k1, v1 := range value.(map[string]interface{}) {
-			addToConfig(fmt.Sprintf("%s.%s", prefix, k1), v1)
-		}
-	} else {
-		validateConfigFileKey(prefix)
-		configObj[prefix] = value
-	}
-}
-
-func mergeCommandLineFlags(commandLineValues []string) error {
-	Log.Debugf("Merging Command line flags")
-	for _, e := range commandLineValues {
-		if !strings.Contains(e, separator) {
-			return errors.New(fmt.Sprintf("invalid set argument %s", e))
-		}
-		split := strings.SplitN(e, separator, 2)
-		if len(split) != 2 {
-			return errors.New(fmt.Sprintf("invalid set argument %s", e))
-		}
-		setFlagKey, argumentValue := split[0], split[1]
-		argumentNameInConfig, err := flagFromAllowed(setFlagKey)
-		if err != nil {
-			return err
-		}
-
-		configObj[argumentNameInConfig] = argumentValue
-	}
-	return nil
-}
-
-func flagFromAllowed(setFlagKey string) (string, error) {
-	for _, allowedFlag := range allowedSetFlags {
-		if strings.ToLower(allowedFlag.CommandLineName) == strings.ToLower(setFlagKey) {
-			return allowedFlag.YamlHierarchyName, nil
-		}
-	}
-	return "", errors.New(fmt.Sprintf("invalid set argument %s", setFlagKey))
-}
-
-func validateConfigFileKey(configFileKey string) {
-	for _, allowedFlag := range allowedSetFlags {
-		if allowedFlag.YamlHierarchyName == configFileKey {
-			return
-		}
-	}
-	Log.Info(fmt.Sprintf("Unknown argument: %s. Exit", configFileKey))
-	os.Exit(1)
-}
-
-func addToConfigObj(key string, value interface{}, configObj map[string]interface{}) {
-	typ := reflect.TypeOf(value).Kind()
-	if typ != reflect.Map {
-		if strings.Contains(key, ".") {
-			split := strings.SplitN(key, ".", 2)
-			firstLevelKey := split[0]
-			if _, ok := configObj[firstLevelKey]; !ok {
-				configObj[firstLevelKey] = map[string]interface{}{}
-			}
-			addToConfigObj(split[1], value, configObj[firstLevelKey].(map[string]interface{}))
-		} else {
-			configObj[key] = value
-		}
-	}
-}

cli/mizu/consts.go

@@ -1,5 +1,10 @@
 package mizu
 
+import (
+	"os"
+	"path"
+)
+
 var (
 	SemVer         = "0.0.1"
 	Branch         = "develop"
@@ -9,12 +14,23 @@ var (
 )
 
 const (
-	ApiServerPodName       = "mizu-api-server"
-	ClusterRoleBindingName = "mizu-cluster-role-binding"
-	ClusterRoleName        = "mizu-cluster-role"
+	MizuResourcesPrefix    = "mizu-"
+	ApiServerPodName       = MizuResourcesPrefix + "api-server"
+	ClusterRoleBindingName = MizuResourcesPrefix + "cluster-role-binding"
+	ClusterRoleName        = MizuResourcesPrefix + "cluster-role"
 	K8sAllNamespaces       = ""
-	ResourcesNamespace     = "mizu"
-	ServiceAccountName     = "mizu-service-account"
-	TapperDaemonSetName    = "mizu-tapper-daemon-set"
-	TapperPodName          = "mizu-tapper"
+	RoleBindingName        = MizuResourcesPrefix + "role-binding"
+	RoleName               = MizuResourcesPrefix + "role"
+	ServiceAccountName     = MizuResourcesPrefix + "service-account"
+	TapperDaemonSetName    = MizuResourcesPrefix + "tapper-daemon-set"
+	TapperPodName          = MizuResourcesPrefix + "tapper"
+	ConfigMapName          = MizuResourcesPrefix + "policy"
 )
+
+func GetMizuFolderPath() string {
+	home, homeDirErr := os.UserHomeDir()
+	if homeDirErr != nil {
+		return ""
+	}
+	return path.Join(home, ".mizu")
+}

cli/mizu/fsUtils/dirUtils.go

@@ -0,0 +1,26 @@
+package fsUtils
+
+import (
+	"errors"
+	"fmt"
+	"os"
+)
+
+func EnsureDir(dirName string) error {
+	err := os.Mkdir(dirName, 0700)
+	if err == nil {
+		return nil
+	}
+	if os.IsExist(err) {
+		// check that the existing path is a directory
+		info, err := os.Stat(dirName)
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			return errors.New(fmt.Sprintf("path exists but is not a directory: %s", dirName))
+		}
+		return nil
+	}
+	return err
+}

cli/mizu/fsUtils/mizuLogsUtils.go

@@ -0,0 +1,60 @@
+package fsUtils
+
+import (
+	"archive/zip"
+	"context"
+	"fmt"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu"
+	"os"
+	"regexp"
+)
+
+func DumpLogs(provider *kubernetes.Provider, ctx context.Context, filePath string) error {
+	podExactRegex := regexp.MustCompile("^" + mizu.MizuResourcesPrefix)
+	pods, err := provider.ListAllPodsMatchingRegex(ctx, podExactRegex, []string{config.Config.MizuResourcesNamespace})
+	if err != nil {
+		return err
+	}
+
+	if len(pods) == 0 {
+		return fmt.Errorf("no mizu pods found in namespace %s", config.Config.MizuResourcesNamespace)
+	}
+
+	newZipFile, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	defer newZipFile.Close()
+	zipWriter := zip.NewWriter(newZipFile)
+	defer zipWriter.Close()
+
+	for _, pod := range pods {
+		logs, err := provider.GetPodLogs(pod.Namespace, pod.Name, ctx)
+		if err != nil {
+			logger.Log.Errorf("Failed to get logs, %v", err)
+			continue
+		} else {
+			logger.Log.Debugf("Successfully read log length %d for pod: %s.%s", len(logs), pod.Namespace, pod.Name)
+		}
+		if err := AddStrToZip(zipWriter, logs, fmt.Sprintf("%s.%s.log", pod.Namespace, pod.Name)); err != nil {
+			logger.Log.Errorf("Failed write logs, %v", err)
+		} else {
+			logger.Log.Debugf("Successfully added log length %d from pod: %s.%s", len(logs), pod.Namespace, pod.Name)
+		}
+	}
+	if err := AddFileToZip(zipWriter, config.GetConfigFilePath()); err != nil {
+		logger.Log.Debugf("Failed write file, %v", err)
+	} else {
+		logger.Log.Debugf("Successfully added file %s", config.GetConfigFilePath())
+	}
+	if err := AddFileToZip(zipWriter, logger.GetLogFilePath()); err != nil {
+		logger.Log.Debugf("Failed write file, %v", err)
+	} else {
+		logger.Log.Debugf("Successfully added file %s", logger.GetLogFilePath())
+	}
+	logger.Log.Infof("You can find the zip file with all logs in %s\n", filePath)
+	return nil
+}

cli/mizu/fsUtils/zipUtils.go

@@ -0,0 +1,55 @@
+package fsUtils
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func AddFileToZip(zipWriter *zip.Writer, filename string) error {
+
+	fileToZip, err := os.Open(filename)
+	if err != nil {
+		return fmt.Errorf("failed to open file %s, %w", filename, err)
+	}
+	defer fileToZip.Close()
+
+	// Get the file information
+	info, err := fileToZip.Stat()
+	if err != nil {
+		return fmt.Errorf("failed to get file information %s, %w", filename, err)
+	}
+
+	header, err := zip.FileInfoHeader(info)
+	if err != nil {
+		return err
+	}
+
+	// Using FileInfoHeader() above only uses the basename of the file. If we want
+	// to preserve the folder structure we can overwrite this with the full path.
+	header.Name = filepath.Base(filename)
+
+	// Change to deflate to gain better compression
+	// see http://golang.org/pkg/archive/zip/#pkg-constants
+	header.Method = zip.Deflate
+
+	writer, err := zipWriter.CreateHeader(header)
+	if err != nil {
+		return fmt.Errorf("failed to create header in zip for %s, %w", filename, err)
+	}
+	_, err = io.Copy(writer, fileToZip)
+	return err
+}
+
+func AddStrToZip(writer *zip.Writer, logs string, fileName string) error {
+	if zipFile, err := writer.Create(fileName); err != nil {
+		return fmt.Errorf("couldn't create a log file inside zip for %s, %w", fileName, err)
+	} else {
+		if _, err = zipFile.Write([]byte(logs)); err != nil {
+			return fmt.Errorf("couldn't write logs to zip file: %s, %w", fileName, err)
+		}
+	}
+	return nil
+}

cli/mizu/goUtils/funcWrappers.go

@@ -0,0 +1,25 @@
+package goUtils
+
+import (
+	"github.com/up9inc/mizu/cli/logger"
+	"reflect"
+	"runtime/debug"
+)
+
+func HandleExcWrapper(fn interface{}, params ...interface{}) (result []reflect.Value) {
+	defer func() {
+		if panicMessage := recover(); panicMessage != nil {
+			stack := debug.Stack()
+			logger.Log.Fatalf("Unhandled panic: %v\n stack: %s", panicMessage, stack)
+		}
+	}()
+	f := reflect.ValueOf(fn)
+	if f.Type().NumIn() != len(params) {
+		panic("incorrect number of parameters!")
+	}
+	inputs := make([]reflect.Value, len(params))
+	for k, in := range params {
+		inputs[k] = reflect.ValueOf(in)
+	}
+	return f.Call(inputs)
+}

cli/mizu/sliceUtils.go

@@ -0,0 +1,11 @@
+package mizu
+
+func Contains(slice []string, containsValue string) bool {
+	for _, sliceValue := range slice {
+		if sliceValue == containsValue {
+			return true
+		}
+	}
+
+	return false
+}

cli/mizu/sliceUtils_test.go

@@ -0,0 +1,82 @@
+package mizu_test
+
+import (
+	"github.com/up9inc/mizu/cli/mizu"
+	"testing"
+)
+
+func TestContainsExists(t *testing.T) {
+	tests := []struct {
+		slice         []string
+		containsValue string
+		expected      bool
+	}{
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "apple", expected: true},
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "orange", expected: true},
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "banana", expected: true},
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "grapes", expected: true},
+	}
+
+	for _, test := range tests {
+		actual := mizu.Contains(test.slice, test.containsValue)
+		if actual != test.expected {
+			t.Errorf("unexpected result - expected: %v, actual: %v", test.expected, actual)
+		}
+	}
+}
+
+func TestContainsNotExists(t *testing.T) {
+	tests := []struct {
+		slice         []string
+		containsValue string
+		expected      bool
+	}{
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "cat", expected: false},
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "dog", expected: false},
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "apples", expected: false},
+		{slice: []string{"apple", "orange", "banana", "grapes"}, containsValue: "rapes", expected: false},
+	}
+
+	for _, test := range tests {
+		actual := mizu.Contains(test.slice, test.containsValue)
+		if actual != test.expected {
+			t.Errorf("unexpected result - expected: %v, actual: %v", test.expected, actual)
+		}
+	}
+}
+
+func TestContainsEmptySlice(t *testing.T) {
+	tests := []struct {
+		slice         []string
+		containsValue string
+		expected      bool
+	}{
+		{slice: []string{}, containsValue: "cat", expected: false},
+		{slice: []string{}, containsValue: "dog", expected: false},
+	}
+
+	for _, test := range tests {
+		actual := mizu.Contains(test.slice, test.containsValue)
+		if actual != test.expected {
+			t.Errorf("unexpected result - expected: %v, actual: %v", test.expected, actual)
+		}
+	}
+}
+
+func TestContainsNilSlice(t *testing.T) {
+	tests := []struct {
+		slice         []string
+		containsValue string
+		expected      bool
+	}{
+		{slice: nil, containsValue: "cat", expected: false},
+		{slice: nil, containsValue: "dog", expected: false},
+	}
+
+	for _, test := range tests {
+		actual := mizu.Contains(test.slice, test.containsValue)
+		if actual != test.expected {
+			t.Errorf("unexpected result - expected: %v, actual: %v", test.expected, actual)
+		}
+	}
+}

cli/mizu/telemetry.go

@@ -1,39 +0,0 @@
-package mizu
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"net/http"
-)
-
-const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
-
-func ReportRun(cmd string, args interface{}) {
-	if !GetBool(ConfigurationKeyTelemetry) {
-		Log.Debugf("not reporting due to config value")
-		return
-	}
-
-	if Branch != "main" {
-		Log.Debugf("reporting only on main branch")
-		return
-	}
-	argsBytes, _ := json.Marshal(args)
-	argsMap := map[string]string{
-		"telemetry_type": "execution",
-		"cmd":            cmd,
-		"args":           string(argsBytes),
-		"component":      "mizu_cli",
-		"BuildTimestamp": BuildTimestamp,
-		"version":        SemVer}
-	argsMap["message"] = fmt.Sprintf("mizu %v - %v", argsMap["cmd"], string(argsBytes))
-
-	jsonValue, _ := json.Marshal(argsMap)
-
-	if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
-		Log.Debugf("error sending telemetry err: %v, response %v", err, resp)
-	} else {
-		Log.Debugf("Successfully reported telemetry")
-	}
-}

cli/mizu/version/versionCheck.go

@@ -1,17 +1,20 @@
-package mizu
+package version
 
 import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/google/go-github/v37/github"
-	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/semver"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"time"
+
+	"github.com/google/go-github/v37/github"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/semver"
 )
 
 func getApiVersion(port uint16) (string, error) {
@@ -40,22 +43,22 @@ func CheckVersionCompatibility(port uint16) (bool, error) {
 		return false, err
 	}
 
-	if semver.SemVersion(apiSemVer).Major() == semver.SemVersion(SemVer).Major() &&
-		semver.SemVersion(apiSemVer).Minor() == semver.SemVersion(SemVer).Minor() {
+	if semver.SemVersion(apiSemVer).Major() == semver.SemVersion(mizu.SemVer).Major() &&
+		semver.SemVersion(apiSemVer).Minor() == semver.SemVersion(mizu.SemVer).Minor() {
 		return true, nil
 	}
 
-	Log.Infof(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", SemVer, apiSemVer))
+	logger.Log.Errorf(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", mizu.SemVer, apiSemVer))
 	return false, nil
 }
 
 func CheckNewerVersion() {
-	Log.Debugf("Checking for newer version...")
+	logger.Log.Debugf("Checking for newer version...")
 	start := time.Now()
 	client := github.NewClient(nil)
 	latestRelease, _, err := client.Repositories.GetLatestRelease(context.Background(), "up9inc", "mizu")
 	if err != nil {
-		Log.Debugf("Failed to get latest release")
+		logger.Log.Debugf("[ERROR] Failed to get latest release")
 		return
 	}
 
@@ -67,26 +70,26 @@ func CheckNewerVersion() {
 		}
 	}
 	if versionFileUrl == "" {
-		Log.Debugf("Version file not found in the latest release")
+		logger.Log.Debugf("[ERROR] Version file not found in the latest release")
 		return
 	}
 
 	res, err := http.Get(versionFileUrl)
 	if err != nil {
-		Log.Debugf("http.Get version asset -> %v", err)
+		logger.Log.Debugf("[ERROR] Failed to get the version file %v", err)
 		return
 	}
 
 	data, err := ioutil.ReadAll(res.Body)
 	res.Body.Close()
 	if err != nil {
-		Log.Debugf("ioutil.ReadAll -> %v", err)
+		logger.Log.Debugf("[ERROR] Failed to read the version file -> %v", err)
 		return
 	}
 	gitHubVersion := string(data)
 	gitHubVersion = gitHubVersion[:len(gitHubVersion)-1]
-	Log.Debugf("Finished version validation, took %v", time.Since(start))
-	if SemVer < gitHubVersion {
-		Log.Infof(uiUtils.Yellow, fmt.Sprintf("Update available! %v -> %v (%v)", SemVer, gitHubVersion, *latestRelease.HTMLURL))
+	logger.Log.Debugf("Finished version validation, took %v", time.Since(start))
+	if mizu.SemVer < gitHubVersion {
+		logger.Log.Infof(uiUtils.Yellow, fmt.Sprintf("Update available! %v -> %v (%v)", mizu.SemVer, gitHubVersion, *latestRelease.HTMLURL))
 	}
 }

cli/telemetry/telemetry.go

@@ -0,0 +1,109 @@
+package telemetry
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu"
+	"io/ioutil"
+	"net/http"
+)
+
+const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
+
+func ReportRun(cmd string, args interface{}) {
+	if !shouldRunTelemetry() {
+		logger.Log.Debugf("not reporting telemetry")
+		return
+	}
+
+	argsBytes, _ := json.Marshal(args)
+	argsMap := map[string]interface{}{
+		"cmd":  cmd,
+		"args": string(argsBytes),
+	}
+
+	if err := sendTelemetry("Execution", argsMap); err != nil {
+		logger.Log.Debug(err)
+		return
+	}
+
+	logger.Log.Debugf("successfully reported telemetry for cmd %v", cmd)
+}
+
+func ReportAPICalls(mizuPort uint16) {
+	if !shouldRunTelemetry() {
+		logger.Log.Debugf("not reporting telemetry")
+		return
+	}
+
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizuPort)
+	generalStatsUrl := fmt.Sprintf("http://%s/api/generalStats", mizuProxiedUrl)
+
+	response, requestErr := http.Get(generalStatsUrl)
+	if requestErr != nil {
+		logger.Log.Debugf("ERROR: failed to get general stats for telemetry, err: %v", requestErr)
+		return
+	} else if response.StatusCode != 200 {
+		logger.Log.Debugf("ERROR: failed to get general stats for telemetry, status code: %v", response.StatusCode)
+		return
+	}
+
+	defer func() { _ = response.Body.Close() }()
+
+	data, readErr := ioutil.ReadAll(response.Body)
+	if readErr != nil {
+		logger.Log.Debugf("ERROR: failed to read general stats for telemetry, err: %v", readErr)
+		return
+	}
+
+	var generalStats map[string]interface{}
+	if parseErr := json.Unmarshal(data, &generalStats); parseErr != nil {
+		logger.Log.Debugf("ERROR: failed to parse general stats for telemetry, err: %v", parseErr)
+		return
+	}
+
+	argsMap := map[string]interface{}{
+		"apiCallsCount":         generalStats["EntriesCount"],
+		"firstAPICallTimestamp": generalStats["FirstEntryTimestamp"],
+		"lastAPICallTimestamp":  generalStats["LastEntryTimestamp"],
+	}
+
+	if err := sendTelemetry("APICalls", argsMap); err != nil {
+		logger.Log.Debug(err)
+		return
+	}
+
+	logger.Log.Debugf("successfully reported telemetry of api calls")
+}
+
+func shouldRunTelemetry() bool {
+	if !config.Config.Telemetry {
+		return false
+	}
+
+	if mizu.Branch != "main" && mizu.Branch != "develop" {
+		return false
+	}
+
+	return true
+}
+
+func sendTelemetry(telemetryType string, argsMap map[string]interface{}) error {
+	argsMap["telemetryType"] = telemetryType
+	argsMap["component"] = "mizu_cli"
+	argsMap["buildTimestamp"] = mizu.BuildTimestamp
+	argsMap["branch"] = mizu.Branch
+	argsMap["version"] = mizu.SemVer
+
+	jsonValue, _ := json.Marshal(argsMap)
+
+	if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+		return fmt.Errorf("ERROR: failed sending telemetry, err: %v, response %v", err, resp)
+	}
+
+	return nil
+}

cli/uiUtils/colors.go

@@ -2,12 +2,14 @@ package uiUtils
 
 
 const (
-	Black        = "\033[1;30m%s\033[0m"
-	Red          = "\033[1;31m%s\033[0m"
-	Green        = "\033[1;32m%s\033[0m"
-	Yellow       = "\033[1;33m%s\033[0m"
-	Purple       = "\033[1;34m%s\033[0m"
-	Magenta      = "\033[1;35m%s\033[0m"
-	Teal         = "\033[1;36m%s\033[0m"
-	White        = "\033[1;37m%s\033[0m"
-)
+	Black   = "\033[1;30m%s\033[0m"
+	Red     = "\033[1;31m%s\033[0m"
+	Green   = "\033[1;32m%s\033[0m"
+	Yellow  = "\033[1;33m%s\033[0m"
+	Purple  = "\033[1;34m%s\033[0m"
+	Magenta = "\033[1;35m%s\033[0m"
+	Teal    = "\033[1;36m%s\033[0m"
+	White   = "\033[1;37m%s\033[0m"
+	Error   = Red
+	Warning = Yellow
+)

codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+  status:
+    project:
+      default:
+        threshold: 1%
+    patch:
+      default:
+        enabled: no

docs/POLICY_RULES.md

@@ -0,0 +1,78 @@
+
+# API rules validation
+
+This feature allows you to define set of simple rules, and test the API against them.
+Such validation may test response for specific JSON fields, headers, etc.
+
+## Examples
+
+
+Example 1: HTTP request (REST API call) that didn’t pass validation is highlighted in red
+
+![Simple UI](../assets/validation-example1.png)
+
+- - -
+
+
+Example 2: Details pane shows the validation rule details and whether it passed or failed
+
+![Simple UI](../assets/validation-example2.png)
+
+
+## How to use
+To use this feature - create simple rules file (see details below) and pass this file as parameter to `mizu tap` command. For example, if rules are stored in file named `rules.yaml` — run the following command:
+
+
+```shell
+mizu tap --test-rules rules.yaml PODNAME
+```
+
+
+
+## Rules file structure
+
+The structure of the test-rules-file is:
+
+* `name`: string, name of the rule
+* `type`: string, type of the rule, must be `json` or `header` or `latency`
+* `key`: string, [jsonpath](https://code.google.com/archive/p/jsonpath/wikis/Javascript.wiki) used only in `json` or `header` type
+* `value`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) used only in `json` or `header` type
+* `service`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) service name to filter
+* `path`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) URL path to filter
+* `latency`: integer, time in ms of the expected latency.
+
+
+### For example:
+
+```yaml
+rules:
+- name: holy-in-name-property
+  type: json
+  key: "$.name"
+  value: "Holy"
+  service: "catalogue.*"
+  path: "catalogue.*"
+- name: content-length-header
+  type: header
+  key: "Content-Le.*"
+  value: "(\\d+(?:\\.\\d+)?)"
+- name: latency-test
+  type: latency
+  latency: 1
+  service: "carts.*"
+```
+
+### Explanation:
+
+* First rule `holy-in-name-property`:
+
+  > This rule will be applied to all request made to `catalogue.*` services with `catalogue.*` on the URL path with a json response containing a `$.name` field. If the value of `$.name` is `Holy` than is marked as success, marked as failure otherwise.
+
+* Second rule `content-length-header`:
+
+  > This rule will be applied to all request that has `Content-Le.*` on header. If the value of `Content-Le.*` is `(\\d+(?:\\.\\d+)?)` (number), will be marked as success, marked as failure otherwise.
+
+* Third rule `latency-test`:
+
+  > This rule will be applied to all request made to `carts.*` services. If the latency of the response is greater than `1` will be marked as failure, marked as success otherwise.
+

examples/roles/permissions-all-namespaces-without-ip-resolution.yaml

@@ -7,16 +7,16 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["pods"]
-  verbs: ["list", "watch", "create"]
+  verbs: ["list", "watch", "create", "delete"]
 - apiGroups: [""]
   resources: ["services"]
-  verbs: ["create"]
+  verbs: ["create", "delete"]
 - apiGroups: ["apps"]
   resources: ["daemonsets"]
-  verbs: ["create", "patch"]
+  verbs: ["create", "patch", "delete"]
 - apiGroups: [""]
   resources: ["namespaces"]
-  verbs: ["list", "watch", "create", "delete"]
+  verbs: ["get", "list", "watch", "create", "delete"]
 - apiGroups: [""]
   resources: ["services/proxy"]
   verbs: ["get"]

examples/roles/permissions-all-namespaces.yaml

@@ -6,28 +6,34 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["pods"]
-  verbs: ["get", "list", "watch", "create"]
+  verbs: ["get", "list", "watch", "create", "delete"]
 - apiGroups: [""]
   resources: ["services"]
-  verbs: ["get", "list", "watch", "create"]
+  verbs: ["get", "list", "watch", "create", "delete"]
 - apiGroups: ["apps"]
   resources: ["daemonsets"]
-  verbs: ["create", "patch"]
+  verbs: ["create", "patch", "delete"]
 - apiGroups: [""]
   resources: ["namespaces"]
-  verbs: ["list", "watch", "create", "delete"]
+  verbs: ["get", "list", "watch", "create", "delete"]
 - apiGroups: [""]
   resources: ["services/proxy"]
   verbs: ["get"]
 - apiGroups: [""]
   resources: ["serviceaccounts"]
-  verbs: ["get", "create"]
+  verbs: ["get", "create", "delete"]
 - apiGroups: ["rbac.authorization.k8s.io"]
   resources: ["clusterroles"]
-  verbs: ["list", "create", "delete"]
+  verbs: ["get", "create", "delete"]
 - apiGroups: ["rbac.authorization.k8s.io"]
   resources: ["clusterrolebindings"]
-  verbs: ["list", "create", "delete"]
+  verbs: ["get", "create", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["roles"]
+  verbs: ["get", "create", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["rolebindings"]
+  verbs: ["get", "create", "delete"]
 - apiGroups: ["apps", "extensions"]
   resources: ["pods"]
   verbs: ["get", "list", "watch"]

examples/roles/permissions-ns-with-validation.yaml

@@ -0,0 +1,54 @@
+# This example shows the roles required for a user to be able to use Mizu in a single namespace.
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-role
+  namespace: user1
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch", "create", "delete"]
+- apiGroups: [""]
+  resources: ["services"]
+  verbs: ["get", "list", "watch", "create", "delete"]
+- apiGroups: ["apps"]
+  resources: ["daemonsets"]
+  verbs: ["get", "create", "patch", "delete"]
+- apiGroups: [""]
+  resources: ["services/proxy"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["get", "create", "delete"]
+- apiGroups: [""]
+  resources: ["serviceaccounts"]
+  verbs: ["get", "create", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["roles"]
+  verbs: ["get", "create", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["rolebindings"]
+  verbs: ["get", "create", "delete"]
+- apiGroups: ["apps", "extensions"]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["apps", "extensions"]
+  resources: ["services"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["", "apps", "extensions"]
+  resources: ["endpoints"]
+  verbs: ["get", "list", "watch"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-rolebindings
+  namespace: user1
+subjects:
+- kind: User
+  name: user1
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: Role
+  name: mizu-runner-role
+  apiGroup: rbac.authorization.k8s.io

examples/roles/permissions-ns-without-ip-resolution.yaml

@@ -0,0 +1,33 @@
+# This example shows the roles required for a user to be able to use Mizu in a single namespace with IP resolution disabled.
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-role
+  namespace: user1
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch", "create", "delete"]
+- apiGroups: [""]
+  resources: ["services"]
+  verbs: ["get", "create", "delete"]
+- apiGroups: ["apps"]
+  resources: ["daemonsets"]
+  verbs: ["get", "create", "patch", "delete"]
+- apiGroups: [""]
+  resources: ["services/proxy"]
+  verbs: ["get"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-rolebindings
+  namespace: user1
+subjects:
+- kind: User
+  name: user1
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: Role
+  name: mizu-runner-role
+  apiGroup: rbac.authorization.k8s.io

examples/roles/permissions-ns.yaml

@@ -0,0 +1,51 @@
+# This example shows the roles required for a user to be able to use Mizu in a single namespace.
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-role
+  namespace: user1
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch", "create", "delete"]
+- apiGroups: [""]
+  resources: ["services"]
+  verbs: ["get", "list", "watch", "create", "delete"]
+- apiGroups: ["apps"]
+  resources: ["daemonsets"]
+  verbs: ["get", "create", "patch", "delete"]
+- apiGroups: [""]
+  resources: ["services/proxy"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["serviceaccounts"]
+  verbs: ["get", "create", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["roles"]
+  verbs: ["get", "create", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["rolebindings"]
+  verbs: ["get", "create", "delete"]
+- apiGroups: ["apps", "extensions"]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["apps", "extensions"]
+  resources: ["services"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["", "apps", "extensions"]
+  resources: ["endpoints"]
+  verbs: ["get", "list", "watch"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-rolebindings
+  namespace: user1
+subjects:
+- kind: User
+  name: user1
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: Role
+  name: mizu-runner-role
+  apiGroup: rbac.authorization.k8s.io

shared/consts.go

@@ -6,4 +6,6 @@ const (
 	NodeNameEnvVar                   = "NODE_NAME"
 	TappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST"
 	MaxEntriesDBSizeBytesEnvVar      = "MAX_ENTRIES_DB_BYTES"
+	RulePolicyPath                   = "/app/enforce-policy/"
+	RulePolicyFileName               = "enforce-policy.yaml"
 )

shared/debounce/debounce.go

@@ -1,6 +1,7 @@
 package debounce
 
 import (
+	"fmt"
 	"time"
 )
 
@@ -13,9 +14,10 @@ func NewDebouncer(timeout time.Duration, callback func()) *Debouncer {
 
 type Debouncer struct {
 	callback func()
-	running bool
-	timeout time.Duration
-	timer *time.Timer
+	running  bool
+	canceled bool
+	timeout  time.Duration
+	timer    *time.Timer
 }
 
 func (d *Debouncer) setTimeout(timeout time.Duration) {
@@ -25,18 +27,28 @@ func (d *Debouncer) setTimeout(timeout time.Duration) {
 
 func (d *Debouncer) setCallback(callback func()) {
 	callbackWrapped := func() {
-		callback()
+		if !d.canceled {
+			callback()
+		}
 		d.running = false
 	}
 
 	d.callback = callbackWrapped
 }
 
-func (d *Debouncer) SetOn() {
+func (d *Debouncer) Cancel() {
+	d.canceled = true
+}
+
+func (d *Debouncer) SetOn() error {
+	if d.canceled {
+		return fmt.Errorf("debouncer cancelled")
+	}
 	if d.running == true {
-		return
+		return nil
 	}
 
 	d.running = true
 	d.timer = time.AfterFunc(d.timeout, d.callback)
+	return nil
 }

shared/go.mod

@@ -3,7 +3,7 @@ module github.com/up9inc/mizu/shared
 go 1.16
 
 require (
-	github.com/gorilla/websocket v1.4.2
 	github.com/docker/go-units v0.4.0
+	github.com/gorilla/websocket v1.4.2
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
-

shared/go.sum

@@ -2,3 +2,7 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

shared/models.go

@@ -1,5 +1,13 @@
 package shared
 
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
 type WebSocketMessageType string
 
 const (
@@ -7,6 +15,7 @@ const (
 	WebSocketMessageTypeTappedEntry   WebSocketMessageType = "tappedEntry"
 	WebSocketMessageTypeUpdateStatus  WebSocketMessageType = "status"
 	WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus"
+	WebsocketMessageTypeOutboundLink  WebSocketMessageType = "outboundLink"
 )
 
 type WebSocketMessageMetadata struct {
@@ -31,7 +40,8 @@ type WebSocketStatusMessage struct {
 }
 
 type TapStatus struct {
-	Pods []PodInfo `json:"pods"`
+	Pods     []PodInfo     `json:"pods"`
+	TLSLinks []TLSLinkInfo `json:"tlsLinks"`
 }
 
 type PodInfo struct {
@@ -39,6 +49,13 @@ type PodInfo struct {
 	Name      string `json:"name"`
 }
 
+type TLSLinkInfo struct {
+	SourceIP                string `json:"sourceIP"`
+	DestinationAddress      string `json:"destinationAddress"`
+	ResolvedDestinationName string `json:"resolvedDestinationName"`
+	ResolvedSourceName      string `json:"resolvedSourceName"`
+}
+
 func CreateWebSocketStatusMessage(tappingStatus TapStatus) WebSocketStatusMessage {
 	return WebSocketStatusMessage{
 		WebSocketMessageMetadata: &WebSocketMessageMetadata{
@@ -58,12 +75,84 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc
 }
 
 type TrafficFilteringOptions struct {
-	PlainTextMaskingRegexes []*SerializableRegexp
-	HideHealthChecks        bool
-	DisableRedaction        bool
+	HealthChecksUserAgentHeaders []string
+	PlainTextMaskingRegexes      []*SerializableRegexp
+	DisableRedaction             bool
 }
 
 type VersionResponse struct {
 	SemVer string `json:"semver"`
 }
 
+type RulesPolicy struct {
+	Rules []RulePolicy `yaml:"rules"`
+}
+
+type RulePolicy struct {
+	Type    string `yaml:"type"`
+	Service string `yaml:"service"`
+	Path    string `yaml:"path"`
+	Method  string `yaml:"method"`
+	Key     string `yaml:"key"`
+	Value   string `yaml:"value"`
+	Latency int64  `yaml:"latency"`
+	Name    string `yaml:"name"`
+}
+
+func (r *RulePolicy) validateType() bool {
+	permitedTypes := []string{"json", "header", "latency"}
+	_, found := Find(permitedTypes, r.Type)
+	if !found {
+		fmt.Printf("\nRule with name %s will be ignored. Err: only json, header and latency types are supported on rule definition.\n", r.Name)
+	}
+	if strings.ToLower(r.Type) == "latency" {
+		if r.Latency == 0 {
+			fmt.Printf("\nRule with name %s will be ignored. Err: when type=latency, the field Latency should be specified and have a value >= 1\n\n", r.Name)
+			found = false
+		}
+	}
+	return found
+}
+
+func (rules *RulesPolicy) ValidateRulesPolicy() []int {
+	invalidIndex := make([]int, 0)
+	for i := range rules.Rules {
+		validated := rules.Rules[i].validateType()
+		if !validated {
+			invalidIndex = append(invalidIndex, i)
+		}
+	}
+	return invalidIndex
+}
+
+func (rules *RulesPolicy) RemoveRule(idx int) {
+	rules.Rules = append(rules.Rules[:idx], rules.Rules[idx+1:]...)
+}
+
+func Find(slice []string, val string) (int, bool) {
+	for i, item := range slice {
+		if item == val {
+			return i, true
+		}
+	}
+	return -1, false
+}
+
+func DecodeEnforcePolicy(path string) (RulesPolicy, error) {
+	content, err := ioutil.ReadFile(path)
+	enforcePolicy := RulesPolicy{}
+	if err != nil {
+		return enforcePolicy, err
+	}
+	err = yaml.Unmarshal([]byte(content), &enforcePolicy)
+	if err != nil {
+		return enforcePolicy, err
+	}
+	invalidIndex := enforcePolicy.ValidateRulesPolicy()
+	if len(invalidIndex) != 0 {
+		for i := range invalidIndex {
+			enforcePolicy.RemoveRule(invalidIndex[i])
+		}
+	}
+	return enforcePolicy, nil
+}

tap/cleaner.go

@@ -1,6 +1,7 @@
 package tap
 
 import (
+	"github.com/romana/rlog"
 	"sync"
 	"time"
 
@@ -20,19 +21,21 @@ type Cleaner struct {
 	cleanPeriod       time.Duration
 	connectionTimeout time.Duration
 	stats             CleanerStats
-	statsMutex	  sync.Mutex
+	statsMutex        sync.Mutex
 }
 
 func (cl *Cleaner) clean() {
 	startCleanTime := time.Now()
 
 	cl.assemblerMutex.Lock()
+	rlog.Debugf("Assembler Stats before cleaning %s", cl.assembler.Dump())
 	flushed, closed := cl.assembler.FlushCloseOlderThan(startCleanTime.Add(-cl.connectionTimeout))
 	cl.assemblerMutex.Unlock()
 
 	deleted := cl.matcher.deleteOlderThan(startCleanTime.Add(-cl.connectionTimeout))
 
 	cl.statsMutex.Lock()
+	rlog.Debugf("Assembler Stats after cleaning %s", cl.assembler.Dump())
 	cl.stats.flushed += flushed
 	cl.stats.closed += closed
 	cl.stats.deleted += deleted
@@ -55,7 +58,7 @@ func (cl *Cleaner) dumpStats() CleanerStats {
 
 	stats := CleanerStats{
 		flushed: cl.stats.flushed,
-		closed : cl.stats.closed,
+		closed:  cl.stats.closed,
 		deleted: cl.stats.deleted,
 	}
 

tap/go.mod

@@ -8,4 +8,5 @@ require (
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
 	golang.org/x/net v0.0.0-20210421230115-4e50805a0758
+	github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4
 )

tap/go.sum

@@ -1,3 +1,5 @@
+github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M=
+github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=

tap/har_writer.go

@@ -43,7 +43,7 @@ func openNewHarFile(filename string) *HarFile {
 }
 
 type HarFile struct {
-	file *os.File
+	file       *os.File
 	entryCount int
 }
 
@@ -105,13 +105,13 @@ func NewEntry(request *http.Request, requestTime time.Time, response *http.Respo
 
 	harEntry := har.Entry{
 		StartedDateTime: time.Now().UTC(),
-		Time: totalTime,
-		Request: harRequest,
-		Response: harResponse,
-		Cache: &har.Cache{},
+		Time:            totalTime,
+		Request:         harRequest,
+		Response:        harResponse,
+		Cache:           &har.Cache{},
 		Timings: &har.Timings{
-			Send: -1,
-			Wait: -1,
+			Send:    -1,
+			Wait:    -1,
 			Receive: totalTime,
 		},
 	}
@@ -155,14 +155,14 @@ func (f *HarFile) Close() {
 	}
 }
 
-func (f*HarFile) writeHeader() {
+func (f *HarFile) writeHeader() {
 	header := []byte(`{"log": {"version": "1.2", "creator": {"name": "Mizu", "version": "0.0.1"}, "entries": [`)
 	if _, err := f.file.Write(header); err != nil {
 		log.Panicf("Failed to write header to output file: %s (%v,%+v)", err, err, err)
 	}
 }
 
-func (f*HarFile) writeTrailer() {
+func (f *HarFile) writeTrailer() {
 	trailer := []byte("]}}")
 	if _, err := f.file.Write(trailer); err != nil {
 		log.Panicf("Failed to write trailer to output file: %s (%v,%+v)", err, err, err)
@@ -172,26 +172,27 @@ func (f*HarFile) writeTrailer() {
 func NewHarWriter(outputDir string, maxEntries int) *HarWriter {
 	return &HarWriter{
 		OutputDirPath: outputDir,
-		MaxEntries: maxEntries,
-		PairChan: make(chan *PairChanItem),
-		OutChan: make(chan *OutputChannelItem, 1000),
-		currentFile: nil,
-		done: make(chan bool),
+		MaxEntries:    maxEntries,
+		PairChan:      make(chan *PairChanItem),
+		OutChan:       make(chan *OutputChannelItem, 1000),
+		currentFile:   nil,
+		done:          make(chan bool),
 	}
 }
 
 type OutputChannelItem struct {
-	HarEntry       *har.Entry
-	ConnectionInfo *ConnectionInfo
+	HarEntry               *har.Entry
+	ConnectionInfo         *ConnectionInfo
+	ValidationRulesChecker string
 }
 
 type HarWriter struct {
 	OutputDirPath string
-	MaxEntries int
-	PairChan chan *PairChanItem
-	OutChan chan *OutputChannelItem
-	currentFile *HarFile
-	done chan bool
+	MaxEntries    int
+	PairChan      chan *PairChanItem
+	OutChan       chan *OutputChannelItem
+	currentFile   *HarFile
+	done          chan bool
 }
 
 func (hw *HarWriter) WritePair(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time, connectionInfo *ConnectionInfo) {
@@ -240,7 +241,7 @@ func (hw *HarWriter) Start() {
 			hw.closeFile()
 		}
 		hw.done <- true
-	} ()
+	}()
 }
 
 func (hw *HarWriter) Stop() {

tap/http_reader.go

@@ -5,21 +5,25 @@ import (
 	"bytes"
 	"encoding/hex"
 	"fmt"
+	"github.com/bradleyfalzon/tlsx"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strconv"
 	"sync"
 	"time"
 )
 
+const checkTLSPacketAmount = 100
+
 type httpReaderDataMsg struct {
 	bytes     []byte
 	timestamp time.Time
 }
 
 type tcpID struct {
-	srcIP string
-	dstIP string
+	srcIP   string
+	dstIP   string
 	srcPort string
 	dstPort string
 }
@@ -42,28 +46,44 @@ func (tid *tcpID) String() string {
  * Implements io.Reader interface (Read)
  */
 type httpReader struct {
-	ident         string
-	tcpID         tcpID
-	isClient      bool
-	isHTTP2       bool
-	isOutgoing    bool
-	msgQueue      chan httpReaderDataMsg // Channel of captured reassembled tcp payload
-	data          []byte
-	captureTime   time.Time
-	hexdump       bool
-	parent        *tcpStream
-	grpcAssembler GrpcAssembler
-	messageCount  uint
-	harWriter     *HarWriter
+	ident              string
+	tcpID              tcpID
+	isClient           bool
+	isHTTP2            bool
+	isOutgoing         bool
+	msgQueue           chan httpReaderDataMsg // Channel of captured reassembled tcp payload
+	data               []byte
+	captureTime        time.Time
+	hexdump            bool
+	parent             *tcpStream
+	grpcAssembler      GrpcAssembler
+	messageCount       uint
+	harWriter          *HarWriter
+	packetsSeen        uint
+	outboundLinkWriter *OutboundLinkWriter
 }
 
 func (h *httpReader) Read(p []byte) (int, error) {
 	var msg httpReaderDataMsg
+
 	ok := true
 	for ok && len(h.data) == 0 {
 		msg, ok = <-h.msgQueue
 		h.data = msg.bytes
+
 		h.captureTime = msg.timestamp
+		if len(h.data) > 0 {
+			h.packetsSeen += 1
+		}
+		if h.packetsSeen < checkTLSPacketAmount && len(msg.bytes) > 5 { // packets with less than 5 bytes cause tlsx to panic
+			clientHello := tlsx.ClientHello{}
+			err := clientHello.Unmarshall(msg.bytes)
+			if err == nil {
+				fmt.Printf("Detected TLS client hello with SNI %s\n", clientHello.SNI)
+				numericPort, _ := strconv.Atoi(h.tcpID.dstPort)
+				h.outboundLinkWriter.WriteOutboundLink(h.tcpID.srcIP, h.tcpID.dstIP, numericPort, clientHello.SNI, TLSProtocol)
+			}
+		}
 	}
 	if !ok || len(h.data) == 0 {
 		return 0, io.EOF

tap/outboundlinks.go

@@ -1,9 +1,17 @@
 package tap
 
+type OutboundLinkProtocol string
+
+const (
+	TLSProtocol OutboundLinkProtocol = "tls"
+)
+
 type OutboundLink struct {
 	Src     string
 	DstIP   string
 	DstPort int
+	SuggestedResolvedName string
+	SuggestedProtocol OutboundLinkProtocol
 }
 
 func NewOutboundLinkWriter() *OutboundLinkWriter {
@@ -16,11 +24,13 @@ type OutboundLinkWriter struct {
 	OutChan chan *OutboundLink
 }
 
-func (olw *OutboundLinkWriter) WriteOutboundLink(src string, DstIP string, DstPort int) {
+func (olw *OutboundLinkWriter) WriteOutboundLink(src string, DstIP string, DstPort int, SuggestedResolvedName string, SuggestedProtocol OutboundLinkProtocol) {
 	olw.OutChan <- &OutboundLink{
 		Src: src,
 		DstIP: DstIP,
 		DstPort: DstPort,
+		SuggestedResolvedName: SuggestedResolvedName,
+		SuggestedProtocol: SuggestedProtocol,
 	}
 }
 

tap/passive_tapper.go

@@ -35,7 +35,8 @@ const AppPortsEnvVar = "APP_PORTS"
 const maxHTTP2DataLenEnvVar = "HTTP2_DATA_SIZE_LIMIT"
 const maxHTTP2DataLenDefault = 1 * 1024 * 1024 // 1MB
 const cleanPeriod = time.Second * 10
-var remoteOnlyOutboundPorts = []int { 80, 443 }
+
+var remoteOnlyOutboundPorts = []int{80, 443}
 
 func parseAppPorts(appPortsList string) []int {
 	ports := make([]int, 0)
@@ -50,15 +51,15 @@ func parseAppPorts(appPortsList string) []int {
 	return ports
 }
 
-var maxcount = flag.Int("c", -1, "Only grab this many packets, then exit")
+var maxcount = flag.Int64("c", -1, "Only grab this many packets, then exit")
 var decoder = flag.String("decoder", "", "Name of the decoder to use (default: guess from capture)")
 var statsevery = flag.Int("stats", 60, "Output statistics every N seconds")
 var lazy = flag.Bool("lazy", false, "If true, do lazy decoding")
 var nodefrag = flag.Bool("nodefrag", false, "If true, do not do IPv4 defrag")
-var checksum = flag.Bool("checksum", false, "Check TCP checksum")  // global
-var nooptcheck = flag.Bool("nooptcheck", true, "Do not check TCP options (useful to ignore MSS on captures with TSO)")  // global
-var ignorefsmerr = flag.Bool("ignorefsmerr", true, "Ignore TCP FSM errors")  // global
-var allowmissinginit = flag.Bool("allowmissinginit", true, "Support streams without SYN/SYN+ACK/ACK sequence")  // global
+var checksum = flag.Bool("checksum", false, "Check TCP checksum")                                                      // global
+var nooptcheck = flag.Bool("nooptcheck", true, "Do not check TCP options (useful to ignore MSS on captures with TSO)") // global
+var ignorefsmerr = flag.Bool("ignorefsmerr", true, "Ignore TCP FSM errors")                                            // global
+var allowmissinginit = flag.Bool("allowmissinginit", true, "Support streams without SYN/SYN+ACK/ACK sequence")         // global
 var verbose = flag.Bool("verbose", false, "Be verbose")
 var debug = flag.Bool("debug", false, "Display debug information")
 var quiet = flag.Bool("quiet", false, "Be quiet regarding errors")
@@ -68,7 +69,7 @@ var nohttp = flag.Bool("nohttp", false, "Disable HTTP parsing")
 var output = flag.String("output", "", "Path to create file for HTTP 200 OK responses")
 var writeincomplete = flag.Bool("writeincomplete", false, "Write incomplete response")
 
-var hexdump = flag.Bool("dump", false, "Dump HTTP request/response as hex")  // global
+var hexdump = flag.Bool("dump", false, "Dump HTTP request/response as hex") // global
 var hexdumppkt = flag.Bool("dumppkt", false, "Dump packet as hex")
 
 // capture
@@ -83,11 +84,10 @@ var staleTimeoutSeconds = flag.Int("staletimout", 120, "Max time in seconds to k
 var memprofile = flag.String("memprofile", "", "Write memory profile")
 
 // output
-var dumpToHar = flag.Bool("hardump", false, "Dump traffic to har files")
 var HarOutputDir = flag.String("hardir", "", "Directory in which to store output har files")
 var harEntriesPerFile = flag.Int("harentriesperfile", 200, "Number of max number of har entries to store in each file")
 
-var reqResMatcher = createResponseRequestMatcher()  // global
+var reqResMatcher = createResponseRequestMatcher() // global
 var statsTracker = StatsTracker{}
 
 // global
@@ -117,8 +117,8 @@ var outputLevel int
 var errorsMap map[string]uint
 var errorsMapMutex sync.Mutex
 var nErrors uint
-var ownIps []string           // global
-var hostMode bool             // global
+var ownIps []string // global
+var hostMode bool   // global
 
 /* minOutputLevel: Error will be printed only if outputLevel is above this value
  * t:              key for errorsMap (counting errors)
@@ -174,6 +174,10 @@ type Context struct {
 	CaptureInfo gopacket.CaptureInfo
 }
 
+func GetStats() AppStats {
+	return statsTracker.appStats
+}
+
 func (c *Context) GetCaptureInfo() gopacket.CaptureInfo {
 	return c.CaptureInfo
 }
@@ -181,19 +185,43 @@ func (c *Context) GetCaptureInfo() gopacket.CaptureInfo {
 func StartPassiveTapper(opts *TapOpts) (<-chan *OutputChannelItem, <-chan *OutboundLink) {
 	hostMode = opts.HostMode
 
-	var harWriter *HarWriter
-	if *dumpToHar {
-		harWriter = NewHarWriter(*HarOutputDir, *harEntriesPerFile)
-	}
+	harWriter := NewHarWriter(*HarOutputDir, *harEntriesPerFile)
 	outboundLinkWriter := NewOutboundLinkWriter()
 
 	go startPassiveTapper(harWriter, outboundLinkWriter)
 
-	if harWriter != nil {
-		return harWriter.OutChan, outboundLinkWriter.OutChan
-	}
+	return harWriter.OutChan, outboundLinkWriter.OutChan
+}
 
-	return nil, outboundLinkWriter.OutChan
+func startMemoryProfiler() {
+	dirname := "/app/pprof"
+	rlog.Info("Profiling is on, results will be written to %s", dirname)
+	go func() {
+		if _, err := os.Stat(dirname); os.IsNotExist(err) {
+			if err := os.Mkdir(dirname, 0777); err != nil {
+				log.Fatal("could not create directory for profile: ", err)
+			}
+		}
+
+		for true {
+			t := time.Now()
+
+			filename := fmt.Sprintf("%s/%s__mem.prof", dirname, t.Format("15_04_05"))
+
+			rlog.Info("Writing memory profile to %s\n", filename)
+
+			f, err := os.Create(filename)
+			if err != nil {
+				log.Fatal("could not create memory profile: ", err)
+			}
+			runtime.GC() // get up-to-date statistics
+			if err := pprof.WriteHeapProfile(f); err != nil {
+				log.Fatal("could not write memory profile: ", err)
+			}
+			_ = f.Close()
+			time.Sleep(time.Minute)
+		}
+	}()
 }
 
 func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWriter) {
@@ -285,10 +313,8 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 		}
 	}
 
-	if *dumpToHar {
-		harWriter.Start()
-		defer harWriter.Stop()
-	}
+	harWriter.Start()
+	defer harWriter.Stop()
 	defer outboundLinkWriter.Stop()
 
 	var dec gopacket.Decoder
@@ -304,19 +330,23 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 	source.Lazy = *lazy
 	source.NoCopy = true
 	rlog.Info("Starting to read packets")
-	count := 0
-	bytes := int64(0)
-	start := time.Now()
+	statsTracker.setStartTime(time.Now())
 	defragger := ip4defrag.NewIPv4Defragmenter()
 
 	streamFactory := &tcpStreamFactory{
-		doHTTP: !*nohttp,
-		harWriter: harWriter,
+		doHTTP:             !*nohttp,
+		harWriter:          harWriter,
 		outbountLinkWriter: outboundLinkWriter,
-
 	}
 	streamPool := reassembly.NewStreamPool(streamFactory)
 	assembler := reassembly.NewAssembler(streamPool)
+
+	maxBufferedPagesTotal := GetMaxBufferedPagesPerConnection()
+	maxBufferedPagesPerConnection := GetMaxBufferedPagesTotal()
+	rlog.Infof("Assembler options: maxBufferedPagesTotal=%d, maxBufferedPagesPerConnection=%d", maxBufferedPagesTotal, maxBufferedPagesPerConnection)
+	assembler.AssemblerOptions.MaxBufferedPagesTotal = maxBufferedPagesTotal
+	assembler.AssemblerOptions.MaxBufferedPagesPerConnection = maxBufferedPagesPerConnection
+
 	var assemblerMutex sync.Mutex
 
 	signalChan := make(chan os.Signal, 1)
@@ -324,10 +354,10 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 
 	staleConnectionTimeout := time.Second * time.Duration(*staleTimeoutSeconds)
 	cleaner := Cleaner{
-		assembler: assembler,
-		assemblerMutex: &assemblerMutex,
-		matcher: &reqResMatcher,
-		cleanPeriod: cleanPeriod,
+		assembler:         assembler,
+		assemblerMutex:    &assemblerMutex,
+		matcher:           &reqResMatcher,
+		cleanPeriod:       cleanPeriod,
 		connectionTimeout: staleConnectionTimeout,
 	}
 	cleaner.start()
@@ -345,9 +375,9 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 			errorsSummery := fmt.Sprintf("%v", errorsMap)
 			errorsMapMutex.Unlock()
 			log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v) - Errors Summary: %s",
-				count,
-				bytes,
-				time.Since(start),
+				statsTracker.appStats.TotalPacketsCount,
+				statsTracker.appStats.TotalProcessedBytes,
+				time.Since(statsTracker.appStats.StartTime),
 				nErrors,
 				errorMapLen,
 				errorsSummery,
@@ -365,22 +395,26 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 
 			// Since the last print
 			cleanStats := cleaner.dumpStats()
-			appStats := statsTracker.dumpStats()
+			matchedMessages := statsTracker.dumpStats()
 			log.Printf(
 				"flushed connections %d, closed connections: %d, deleted messages: %d, matched messages: %d",
 				cleanStats.flushed,
 				cleanStats.closed,
 				cleanStats.deleted,
-				appStats.matchedMessages,
+				matchedMessages,
 			)
 		}
 	}()
 
+	if GetMemoryProfilingEnabled() {
+		startMemoryProfiler()
+	}
+
 	for packet := range source.Packets() {
-		count++
-		rlog.Debugf("PACKET #%d", count)
+		packetsCount := statsTracker.incPacketsCount()
+		rlog.Debugf("PACKET #%d", packetsCount)
 		data := packet.Data()
-		bytes += int64(len(data))
+		statsTracker.updateProcessedSize(int64(len(data)))
 		if *hexdumppkt {
 			rlog.Debugf("Packet content (%d/0x%x) - %s", len(data), len(data), hex.Dump(data))
 		}
@@ -431,12 +465,17 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 			assemblerMutex.Unlock()
 		}
 
-		done := *maxcount > 0 && count >= *maxcount
+		done := *maxcount > 0 && statsTracker.appStats.TotalPacketsCount >= *maxcount
 		if done {
 			errorsMapMutex.Lock()
 			errorMapLen := len(errorsMap)
 			errorsMapMutex.Unlock()
-			log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)", count, bytes, time.Since(start), nErrors, errorMapLen)
+			log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)",
+				statsTracker.appStats.TotalPacketsCount,
+				statsTracker.appStats.TotalProcessedBytes,
+				time.Since(statsTracker.appStats.StartTime),
+				nErrors,
+				errorMapLen)
 		}
 		select {
 		case <-signalChan:
@@ -493,4 +532,5 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 	for e := range errorsMap {
 		log.Printf(" %s:\t\t%d", e, errorsMap[e])
 	}
+	log.Printf("AppStats: %v", GetStats())
 }

tap/settings.go

@@ -1,5 +1,18 @@
 package tap
 
+import (
+	"os"
+	"strconv"
+)
+
+const (
+	MemoryProfilingEnabledEnvVarName          = "MEMORY_PROFILING_ENABLED"
+	MaxBufferedPagesTotalEnvVarName           = "MAX_BUFFERED_PAGES_TOTAL"
+	MaxBufferedPagesPerConnectionEnvVarName   = "MAX_BUFFERED_PAGES_PER_CONNECTION"
+	MaxBufferedPagesTotalDefaultValue         = 5000
+	MaxBufferedPagesPerConnectionDefaultValue = 5000
+)
+
 type globalSettings struct {
 	filterPorts       []int
 	filterAuthorities []string
@@ -29,3 +42,23 @@ func GetFilterIPs() []string {
 	copy(addresses, gSettings.filterAuthorities)
 	return addresses
 }
+
+func GetMaxBufferedPagesTotal() int {
+	valueFromEnv, err := strconv.Atoi(os.Getenv(MaxBufferedPagesTotalEnvVarName))
+	if err != nil {
+		return MaxBufferedPagesTotalDefaultValue
+	}
+	return valueFromEnv
+}
+
+func GetMaxBufferedPagesPerConnection() int {
+	valueFromEnv, err := strconv.Atoi(os.Getenv(MaxBufferedPagesPerConnectionEnvVarName))
+	if err != nil {
+		return MaxBufferedPagesPerConnectionDefaultValue
+	}
+	return valueFromEnv
+}
+
+func GetMemoryProfilingEnabled() bool {
+	return os.Getenv(MemoryProfilingEnabledEnvVarName) == "1"
+}

tap/stats_tracker.go

@@ -2,34 +2,54 @@ package tap
 
 import (
 	"sync"
+	"time"
 )
 
 type AppStats struct {
-	matchedMessages int
+	StartTime            time.Time `json:"startTime"`
+	MatchedMessages      int       `json:"matchedMessages"`
+	TotalPacketsCount    int64     `json:"totalPacketsCount"`
+	TotalProcessedBytes  int64     `json:"totalProcessedBytes"`
+	TotalMatchedMessages int64     `json:"totalMatchedMessages"`
 }
 
 type StatsTracker struct {
-	stats             AppStats
-	statsMutex	  sync.Mutex
+	appStats                AppStats
+	matchedMessagesMutex    sync.Mutex
+	totalPacketsCountMutex  sync.Mutex
+	totalProcessedSizeMutex sync.Mutex
 }
 
 func (st *StatsTracker) incMatchedMessages() {
-	st.statsMutex.Lock()
-	st.stats.matchedMessages++
-	st.statsMutex.Unlock()
+	st.matchedMessagesMutex.Lock()
+	st.appStats.MatchedMessages++
+	st.appStats.TotalMatchedMessages++
+	st.matchedMessagesMutex.Unlock()
 }
 
-func (st *StatsTracker) dumpStats() AppStats {
-	st.statsMutex.Lock()
-
-	stats := AppStats{
-		matchedMessages: st.stats.matchedMessages,
-	}
-
-	st.stats.matchedMessages = 0
-
-	st.statsMutex.Unlock()
-
-	return stats
+func (st *StatsTracker) incPacketsCount() int64 {
+	st.totalPacketsCountMutex.Lock()
+	st.appStats.TotalPacketsCount++
+	currentPacketsCount := st.appStats.TotalPacketsCount
+	st.totalPacketsCountMutex.Unlock()
+	return currentPacketsCount
 }
 
+func (st *StatsTracker) updateProcessedSize(size int64) {
+	st.totalProcessedSizeMutex.Lock()
+	st.appStats.TotalProcessedBytes += size
+	st.totalProcessedSizeMutex.Unlock()
+}
+
+func (st *StatsTracker) setStartTime(startTime time.Time) {
+	st.appStats.StartTime = startTime
+}
+
+func (st *StatsTracker) dumpStats() int {
+	st.matchedMessagesMutex.Lock()
+	matchedMessages := st.appStats.MatchedMessages
+	st.appStats.MatchedMessages = 0
+	st.matchedMessagesMutex.Unlock()
+
+	return matchedMessages
+}

tap/tcp_stream_factory.go

@@ -33,7 +33,7 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
 	dstPort := int(tcp.DstPort)
 
 	if factory.shouldNotifyOnOutboundLink(dstIp, dstPort) {
-		factory.outbountLinkWriter.WriteOutboundLink(net.Src().String(), dstIp, dstPort)
+		factory.outbountLinkWriter.WriteOutboundLink(net.Src().String(), dstIp, dstPort, "", "")
 	}
 	props := factory.getStreamProps(srcIp, dstIp, dstPort)
 	isHTTP := props.isTapTarget
@@ -57,11 +57,12 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
 				srcPort: transport.Src().String(),
 				dstPort: transport.Dst().String(),
 			},
-			hexdump:  *hexdump,
-			parent:   stream,
-			isClient: true,
-			isOutgoing: props.isOutgoing,
-			harWriter: factory.harWriter,
+			hexdump:            *hexdump,
+			parent:             stream,
+			isClient:           true,
+			isOutgoing:         props.isOutgoing,
+			harWriter:          factory.harWriter,
+			outboundLinkWriter: factory.outbountLinkWriter,
 		}
 		stream.server = httpReader{
 			msgQueue: make(chan httpReaderDataMsg),
@@ -72,10 +73,11 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
 				srcPort: transport.Dst().String(),
 				dstPort: transport.Src().String(),
 			},
-			hexdump: *hexdump,
-			parent:  stream,
-			isOutgoing: props.isOutgoing,
-			harWriter: factory.harWriter,
+			hexdump:            *hexdump,
+			parent:             stream,
+			isOutgoing:         props.isOutgoing,
+			harWriter:          factory.harWriter,
+			outboundLinkWriter: factory.outbountLinkWriter,
 		}
 		factory.wg.Add(2)
 		// Start reading from channels stream.client.bytes and stream.server.bytes
@@ -131,6 +133,5 @@ func (factory *tcpStreamFactory) shouldNotifyOnOutboundLink(dstIP string, dstPor
 
 type streamProps struct {
 	isTapTarget bool
-	isOutgoing bool
+	isOutgoing  bool
 }
-

ui/package.json

@@ -4,6 +4,7 @@
   "private": true,
   "dependencies": {
     "@material-ui/core": "^4.11.3",
+    "@material-ui/lab": "^4.0.0-alpha.60",
     "@testing-library/jest-dom": "^5.11.10",
     "@testing-library/react": "^11.2.6",
     "@testing-library/user-event": "^12.8.3",
@@ -11,6 +12,7 @@
     "@types/node": "^12.20.10",
     "@types/react": "^17.0.3",
     "@types/react-dom": "^17.0.3",
+    "jsonpath": "^1.1.1",
     "axios": "^0.21.1",
     "node-sass": "^5.0.0",
     "numeral": "^2.0.6",