added custom config path option (#247)

change prerequisite to permissions and kubeconfig. These are more FYIs as Mizu requires very little prerequisites. 
Change the description to match getmizu.io

.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# This is a comment.
+# Each line is a file pattern followed by one or more owners.
+
+/ui/ @frontend

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Run mizu <command> '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Logs**
+Upload logs:
+1. Run the mizu command with `--set dump-logs=true` (e.g `mizu tap --set dump-logs=true`) 
+2. Try to reproduce the issue
+3. CNTRL+C on terminal tab which runs mizu
+4. Upload the logs zip file from ~/.mizu/mizu_logs_**.zip
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome]
+
+**Additional context**
+Add any other context about the problem here.

.github/workflows/acceptance_tests.yml

@@ -0,0 +1,32 @@
+name: acceptance tests
+
+on:
+  pull_request:
+    branches:
+      - 'main'
+  push:
+    branches:
+      - 'develop'
+
+concurrency:
+  group: mizu-acceptance-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-acceptance-tests:
+    name: Run acceptance tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Setup acceptance test
+        run: source ./acceptanceTests/setup.sh
+
+      - name: Test
+        run: make acceptance-test

.github/workflows/pr_validation.yml

@@ -0,0 +1,46 @@
+name: PR validation
+
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+
+concurrency:
+  group: mizu-pr-validation-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-cli:
+    name: Build CLI
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Build CLI
+        run: make cli
+
+  build-agent:
+    name: Build Agent
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - shell: bash
+        run: |
+          sudo apt-get install libpcap-dev
+
+      - name: Build Agent
+        run: make agent

.github/workflows/publish.yml

@@ -1,9 +1,15 @@
 name: publish
+
 on:
   push:
     branches:
       - 'develop'
       - 'main'
+
+concurrency:
+  group: mizu-publish-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   docker:
     runs-on: ubuntu-latest
@@ -78,4 +84,3 @@ jobs:
           tag: ${{ steps.versioning.outputs.version }}
           prerelease: ${{ github.ref != 'refs/heads/main' }}
           bodyFile: 'cli/bin/README.md'
-

.github/workflows/tests_validation.yml

@@ -0,0 +1,56 @@
+name: tests validation
+
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+  push:
+    branches:
+      - 'develop'
+      - 'main'
+
+concurrency:
+  group: mizu-tests-validation-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-tests-cli:
+    name: Run CLI tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: make test-cli
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2
+
+  run-tests-agent:
+    name: Run Agent tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - shell: bash
+        run: |
+          sudo apt-get install libpcap-dev
+
+      - name: Test
+        run: make test-agent
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2

CONTRIBUTE.md

@@ -0,0 +1,18 @@
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+# CONTRIBUTE
+We welcome code contributions from the community.
+Please read and follow the guidelines below.
+
+## Communication
+* Before starting work on a major feature, please reach out to us via [GitHub](https://github.com/up9inc/mizu), [Slack](https://join.slack.com/share/zt-u6bbs3pg-X1zhQOXOH0yEoqILgH~csw), [email](mailto:mizu@up9.com), etc. We will make sure no one else is already working on it. A _major feature_ is defined as any change that is > 100 LOC altered (not including tests), or changes any user-facing behavior
+* Small patches and bug fixes don't need prior communication.
+
+## Contribution requirements
+* Code style - most of the code is written in Go, please follow [these guidelines](https://golang.org/doc/effective_go)
+* Go-tools compatible (`go get`, `go test`, etc)
+* Unit-test coverage can’t go down ..
+* Code must be usefully commented. Not only for developers on the project, but also for external users of these packages
+* When reviewing PRs, you are encouraged to use Golang's [code review comments page](https://github.com/golang/go/wiki/CodeReviewComments)
+
+
+

Makefile

@@ -28,6 +28,9 @@ ui: ## Build UI.
 cli: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build
 
+build-cli-ci: ## Build CLI for CI.
+	@echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci
+
 agent: ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
@@ -42,6 +45,10 @@ push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
 	./build-push-featurebranch.sh
 
+build-docker-ci: ## Build agent docker image for CI.
+	@echo "building docker image for ci"
+	./build-agent-ci.sh
+
 push-cli: ## Build and publish CLI.
 	@echo "publishing CLI .. "
 	@cd cli; $(MAKE) build-all
@@ -50,7 +57,6 @@ push-cli: ## Build and publish CLI.
 	gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
-
 clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
 
 clean-ui: ## Clean UI.
@@ -65,3 +71,11 @@ clean-cli:  ## Clean CLI.
 clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
+test-cli:
+	@echo "running cli tests"; cd cli && $(MAKE) test
+
+test-agent:
+	@echo "running agent tests"; cd agent && $(MAKE) test
+
+acceptance-test:
+	@echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test

README.md

@@ -1,16 +1,19 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+
 # The API Traffic Viewer for Kubernetes
 
-A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined.
+A simple-yet-powerful API traffic viewer for Kubernetes enabling you to view all API communication between microservices to help your debug and troubleshoot regressions.
+
+Think TCPDump and Chrome Dev Tools combined.
 
 ![Simple UI](assets/mizu-ui.png)
 
 ## Features
 
 - Simple and powerful CLI
-- Real time view of all HTTP requests, REST and gRPC API calls
+- Real-time view of all HTTP requests, REST and gRPC API calls
 - No installation or code instrumentation
-- Works completely on premises (on-prem)
+- Works completely on premises
 
 ## Download
 
@@ -32,13 +35,15 @@ https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 \
 && chmod 755 mizu
 ``` 
 
-SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page.
+SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page
 
 ### Development (unstable) Build
-Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page.
+Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page
 
-## Prerequisites
-1. Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
+## Kubeconfig & Permissions
+While `mizu`most often works out of the box, you can influence its behavior:
+
+1. [OPTIONAL] Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
 2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination)
 
 For detailed list of k8s permissions see [PERMISSIONS](PERMISSIONS.md) document
@@ -48,8 +53,8 @@ For detailed list of k8s permissions see [PERMISSIONS](PERMISSIONS.md) document
 
 1. Find pods you'd like to tap to in your Kubernetes cluster
 2. Run `mizu tap` or `mizu tap PODNAME`  
-3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI .. 
-4. Watch the API traffic flowing ..
+3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI
+4. Watch the API traffic flowing
 5. Type ^C to stop
 
 ## Examples
@@ -106,44 +111,64 @@ To tap multiple pods using regex -
 ## Configuration
 
 Mizu can work with config file which should be stored in ${HOME}/.mizu/config.yaml (macOS: ~/.mizu/config.yaml) <br />
-In case no config file found, defaults will be used. <br />
-In case of partial configuration defined, all other fields will be used with defaults. <br />
-You can always override the defaults or config file with CLI flags.
+In case no config file found, defaults will be used <br />
+In case of partial configuration defined, all other fields will be used with defaults <br />
+You can always override the defaults or config file with CLI flags
 
 To get the default config params run `mizu config` <br />
 To generate a new config file with default values use `mizu config -r`
 
-Mizu has several undocumented flags which can be set by using --set flag (e.g., `mizu tap --set dump-logs=true`)
-* **mizu-resources-namespace**: Type - String, See [Namespace-Restricted Mode](#namespace-restricted-mode)
-* **telemetry**: Type - Boolean, Reports telemetry
-* **dump-logs**: Type - Boolean, At the end of the execution it creates a zip file with logs (in .mizu folder)
-* **kube-config-path**: Type - String, Setting the path to kube config (which isn't in standard path)
+### Telemetry
+
+By default, mizu reports usage telemetry. It can be disabled by adding a line of `telemetry: false` in the `${HOME}/.mizu/config.yaml` file
+
 
 ## Advanced Usage
 
 ### Namespace-Restricted Mode
 
-Some users have permission to only manage resources in one particular namespace assigned to them.
+Some users have permission to only manage resources in one particular namespace assigned to them
 By default `mizu tap` creates a new namespace `mizu` for all of its Kubernetes resources. In order to instead install
-Mizu in an existing namespace, set the `mizu-resources-namespace` config option.
+Mizu in an existing namespace, set the `mizu-resources-namespace` config option
 
 If `mizu-resources-namespace` is set to a value other than the default `mizu`, Mizu will operate in a
 Namespace-Restricted mode. It will only tap pods in `mizu-resources-namespace`. This way Mizu only requires permissions
 to the namespace set by `mizu-resources-namespace`. The user must set the tapped namespace to the same namespace by
-using the `--namespace` flag or by setting `tap.namespaces` in the config file.
+using the `--namespace` flag or by setting `tap.namespaces` in the config file
 
-Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior.
+Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
 
 ### User agent filtering
 
-User-agent filtering (like health checks) - can be configured:
+User-agent filtering (like health checks) - can be configured using command-line options:
 
-Any request that contains one of those values in the user-agent header will not be captured
-
-```bash
+```shell
 $ mizu tap "^ca.*" --set ignored-user-agents=kube-probe --set ignored-user-agents=prometheus
 +carts-66c77f5fbb-fq65r
 +catalogue-5f4cb7cf5-7zrmn
 Web interface is now available at http://localhost:8899
 ^C
-```
+
+```
+
+Any request that contains `User-Agent` header with one of the specified values (`kube-probe` or `prometheus`) will not be captured
+
+### API Rules validation
+
+This feature allows you to define set of simple rules, and test the API against them.
+Such validation may test response for specific JSON fields, headers, etc.
+
+Please see [API RULES](docs/POLICY_RULES.md) page for more details and syntax.
+
+
+## How to Run local UI
+
+- run from mizu/agent `go run main.go --hars-read --hars-dir <folder>`
+
+- copy Har files into the folder from last command
+
+- change `MizuWebsocketURL` and `apiURL` in `api.js` file
+
+- run from mizu/ui - `npm run start`
+
+- open browser on `localhost:3000`

TESTING.md

@@ -0,0 +1,15 @@
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+# TESTING
+Testing guidelines for Mizu project 
+
+## Unit-tests
+* TBD 
+* TBD
+* TBD
+
+
+
+## System tests
+* TBD 
+* TBD
+* TBD

acceptanceTests/Makefile

@@ -0,0 +1,2 @@
+test: ## Run acceptance tests.
+	@go test ./... -timeout 1h

acceptanceTests/config_test.go

@@ -0,0 +1,283 @@
+package acceptanceTests
+
+import (
+	"fmt"
+	"gopkg.in/yaml.v3"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"testing"
+)
+
+type tapConfig struct {
+	GuiPort uint16 `yaml:"gui-port"`
+}
+
+type configStruct struct {
+	Tap tapConfig `yaml:"tap"`
+}
+
+func TestConfigRegenerate(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	configPath, configPathErr := getConfigPath()
+	if configPathErr != nil {
+		t.Errorf("failed to get config path, err: %v", cliPathErr)
+		return
+	}
+
+	configCmdArgs := getDefaultConfigCommandArgs()
+
+	configCmdArgs = append(configCmdArgs, "-r")
+
+	configCmd := exec.Command(cliPath, configCmdArgs...)
+	t.Logf("running command: %v", configCmd.String())
+
+	t.Cleanup(func() {
+		if err := os.Remove(configPath); err != nil {
+			t.Logf("failed to delete config file, err: %v", err)
+		}
+	})
+
+	if err := configCmd.Start(); err != nil {
+		t.Errorf("failed to start config command, err: %v", err)
+		return
+	}
+
+	if err := configCmd.Wait(); err != nil {
+		t.Errorf("failed to wait config command, err: %v", err)
+		return
+	}
+
+	_, readFileErr := ioutil.ReadFile(configPath)
+	if readFileErr != nil {
+		t.Errorf("failed to read config file, err: %v", readFileErr)
+		return
+	}
+}
+
+func TestConfigGuiPort(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []uint16{8898}
+
+	for _, guiPort := range tests {
+		t.Run(fmt.Sprintf("%d", guiPort), func(t *testing.T) {
+			cliPath, cliPathErr := getCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			configPath, configPathErr := getConfigPath()
+			if configPathErr != nil {
+				t.Errorf("failed to get config path, err: %v", cliPathErr)
+				return
+			}
+
+			config := configStruct{}
+			config.Tap.GuiPort = guiPort
+
+			configBytes, marshalErr := yaml.Marshal(config)
+			if marshalErr != nil {
+				t.Errorf("failed to marshal config, err: %v", marshalErr)
+				return
+			}
+
+			if writeErr := ioutil.WriteFile(configPath, configBytes, 0644); writeErr != nil {
+				t.Errorf("failed to write config to file, err: %v", writeErr)
+				return
+			}
+
+			tapCmdArgs := getDefaultTapCommandArgs()
+
+			tapNamespace := getDefaultTapNamespace()
+			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+			tapCmd := exec.Command(cliPath, tapCmdArgs...)
+			t.Logf("running command: %v", tapCmd.String())
+
+			t.Cleanup(func() {
+				if err := cleanupCommand(tapCmd); err != nil {
+					t.Logf("failed to cleanup tap command, err: %v", err)
+				}
+
+				if err := os.Remove(configPath); err != nil {
+					t.Logf("failed to delete config file, err: %v", err)
+				}
+			})
+
+			if err := tapCmd.Start(); err != nil {
+				t.Errorf("failed to start tap command, err: %v", err)
+				return
+			}
+
+			apiServerUrl := getApiServerUrl(guiPort)
+
+			if err := waitTapPodsReady(apiServerUrl); err != nil {
+				t.Errorf("failed to start tap pods on time, err: %v", err)
+				return
+			}
+		})
+	}
+}
+
+func TestConfigSetGuiPort(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []struct {
+		ConfigFileGuiPort uint16
+		SetGuiPort        uint16
+	}{
+		{ConfigFileGuiPort: 8898, SetGuiPort: 8897},
+	}
+
+	for _, guiPortStruct := range tests {
+		t.Run(fmt.Sprintf("%d", guiPortStruct.SetGuiPort), func(t *testing.T) {
+			cliPath, cliPathErr := getCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			configPath, configPathErr := getConfigPath()
+			if configPathErr != nil {
+				t.Errorf("failed to get config path, err: %v", cliPathErr)
+				return
+			}
+
+			config := configStruct{}
+			config.Tap.GuiPort = guiPortStruct.ConfigFileGuiPort
+
+			configBytes, marshalErr := yaml.Marshal(config)
+			if marshalErr != nil {
+				t.Errorf("failed to marshal config, err: %v", marshalErr)
+				return
+			}
+
+			if writeErr := ioutil.WriteFile(configPath, configBytes, 0644); writeErr != nil {
+				t.Errorf("failed to write config to file, err: %v", writeErr)
+				return
+			}
+
+			tapCmdArgs := getDefaultTapCommandArgs()
+
+			tapNamespace := getDefaultTapNamespace()
+			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+			tapCmdArgs = append(tapCmdArgs, "--set", fmt.Sprintf("tap.gui-port=%v", guiPortStruct.SetGuiPort))
+
+			tapCmd := exec.Command(cliPath, tapCmdArgs...)
+			t.Logf("running command: %v", tapCmd.String())
+
+			t.Cleanup(func() {
+				if err := cleanupCommand(tapCmd); err != nil {
+					t.Logf("failed to cleanup tap command, err: %v", err)
+				}
+
+				if err := os.Remove(configPath); err != nil {
+					t.Logf("failed to delete config file, err: %v", err)
+				}
+			})
+
+			if err := tapCmd.Start(); err != nil {
+				t.Errorf("failed to start tap command, err: %v", err)
+				return
+			}
+
+			apiServerUrl := getApiServerUrl(guiPortStruct.SetGuiPort)
+
+			if err := waitTapPodsReady(apiServerUrl); err != nil {
+				t.Errorf("failed to start tap pods on time, err: %v", err)
+				return
+			}
+		})
+	}
+}
+
+func TestConfigFlagGuiPort(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []struct {
+		ConfigFileGuiPort uint16
+		FlagGuiPort       uint16
+	}{
+		{ConfigFileGuiPort: 8898, FlagGuiPort: 8896},
+	}
+
+	for _, guiPortStruct := range tests {
+		t.Run(fmt.Sprintf("%d", guiPortStruct.FlagGuiPort), func(t *testing.T) {
+			cliPath, cliPathErr := getCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			configPath, configPathErr := getConfigPath()
+			if configPathErr != nil {
+				t.Errorf("failed to get config path, err: %v", cliPathErr)
+				return
+			}
+
+			config := configStruct{}
+			config.Tap.GuiPort = guiPortStruct.ConfigFileGuiPort
+
+			configBytes, marshalErr := yaml.Marshal(config)
+			if marshalErr != nil {
+				t.Errorf("failed to marshal config, err: %v", marshalErr)
+				return
+			}
+
+			if writeErr := ioutil.WriteFile(configPath, configBytes, 0644); writeErr != nil {
+				t.Errorf("failed to write config to file, err: %v", writeErr)
+				return
+			}
+
+			tapCmdArgs := getDefaultTapCommandArgs()
+
+			tapNamespace := getDefaultTapNamespace()
+			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+			tapCmdArgs = append(tapCmdArgs, "-p", fmt.Sprintf("%v", guiPortStruct.FlagGuiPort))
+
+			tapCmd := exec.Command(cliPath, tapCmdArgs...)
+			t.Logf("running command: %v", tapCmd.String())
+
+			t.Cleanup(func() {
+				if err := cleanupCommand(tapCmd); err != nil {
+					t.Logf("failed to cleanup tap command, err: %v", err)
+				}
+
+				if err := os.Remove(configPath); err != nil {
+					t.Logf("failed to delete config file, err: %v", err)
+				}
+			})
+
+			if err := tapCmd.Start(); err != nil {
+				t.Errorf("failed to start tap command, err: %v", err)
+				return
+			}
+
+			apiServerUrl := getApiServerUrl(guiPortStruct.FlagGuiPort)
+
+			if err := waitTapPodsReady(apiServerUrl); err != nil {
+				t.Errorf("failed to start tap pods on time, err: %v", err)
+				return
+			}
+		})
+	}
+}

acceptanceTests/go.mod

@@ -0,0 +1,5 @@
+module github.com/up9inc/mizu/tests
+
+go 1.16
+
+require gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b

acceptanceTests/go.sum

@@ -0,0 +1,4 @@
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

acceptanceTests/setup.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+PREFIX=$HOME/local/bin
+VERSION=v1.22.0
+
+echo "Attempting to install minikube and assorted tools to $PREFIX"
+
+if ! [ -x "$(command -v kubectl)" ]; then
+  echo "Installing kubectl version $VERSION"
+  curl -LO "https://storage.googleapis.com/kubernetes-release/release/$VERSION/bin/linux/amd64/kubectl"
+  chmod +x kubectl
+  mv kubectl "$PREFIX"
+else
+  echo "kubetcl is already installed"
+fi
+
+if ! [ -x "$(command -v minikube)" ]; then
+  echo "Installing minikube version $VERSION"
+  curl -Lo minikube https://storage.googleapis.com/minikube/releases/$VERSION/minikube-linux-amd64
+  chmod +x minikube
+  mv minikube "$PREFIX"
+else
+  echo "minikube is already installed"
+fi
+
+echo "Starting minikube..."
+minikube start
+
+echo "Creating mizu tests namespaces"
+kubectl create namespace mizu-tests
+kubectl create namespace mizu-tests2
+
+echo "Creating httpbin deployments"
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests
+kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests
+
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2
+
+echo "Creating httpbin services"
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
+kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests
+
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2
+
+echo "Starting proxy"
+kubectl proxy --port=8080 &
+
+echo "Setting minikube docker env"
+eval $(minikube docker-env)
+
+echo "Build agent image"
+make build-docker-ci
+
+echo "Build cli"
+make build-cli-ci

acceptanceTests/tap_test.go

@@ -0,0 +1,754 @@
+package acceptanceTests
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestTapAndFetch(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []int{50}
+
+	for _, entriesCount := range tests {
+		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
+			cliPath, cliPathErr := getCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			tapCmdArgs := getDefaultTapCommandArgs()
+
+			tapNamespace := getDefaultTapNamespace()
+			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+			tapCmd := exec.Command(cliPath, tapCmdArgs...)
+			t.Logf("running command: %v", tapCmd.String())
+
+			t.Cleanup(func() {
+				if err := cleanupCommand(tapCmd); err != nil {
+					t.Logf("failed to cleanup tap command, err: %v", err)
+				}
+			})
+
+			if err := tapCmd.Start(); err != nil {
+				t.Errorf("failed to start tap command, err: %v", err)
+				return
+			}
+
+			apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+			if err := waitTapPodsReady(apiServerUrl); err != nil {
+				t.Errorf("failed to start tap pods on time, err: %v", err)
+				return
+			}
+
+			proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+			for i := 0; i < entriesCount; i++ {
+				if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+					t.Errorf("failed to send proxy request, err: %v", requestErr)
+					return
+				}
+			}
+
+			entriesCheckFunc := func() error {
+				timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+				entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, entriesCount, timestamp)
+				requestResult, requestErr := executeHttpGetRequest(entriesUrl)
+				if requestErr != nil {
+					return fmt.Errorf("failed to get entries, err: %v", requestErr)
+				}
+
+				entries := requestResult.([]interface{})
+
+				if len(entries) == 0 {
+					return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+				}
+
+				entry :=  entries[0].(map[string]interface{})
+
+				entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, entry["id"])
+				requestResult, requestErr = executeHttpGetRequest(entryUrl)
+				if requestErr != nil {
+					return fmt.Errorf("failed to get entry, err: %v", requestErr)
+				}
+
+				if requestResult == nil {
+					return fmt.Errorf("unexpected nil entry result")
+				}
+
+				return nil
+			}
+			if err := retriesExecute(shortRetriesCount, entriesCheckFunc); err != nil {
+				t.Errorf("%v", err)
+				return
+			}
+
+			fetchCmdArgs := getDefaultFetchCommandArgs()
+			fetchCmd := exec.Command(cliPath, fetchCmdArgs...)
+			t.Logf("running command: %v", fetchCmd.String())
+
+			if err := fetchCmd.Start(); err != nil {
+				t.Errorf("failed to start fetch command, err: %v", err)
+				return
+			}
+
+			harCheckFunc := func() error {
+				harBytes, readFileErr := ioutil.ReadFile("./unknown_source.har")
+				if readFileErr != nil {
+					return fmt.Errorf("failed to read har file, err: %v", readFileErr)
+				}
+
+				harEntries, err := getEntriesFromHarBytes(harBytes)
+				if err != nil {
+					return fmt.Errorf("failed to get entries from har, err: %v", err)
+				}
+
+				if len(harEntries) == 0 {
+					return fmt.Errorf("unexpected har entries result - Expected more than 0 entries")
+				}
+
+				return nil
+			}
+			if err := retriesExecute(shortRetriesCount, harCheckFunc); err != nil {
+				t.Errorf("%v", err)
+				return
+			}
+		})
+	}
+}
+
+func TestTapGuiPort(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []uint16{8898}
+
+	for _, guiPort := range tests {
+		t.Run(fmt.Sprintf("%d", guiPort), func(t *testing.T) {
+			cliPath, cliPathErr := getCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			tapCmdArgs := getDefaultTapCommandArgs()
+
+			tapNamespace := getDefaultTapNamespace()
+			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+			tapCmdArgs = append(tapCmdArgs, "-p", fmt.Sprintf("%d", guiPort))
+
+			tapCmd := exec.Command(cliPath, tapCmdArgs...)
+			t.Logf("running command: %v", tapCmd.String())
+
+			t.Cleanup(func() {
+				if err := cleanupCommand(tapCmd); err != nil {
+					t.Logf("failed to cleanup tap command, err: %v", err)
+				}
+			})
+
+			if err := tapCmd.Start(); err != nil {
+				t.Errorf("failed to start tap command, err: %v", err)
+				return
+			}
+
+			apiServerUrl := getApiServerUrl(guiPort)
+
+			if err := waitTapPodsReady(apiServerUrl); err != nil {
+				t.Errorf("failed to start tap pods on time, err: %v", err)
+				return
+			}
+		})
+	}
+}
+
+func TestTapAllNamespaces(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	expectedPods := []struct{
+		Name      string
+		Namespace string
+	}{
+		{Name: "httpbin", Namespace: "mizu-tests"},
+		{Name: "httpbin", Namespace: "mizu-tests2"},
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+	tapCmdArgs = append(tapCmdArgs, "-A")
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := cleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	podsUrl := fmt.Sprintf("%v/api/tapStatus", apiServerUrl)
+	requestResult, requestErr := executeHttpGetRequest(podsUrl)
+	if requestErr != nil {
+		t.Errorf("failed to get tap status, err: %v", requestErr)
+		return
+	}
+
+	pods, err := getPods(requestResult)
+	if err != nil {
+		t.Errorf("failed to get pods, err: %v", err)
+		return
+	}
+
+	for _, expectedPod := range expectedPods {
+		podFound := false
+
+		for _, pod := range pods {
+			podNamespace :=  pod["namespace"].(string)
+			podName := pod["name"].(string)
+
+			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
+				podFound = true
+				break
+			}
+		}
+
+		if !podFound {
+			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
+			return
+		}
+	}
+}
+
+func TestTapMultipleNamespaces(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	expectedPods := []struct{
+		Name      string
+		Namespace string
+	}{
+		{Name: "httpbin", Namespace: "mizu-tests"},
+		{Name: "httpbin2", Namespace: "mizu-tests"},
+		{Name: "httpbin", Namespace: "mizu-tests2"},
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+	var namespacesCmd []string
+	for _, expectedPod := range expectedPods {
+		namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace)
+	}
+	tapCmdArgs = append(tapCmdArgs, namespacesCmd...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := cleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	podsUrl := fmt.Sprintf("%v/api/tapStatus", apiServerUrl)
+	requestResult, requestErr := executeHttpGetRequest(podsUrl)
+	if requestErr != nil {
+		t.Errorf("failed to get tap status, err: %v", requestErr)
+		return
+	}
+
+	pods, err := getPods(requestResult)
+	if err != nil {
+		t.Errorf("failed to get pods, err: %v", err)
+		return
+	}
+
+	if len(expectedPods) != len(pods) {
+		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
+		return
+	}
+
+	for _, expectedPod := range expectedPods {
+		podFound := false
+
+		for _, pod := range pods {
+			podNamespace :=  pod["namespace"].(string)
+			podName := pod["name"].(string)
+
+			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
+				podFound = true
+				break
+			}
+		}
+
+		if !podFound {
+			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
+			return
+		}
+	}
+}
+
+func TestTapRegex(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	regexPodName := "httpbin2"
+	expectedPods := []struct{
+		Name      string
+		Namespace string
+	}{
+		{Name: regexPodName, Namespace: "mizu-tests"},
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgsWithRegex(regexPodName)
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := cleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	podsUrl := fmt.Sprintf("%v/api/tapStatus", apiServerUrl)
+	requestResult, requestErr := executeHttpGetRequest(podsUrl)
+	if requestErr != nil {
+		t.Errorf("failed to get tap status, err: %v", requestErr)
+		return
+	}
+
+	pods, err := getPods(requestResult)
+	if err != nil {
+		t.Errorf("failed to get pods, err: %v", err)
+		return
+	}
+
+	if len(expectedPods) != len(pods) {
+		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
+		return
+	}
+
+	for _, expectedPod := range expectedPods {
+		podFound := false
+
+		for _, pod := range pods {
+			podNamespace :=  pod["namespace"].(string)
+			podName := pod["name"].(string)
+
+			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
+				podFound = true
+				break
+			}
+		}
+
+		if !podFound {
+			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
+			return
+		}
+	}
+}
+
+func TestTapDryRun(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmdArgs = append(tapCmdArgs, "--dry-run")
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	resultChannel := make(chan string, 1)
+
+	go func() {
+		if err := tapCmd.Wait(); err != nil {
+			resultChannel <- "fail"
+			return
+		}
+		resultChannel <- "success"
+	}()
+
+	go func() {
+		time.Sleep(shortRetriesCount * time.Second)
+		resultChannel <- "fail"
+	}()
+
+	testResult := <- resultChannel
+	if testResult != "success" {
+		t.Errorf("unexpected result - dry run cmd not done")
+	}
+}
+
+func TestTapRedact(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := cleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+	requestBody := map[string]string{"User": "Mizu"}
+	for i := 0; i < defaultEntriesCount; i++ {
+		if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil {
+			t.Errorf("failed to send proxy request, err: %v", requestErr)
+			return
+		}
+	}
+
+	redactCheckFunc := func() error {
+		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+		entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
+		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		}
+
+		entries := requestResult.([]interface{})
+		firstEntry :=  entries[0].(map[string]interface{})
+
+		entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, firstEntry["id"])
+		requestResult, requestErr = executeHttpGetRequest(entryUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entry, err: %v", requestErr)
+		}
+
+		entry := requestResult.(map[string]interface{})["entry"].(map[string]interface{})
+		entryRequest := entry["request"].(map[string]interface{})
+
+		headers :=  entryRequest["headers"].([]interface{})
+		for _, headerInterface := range headers {
+			header := headerInterface.(map[string]interface{})
+			if header["name"].(string) != "User-Agent" {
+				continue
+			}
+
+			userAgent := header["value"].(string)
+			if userAgent != "[REDACTED]" {
+				return fmt.Errorf("unexpected result - user agent is not redacted")
+			}
+		}
+
+		data := entryRequest["postData"].(map[string]interface{})
+		textDataStr := data["text"].(string)
+
+		var textData map[string]string
+		if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
+			return fmt.Errorf("failed to parse text data, err: %v", parseErr)
+		}
+
+		if textData["User"] != "[REDACTED]" {
+			return fmt.Errorf("unexpected result - user in body is not redacted")
+		}
+
+		return nil
+	}
+	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
+		t.Errorf("%v", err)
+		return
+	}
+}
+
+func TestTapNoRedact(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmdArgs = append(tapCmdArgs, "--no-redact")
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := cleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+	requestBody := map[string]string{"User": "Mizu"}
+	for i := 0; i < defaultEntriesCount; i++ {
+		if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil {
+			t.Errorf("failed to send proxy request, err: %v", requestErr)
+			return
+		}
+	}
+
+	redactCheckFunc := func() error {
+		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+		entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
+		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		}
+
+		entries := requestResult.([]interface{})
+		firstEntry :=  entries[0].(map[string]interface{})
+
+		entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, firstEntry["id"])
+		requestResult, requestErr = executeHttpGetRequest(entryUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entry, err: %v", requestErr)
+		}
+
+		entry := requestResult.(map[string]interface{})["entry"].(map[string]interface{})
+		entryRequest := entry["request"].(map[string]interface{})
+
+		headers :=  entryRequest["headers"].([]interface{})
+		for _, headerInterface := range headers {
+			header := headerInterface.(map[string]interface{})
+			if header["name"].(string) != "User-Agent" {
+				continue
+			}
+
+			userAgent := header["value"].(string)
+			if userAgent == "[REDACTED]" {
+				return fmt.Errorf("unexpected result - user agent is redacted")
+			}
+		}
+
+		data := entryRequest["postData"].(map[string]interface{})
+		textDataStr := data["text"].(string)
+
+		var textData map[string]string
+		if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
+			return fmt.Errorf("failed to parse text data, err: %v", parseErr)
+		}
+
+		if textData["User"] == "[REDACTED]" {
+			return fmt.Errorf("unexpected result - user in body is redacted")
+		}
+
+		return nil
+	}
+	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
+		t.Errorf("%v", err)
+		return
+	}
+}
+
+func TestTapRegexMasking(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmdArgs = append(tapCmdArgs, "-r", "Mizu")
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := cleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+	for i := 0; i < defaultEntriesCount; i++ {
+		response, requestErr := http.Post(fmt.Sprintf("%v/post", proxyUrl), "text/plain", bytes.NewBufferString("Mizu"))
+		if _, requestErr = executeHttpRequest(response, requestErr); requestErr != nil {
+			t.Errorf("failed to send proxy request, err: %v", requestErr)
+			return
+		}
+	}
+
+	redactCheckFunc := func() error {
+		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+		entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
+		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		}
+
+		entries := requestResult.([]interface{})
+		firstEntry :=  entries[0].(map[string]interface{})
+
+		entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, firstEntry["id"])
+		requestResult, requestErr = executeHttpGetRequest(entryUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entry, err: %v", requestErr)
+		}
+
+		entry := requestResult.(map[string]interface{})["entry"].(map[string]interface{})
+		entryRequest := entry["request"].(map[string]interface{})
+
+		data := entryRequest["postData"].(map[string]interface{})
+		textData := data["text"].(string)
+
+		if textData != "[REDACTED]" {
+			return fmt.Errorf("unexpected result - body is not redacted")
+		}
+
+		return nil
+	}
+	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
+		t.Errorf("%v", err)
+		return
+	}
+}

acceptanceTests/testsUtils.go

@@ -0,0 +1,205 @@
+package acceptanceTests
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"syscall"
+	"time"
+)
+
+const (
+	longRetriesCount     = 100
+	shortRetriesCount    = 10
+	defaultApiServerPort = 8899
+	defaultNamespaceName = "mizu-tests"
+	defaultServiceName   = "httpbin"
+	defaultEntriesCount  = 50
+)
+
+func getCliPath() (string, error) {
+	dir, filePathErr := os.Getwd()
+	if filePathErr != nil {
+		return "", filePathErr
+	}
+
+	cliPath := path.Join(dir, "../cli/bin/mizu_ci")
+	return cliPath, nil
+}
+
+func getConfigPath() (string, error) {
+	home, homeDirErr := os.UserHomeDir()
+	if homeDirErr != nil {
+		return "", homeDirErr
+	}
+
+	return path.Join(home, ".mizu", "config.yaml"), nil
+}
+
+func getProxyUrl(namespace string, service string) string {
+	return fmt.Sprintf("http://localhost:8080/api/v1/namespaces/%v/services/%v/proxy", namespace, service)
+}
+
+func getApiServerUrl(port uint16) string {
+	return fmt.Sprintf("http://localhost:%v/mizu", port)
+}
+
+func getDefaultCommandArgs() []string {
+	setFlag := "--set"
+	telemetry := "telemetry=false"
+	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
+	imagePullPolicy := "image-pull-policy=Never"
+
+	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy}
+}
+
+func getDefaultTapCommandArgs() []string {
+	tapCommand := "tap"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{tapCommand}, defaultCmdArgs...)
+}
+
+func getDefaultTapCommandArgsWithRegex(regex string) []string {
+	tapCommand := "tap"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{tapCommand, regex}, defaultCmdArgs...)
+}
+
+func getDefaultTapNamespace() []string {
+	return []string{"-n", "mizu-tests"}
+}
+
+func getDefaultFetchCommandArgs() []string {
+	fetchCommand := "fetch"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{fetchCommand}, defaultCmdArgs...)
+}
+
+func getDefaultConfigCommandArgs() []string {
+	configCommand := "config"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{configCommand}, defaultCmdArgs...)
+}
+
+func retriesExecute(retriesCount int, executeFunc func() error) error {
+	var lastError error
+
+	for i := 0; i < retriesCount; i++ {
+		if err := executeFunc(); err != nil {
+			lastError = err
+
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		return nil
+	}
+
+	return fmt.Errorf("reached max retries count, retries count: %v, last err: %v", retriesCount, lastError)
+}
+
+func waitTapPodsReady(apiServerUrl string) error {
+	resolvingUrl := fmt.Sprintf("%v/status/tappersCount", apiServerUrl)
+	tapPodsReadyFunc := func() error {
+		requestResult, requestErr := executeHttpGetRequest(resolvingUrl)
+		if requestErr != nil {
+			return requestErr
+		}
+
+		tappersCount := requestResult.(float64)
+		if tappersCount == 0 {
+			return fmt.Errorf("no tappers running")
+		}
+
+		return nil
+	}
+
+	return retriesExecute(longRetriesCount, tapPodsReadyFunc)
+}
+
+func jsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
+	var result interface{}
+	if parseErr := json.Unmarshal(jsonBytes, &result); parseErr != nil {
+		return nil, parseErr
+	}
+
+	return result, nil
+}
+
+func executeHttpRequest(response *http.Response, requestErr error) (interface{}, error) {
+	if requestErr != nil {
+		return nil, requestErr
+	} else if response.StatusCode != 200 {
+		return nil, fmt.Errorf("invalid status code %v", response.StatusCode)
+	}
+
+	defer func() { response.Body.Close() }()
+
+	data, readErr := ioutil.ReadAll(response.Body)
+	if readErr != nil {
+		return nil, readErr
+	}
+
+	return jsonBytesToInterface(data)
+}
+
+func executeHttpGetRequest(url string) (interface{}, error) {
+	response, requestErr := http.Get(url)
+	return executeHttpRequest(response, requestErr)
+}
+
+func executeHttpPostRequest(url string, body interface{}) (interface{}, error) {
+	requestBody, jsonErr := json.Marshal(body)
+	if jsonErr != nil {
+		return nil, jsonErr
+	}
+
+	response, requestErr := http.Post(url, "application/json", bytes.NewBuffer(requestBody))
+	return executeHttpRequest(response, requestErr)
+}
+
+func cleanupCommand(cmd *exec.Cmd) error {
+	if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
+		return err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getEntriesFromHarBytes(harBytes []byte) ([]interface{}, error) {
+	harInterface, convertErr := jsonBytesToInterface(harBytes)
+	if convertErr != nil {
+		return nil, convertErr
+	}
+
+	har := harInterface.(map[string]interface{})
+	harLog := har["log"].(map[string]interface{})
+	harEntries := harLog["entries"].([]interface{})
+
+	return harEntries, nil
+}
+
+func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) {
+	tapStatus := tapStatusInterface.(map[string]interface{})
+	podsInterface := tapStatus["pods"].([]interface{})
+
+	var pods []map[string]interface{}
+	for _, podInterface := range podsInterface {
+		pods = append(pods, podInterface.(map[string]interface{}))
+	}
+
+	return pods, nil
+}

agent/Makefile

@@ -0,0 +1,2 @@
+test: ## Run agent tests.
+	@go test ./... -coverpkg=./... -race -coverprofile=coverage.out -covermode=atomic

agent/main.go

@@ -26,14 +26,17 @@ var apiServerMode = flag.Bool("api-server", false, "Run in API server mode with
 var standaloneMode = flag.Bool("standalone", false, "Run in standalone tapper and API mode")
 var apiServerAddress = flag.String("api-server-address", "", "Address of mizu API server")
 var namespace = flag.String("namespace", "", "Resolve IPs if they belong to resources in this namespace (default is all)")
+var harsReaderMode = flag.Bool("hars-read", false, "Run in hars-read mode")
+var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
 
 func main() {
 	flag.Parse()
 	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
 	tapOpts := &tap.TapOpts{HostMode: hostMode}
 
-	if !*tapperMode && !*apiServerMode && !*standaloneMode {
-		panic("One of the flags --tap, --api or --standalone must be provided")
+
+	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode{
+		panic("One of the flags --tap, --api or --standalone or --hars-read must be provided")
 	}
 
 	if *standaloneMode {
@@ -77,6 +80,13 @@ func main() {
 		go api.StartReadingEntries(filteredHarChannel, nil)
 
 		hostApi(socketHarOutChannel)
+	} else if *harsReaderMode {
+		socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000)
+		filteredHarChannel := make(chan *tap.OutputChannelItem)
+
+		go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions())
+		go api.StartReadingEntries(filteredHarChannel, harsDir)
+		hostApi(nil)
 	}
 
 	signalChan := make(chan os.Signal, 1)

agent/pkg/api/main.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"mizuserver/pkg/holder"
+	"mizuserver/pkg/providers"
 	"net/url"
 	"os"
 	"path"
@@ -108,6 +109,7 @@ func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) {
 	}
 
 	for item := range outputItems {
+		providers.EntryAdded()
 		saveHarToDb(item.HarEntry, item.ConnectionInfo)
 	}
 }

agent/pkg/api/socket_server_handlers.go

@@ -28,6 +28,7 @@ func init() {
 func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 	if isTapper {
 		rlog.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
+		providers.TapperAdded()
 	} else {
 		rlog.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
 		socketListLock.Lock()
@@ -39,6 +40,7 @@ func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 	if isTapper {
 		rlog.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
+		providers.TapperRemoved()
 	} else {
 		rlog.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
 		socketListLock.Lock()

agent/pkg/controllers/entries_controller.go

@@ -241,14 +241,7 @@ func DeleteAllEntries(c *gin.Context) {
 }
 
 func GetGeneralStats(c *gin.Context) {
-	sqlQuery := "SELECT count(*) as count, min(timestamp) as min, max(timestamp) as max from mizu_entries"
-	var result struct {
-		Count int
-		Min   int
-		Max   int
-	}
-	database.GetEntriesTable().Raw(sqlQuery).Scan(&result)
-	c.JSON(http.StatusOK, result)
+	c.JSON(http.StatusOK, providers.GetGeneralStats())
 }
 
 func GetTappingStatus(c *gin.Context) {

agent/pkg/controllers/status_controller.go

@@ -30,3 +30,7 @@ func PostTappedPods(c *gin.Context) {
 		api.BroadcastToBrowserClients(jsonBytes)
 	}
 }
+
+func GetTappersCount(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.TappersCount)
+}

agent/pkg/models/models.go

@@ -56,12 +56,14 @@ type BaseEntryDetails struct {
 type ApplicableRules struct {
 	Latency int64 `json:"latency,omitempty"`
 	Status  bool  `json:"status,omitempty"`
+	NumberOfRules int `json:"numberOfRules,omitempty"`
 }
 
-func NewApplicableRules(status bool, latency int64) ApplicableRules {
+func NewApplicableRules(status bool, latency int64, number int) ApplicableRules {
 	ar := ApplicableRules{}
 	ar.Status = status
 	ar.Latency = latency
+	ar.NumberOfRules = number
 	return ar
 }
 
@@ -218,7 +220,7 @@ func (fewp *FullEntryWithPolicy) UnmarshalData(entry *MizuEntry) error {
 
 func RunValidationRulesState(harEntry har.Entry, service string) ApplicableRules {
 	numberOfRules, resultPolicyToSend := rules.MatchRequestPolicy(harEntry, service)
-	statusPolicyToSend, latency := rules.PassedValidationRules(resultPolicyToSend, numberOfRules)
-	ar := NewApplicableRules(statusPolicyToSend, latency)
+	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend, numberOfRules)
+	ar := NewApplicableRules(statusPolicyToSend, latency, numberOfRules)
 	return ar
 }

agent/pkg/providers/stats_provider.go

@@ -0,0 +1,36 @@
+package providers
+
+import (
+	"reflect"
+	"time"
+)
+
+type GeneralStats struct {
+	EntriesCount        int
+	FirstEntryTimestamp int
+	LastEntryTimestamp  int
+}
+
+var generalStats = GeneralStats{}
+
+func ResetGeneralStats() {
+	generalStats = GeneralStats{}
+}
+
+func GetGeneralStats() GeneralStats {
+	return generalStats
+}
+
+func EntryAdded() {
+	generalStats.EntriesCount++
+
+	currentTimestamp := int(time.Now().Unix())
+
+	if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) {
+		generalStats.FirstEntryTimestamp = currentTimestamp
+	}
+
+	generalStats.LastEntryTimestamp = currentTimestamp
+}
+
+

agent/pkg/providers/stats_provider_test.go

@@ -0,0 +1,35 @@
+package providers_test
+
+import (
+	"fmt"
+	"mizuserver/pkg/providers"
+	"testing"
+)
+
+func TestNoEntryAddedCount(t *testing.T) {
+	entriesStats := providers.GetGeneralStats()
+
+	if entriesStats.EntriesCount != 0 {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesCount)
+	}
+}
+
+func TestEntryAddedCount(t *testing.T) {
+	tests := []int{1, 5, 10, 100, 500, 1000}
+
+	for _, entriesCount := range tests {
+		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
+			for i := 0; i < entriesCount; i++ {
+				providers.EntryAdded()
+			}
+
+			entriesStats := providers.GetGeneralStats()
+
+			if entriesStats.EntriesCount != entriesCount {
+				t.Errorf("unexpected result - expected: %v, actual: %v", entriesCount, entriesStats.EntriesCount)
+			}
+
+			t.Cleanup(providers.ResetGeneralStats)
+		})
+	}
+}

agent/pkg/providers/status_provider.go

@@ -4,14 +4,18 @@ import (
 	"github.com/patrickmn/go-cache"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
+	"sync"
 	"time"
 )
 
 const tlsLinkRetainmentTime = time.Minute * 15
 
 var (
-	TapStatus shared.TapStatus
+	TappersCount   int
+	TapStatus      shared.TapStatus
 	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
+
+	tappersCountLock = sync.Mutex{}
 )
 
 func GetAllRecentTLSAddresses() []string {
@@ -26,3 +30,15 @@ func GetAllRecentTLSAddresses() []string {
 
 	return recentTLSLinks
 }
+
+func TapperAdded() {
+	tappersCountLock.Lock()
+	TappersCount++
+	tappersCountLock.Unlock()
+}
+
+func TapperRemoved() {
+	tappersCountLock.Lock()
+	TappersCount--
+	tappersCountLock.Unlock()
+}

agent/pkg/routes/status_routes.go

@@ -9,4 +9,6 @@ func StatusRoutes(ginApp *gin.Engine) {
 	routeGroup := ginApp.Group("/status")
 
 	routeGroup.POST("/tappedPods", controllers.PostTappedPods)
+
+	routeGroup.GET("/tappersCount", controllers.GetTappersCount)
 }

agent/pkg/rules/models.go

@@ -92,19 +92,19 @@ func MatchRequestPolicy(harEntry har.Entry, service string) (int, []RulesMatched
 	return len(enforcePolicy.Rules), resultPolicyToSend
 }
 
-func PassedValidationRules(rulesMatched []RulesMatched, numberOfRules int) (bool, int64) {
+func PassedValidationRules(rulesMatched []RulesMatched, numberOfRules int) (bool, int64, int) {
 	if len(rulesMatched) == 0 {
-		return false, 0
+		return false, 0, 0
 	}
 	for _, rule := range rulesMatched {
 		if rule.Matched == false {
-			return false, -1
+			return false, -1, len(rulesMatched)
 		}
 	}
 	for _, rule := range rulesMatched {
 		if strings.ToLower(rule.Rule.Type) == "latency" {
-			return true, rule.Rule.Latency
+			return true, rule.Rule.Latency, len(rulesMatched)
 		}
 	}
-	return true, -1
+	return true, -1, len(rulesMatched)
 }

agent/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go

@@ -158,9 +158,11 @@ func filterJsonBody(bytes []byte) ([]byte, error) {
 
 func filterJsonMap(jsonMap map[string] interface{}) {
 	for key, value := range jsonMap {
+		// Do not replace nil values with maskedFieldPlaceholderValue
 		if value == nil {
-			return
+			continue
 		}
+
 		nestedMap, isNested := value.(map[string] interface{})
 		if isNested {
 			filterJsonMap(nestedMap)

build-agent-ci.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+GCP_PROJECT=up9-docker-hub
+REPOSITORY=gcr.io/$GCP_PROJECT
+SERVER_NAME=mizu
+GIT_BRANCH=ci
+
+DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
+SEM_VER=${SEM_VER=0.0.0}
+
+DOCKER_TAGGED_BUILD="$DOCKER_REPO:$SEM_VER"
+
+echo "building $DOCKER_TAGGED_BUILD"
+docker build -t ${DOCKER_TAGGED_BUILD} --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .

build-push-featurebranch.sh

@@ -1,12 +1,14 @@
 #!/bin/bash
 set -e
 
-SERVER_NAME=mizu
 GCP_PROJECT=up9-docker-hub
 REPOSITORY=gcr.io/$GCP_PROJECT
+SERVER_NAME=mizu
 GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
-SEM_VER=${SEM_VER=0.0.0}
+
 DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
+SEM_VER=${SEM_VER=0.0.0}
+
 DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
 
 if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
@@ -21,6 +23,6 @@ docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_
 
 for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
 do
-        echo pushing "$DOCKER_TAG"
-        docker push "$DOCKER_TAG"
+  echo pushing "$DOCKER_TAG"
+  docker push "$DOCKER_TAG"
 done

cli/Makefile

@@ -24,7 +24,7 @@ build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables
 
 build-all: ## Build for all supported platforms.
 	@echo "Compiling for every OS and Platform"
-	@mkdir -p bin && echo "SHA256 checksums available for compiled binaries \n\nRun \`shasum -a 256 -c mizu_OS_ARCH.sha256\` to verify\n\n" >  bin/README.md
+	@mkdir -p bin && sed s/_SEM_VER_/$(SEM_VER)/g README.md.TEMPLATE >  bin/README.md
 	@$(MAKE) build GOOS=darwin GOARCH=amd64
 	@$(MAKE) build GOOS=linux GOARCH=amd64
 	@# $(MAKE) build GOOS=darwin GOARCH=arm64
@@ -39,3 +39,6 @@ build-all: ## Build for all supported platforms.
 clean: ## Clean all build artifacts.
 	go clean
 	rm -rf ./bin/*
+
+test: ## Run cli tests.
+	@go test ./... -coverpkg=./... -race -coverprofile=coverage.out -covermode=atomic

cli/README.md.TEMPLATE

@@ -0,0 +1,20 @@
+# Mizu release _SEM_VER_
+
+Download Mizu for your platform
+
+**Mac** (on Intel chip)
+```
+curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_darwin_amd64 && chmod 755 mizu
+```
+
+**Linux** 
+```
+curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_linux_amd64 && chmod 755 mizu
+```
+
+
+### Checksums
+SHA256 checksums available for compiled binaries.
+Run `shasum -a 256 -c mizu_OS_ARCH.sha256` to verify.
+
+

cli/apiserver/provider.go

@@ -0,0 +1,168 @@
+package apiserver
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/shared"
+	"io/ioutil"
+	core "k8s.io/api/core/v1"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+type apiServerProvider struct {
+	url     string
+	isReady bool
+}
+
+var Provider = apiServerProvider{}
+
+func (provider *apiServerProvider) InitAndTestConnection(url string, retries int) error {
+	healthUrl := fmt.Sprintf("%s/", url)
+	retriesLeft := retries
+	for retriesLeft > 0 {
+		if response, err := http.Get(healthUrl); err != nil {
+			logger.Log.Debugf("[ERROR] failed connecting to api server %v", err)
+		} else if response.StatusCode != 200 {
+			logger.Log.Debugf("can't connect to api server yet, response status code %v", response.StatusCode)
+		} else {
+			logger.Log.Debugf("connection test to api server passed successfully")
+			break
+		}
+		retriesLeft -= 1
+		time.Sleep(time.Second)
+	}
+
+	if retriesLeft == 0 {
+		provider.isReady = false
+		return fmt.Errorf("couldn't reach the api server after %v retries", retries)
+	}
+	provider.url = url
+	provider.isReady = true
+	return nil
+}
+
+func (provider *apiServerProvider) ReportTappedPods(pods []core.Pod) error {
+	if !provider.isReady {
+		return fmt.Errorf("trying to reach api server when not initialized yet")
+	}
+	tappedPodsUrl := fmt.Sprintf("%s/status/tappedPods", provider.url)
+
+	podInfos := make([]shared.PodInfo, 0)
+	for _, pod := range pods {
+		podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
+	}
+	tapStatus := shared.TapStatus{Pods: podInfos}
+
+	if jsonValue, err := json.Marshal(tapStatus); err != nil {
+		return fmt.Errorf("failed Marshal the tapped pods %w", err)
+	} else {
+		if response, err := http.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+			return fmt.Errorf("failed sending to API server the tapped pods %w", err)
+		} else if response.StatusCode != 200 {
+			return fmt.Errorf("failed sending to API server the tapped pods, response status code %v", response.StatusCode)
+		} else {
+			logger.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos))
+			return nil
+		}
+	}
+}
+
+func (provider *apiServerProvider) RequestAnalysis(analysisDestination string, sleepIntervalSec int) error {
+	if !provider.isReady {
+		return fmt.Errorf("trying to reach api server when not initialized yet")
+	}
+	urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", provider.url, url.QueryEscape(analysisDestination), sleepIntervalSec)
+	u, parseErr := url.ParseRequestURI(urlPath)
+	if parseErr != nil {
+		logger.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr)
+	}
+
+	logger.Log.Debugf("Analysis url %v", u.String())
+	if response, requestErr := http.Get(u.String()); requestErr != nil {
+		return fmt.Errorf("failed to notify agent for analysis, err: %w", requestErr)
+	} else if response.StatusCode != 200 {
+		return fmt.Errorf("failed to notify agent for analysis, status code: %v", response.StatusCode)
+	} else {
+		logger.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis")
+		return nil
+	}
+}
+
+func (provider *apiServerProvider) GetGeneralStats() (map[string]interface{}, error) {
+	if !provider.isReady {
+		return nil, fmt.Errorf("trying to reach api server when not initialized yet")
+	}
+	generalStatsUrl := fmt.Sprintf("%s/api/generalStats", provider.url)
+
+	response, requestErr := http.Get(generalStatsUrl)
+	if requestErr != nil {
+		return nil, fmt.Errorf("failed to get general stats for telemetry, err: %w", requestErr)
+	} else if response.StatusCode != 200 {
+		return nil, fmt.Errorf("failed to get general stats for telemetry, status code: %v", response.StatusCode)
+	}
+
+	defer func() { _ = response.Body.Close() }()
+
+	data, readErr := ioutil.ReadAll(response.Body)
+	if readErr != nil {
+		return nil, fmt.Errorf("failed to read general stats for telemetry, err: %v", readErr)
+	}
+
+	var generalStats map[string]interface{}
+	if parseErr := json.Unmarshal(data, &generalStats); parseErr != nil {
+		return nil, fmt.Errorf("failed to parse general stats for telemetry, err: %v", parseErr)
+	}
+	return generalStats, nil
+}
+
+func (provider *apiServerProvider) GetHars(fromTimestamp int, toTimestamp int) (*zip.Reader, error) {
+	if !provider.isReady {
+		return nil, fmt.Errorf("trying to reach api server when not initialized yet")
+	}
+	resp, err := http.Get(fmt.Sprintf("%s/api/har?from=%v&to=%v", provider.url, fromTimestamp, toTimestamp))
+	if err != nil {
+		return nil, fmt.Errorf("failed getting har from api server %w", err)
+	}
+
+	defer func() { _ = resp.Body.Close() }()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed reading hars %w", err)
+	}
+
+	zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
+	if err != nil {
+		return nil, fmt.Errorf("failed craeting zip reader %w", err)
+	}
+	return zipReader, nil
+}
+
+func (provider *apiServerProvider) GetVersion() (string, error) {
+	if !provider.isReady {
+		return "", fmt.Errorf("trying to reach api server when not initialized yet")
+	}
+	versionUrl, _ := url.Parse(fmt.Sprintf("%s/metadata/version", provider.url))
+	req := &http.Request{
+		Method: http.MethodGet,
+		URL:    versionUrl,
+	}
+	statusResp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer statusResp.Body.Close()
+
+	versionResponse := &shared.VersionResponse{}
+	if err := json.NewDecoder(statusResp.Body).Decode(&versionResponse); err != nil {
+		return "", err
+	}
+
+	return versionResponse.SemVer, nil
+}

cli/cmd/common.go

@@ -0,0 +1,45 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/errormessage"
+	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+func GetApiServerUrl() string {
+	return fmt.Sprintf("http://%s", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort))
+}
+
+func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+	err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
+	if err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+
+			"Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName))
+		cancel()
+	}
+}
+
+func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
+	logger.Log.Debugf("waiting for finish...")
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+
+	// block until ctx cancel is called or termination signal is received
+	select {
+	case <-ctx.Done():
+		logger.Log.Debugf("ctx done")
+		break
+	case <-sigChan:
+		logger.Log.Debugf("Got termination signal, canceling execution...")
+		cancel()
+	}
+}

cli/cmd/config.go

@@ -2,32 +2,36 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"io/ioutil"
 )
 
-var regenerateFile bool
-
 var configCmd = &cobra.Command{
 	Use:   "config",
 	Short: "Generate config with default values",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		template, err := mizu.GetConfigWithDefaults()
+		go telemetry.ReportRun("config", config.Config.Config)
+
+		template, err := config.GetConfigWithDefaults()
 		if err != nil {
-			mizu.Log.Errorf("Failed generating config with defaults %v", err)
+			logger.Log.Errorf("Failed generating config with defaults %v", err)
 			return nil
 		}
-		if regenerateFile {
+		if config.Config.Config.Regenerate {
 			data := []byte(template)
-			if err := ioutil.WriteFile(mizu.GetConfigFilePath(), data, 0644); err != nil {
-				mizu.Log.Errorf("Failed writing config %v", err)
+			if err := ioutil.WriteFile(config.Config.ConfigFilePath, data, 0644); err != nil {
+				logger.Log.Errorf("Failed writing config %v", err)
 				return nil
 			}
-			mizu.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, mizu.GetConfigFilePath())))
+			logger.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, config.Config.ConfigFilePath)))
 		} else {
-			mizu.Log.Debugf("Writing template config.\n%v", template)
+			logger.Log.Debugf("Writing template config.\n%v", template)
 			fmt.Printf("%v", template)
 		}
 		return nil
@@ -36,5 +40,9 @@ var configCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(configCmd)
-	configCmd.Flags().BoolVarP(&regenerateFile, "regenerate", "r", false, fmt.Sprintf("Regenerate the config file with default values %s", mizu.GetConfigFilePath()))
+
+	defaultConfig := config.ConfigStruct{}
+	defaults.Set(&defaultConfig)
+
+	configCmd.Flags().BoolP(configStructs.RegenerateConfigName, "r", defaultConfig.Config.Regenerate, fmt.Sprintf("Regenerate the config file with default values to path %s or to chosen path using --%s", defaultConfig.ConfigFilePath, config.ConfigFilePathCommandName))
 }

cli/cmd/fetch.go

@@ -3,16 +3,27 @@ package cmd
 import (
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/cli/mizu/configStructs"
+	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu/version"
+	"github.com/up9inc/mizu/cli/telemetry"
+	"github.com/up9inc/mizu/cli/uiUtils"
 )
 
 var fetchCmd = &cobra.Command{
 	Use:   "fetch",
 	Short: "Download recorded traffic to files",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go mizu.ReportRun("fetch", mizu.Config.Fetch)
-		if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.Fetch.GuiPort); err != nil {
+		go telemetry.ReportRun("fetch", config.Config.Fetch)
+
+		if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl(), 1); err != nil {
+			logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, make sure one running")
+			return nil
+		}
+
+		if isCompatible, err := version.CheckVersionCompatibility(); err != nil {
 			return err
 		} else if !isCompatible {
 			return nil

cli/cmd/fetchRunner.go

@@ -1,95 +1,25 @@
 package cmd
 
 import (
-	"archive/zip"
-	"bytes"
-	"fmt"
-	"github.com/up9inc/mizu/cli/kubernetes"
-	"github.com/up9inc/mizu/cli/mizu"
-	"io"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
+	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu/fsUtils"
+	"github.com/up9inc/mizu/cli/uiUtils"
 )
 
 func RunMizuFetch() {
-	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Fetch.GuiPort)
-	resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, mizu.Config.Fetch.FromTimestamp, mizu.Config.Fetch.ToTimestamp))
-	if err != nil {
-		log.Fatal(err)
+	if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl(), 5); err != nil {
+		logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, check logs")
 	}
 
-	defer func() { _ = resp.Body.Close() }()
-
-	body, err := ioutil.ReadAll(resp.Body)
+	zipReader, err := apiserver.Provider.GetHars(config.Config.Fetch.FromTimestamp, config.Config.Fetch.ToTimestamp)
 	if err != nil {
-		log.Fatal(err)
+		logger.Log.Errorf("Failed fetch data from API server %v", err)
+		return
 	}
 
-	zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
-	if err != nil {
-		log.Fatal(err)
+	if err := fsUtils.Unzip(zipReader, config.Config.Fetch.Directory); err != nil {
+		logger.Log.Debugf("[ERROR] failed unzip %v", err)
 	}
-
-	_ = Unzip(zipReader, mizu.Config.Fetch.Directory)
-}
-
-func Unzip(reader *zip.Reader, dest string) error {
-	dest, _ = filepath.Abs(dest)
-	_ = os.MkdirAll(dest, os.ModePerm)
-
-	// Closure to address file descriptors issue with all the deferred .Close() methods
-	extractAndWriteFile := func(f *zip.File) error {
-		rc, err := f.Open()
-		if err != nil {
-			return err
-		}
-		defer func() {
-			if err := rc.Close(); err != nil {
-				panic(err)
-			}
-		}()
-
-		path := filepath.Join(dest, f.Name)
-
-		// Check for ZipSlip (Directory traversal)
-		if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) {
-			return fmt.Errorf("illegal file path: %s", path)
-		}
-
-		if f.FileInfo().IsDir() {
-			_ = os.MkdirAll(path, f.Mode())
-		} else {
-			_ = os.MkdirAll(filepath.Dir(path), f.Mode())
-			mizu.Log.Infof("writing HAR file [ %v ]", path)
-			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
-			if err != nil {
-				return err
-			}
-			defer func() {
-				if err := f.Close(); err != nil {
-					panic(err)
-				}
-				mizu.Log.Info(" done")
-			}()
-
-			_, err = io.Copy(f, rc)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	for _, f := range reader.File {
-		err := extractAndWriteFile(f)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
 }

cli/cmd/logs.go

@@ -2,38 +2,38 @@ package cmd
 
 import (
 	"context"
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/fsUtils"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/kubernetes"
-	"github.com/up9inc/mizu/cli/mizu"
-	"os"
-	"path"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu/fsUtils"
+	"github.com/up9inc/mizu/cli/telemetry"
 )
 
-var filePath string
-
 var logsCmd = &cobra.Command{
 	Use:   "logs",
 	Short: "Create a zip file with logs for Github issue or troubleshoot",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.View.KubeConfigPath)
+		go telemetry.ReportRun("logs", config.Config.Logs)
+
+		kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
 		if err != nil {
+			logger.Log.Error(err)
 			return nil
 		}
 		ctx, _ := context.WithCancel(context.Background())
 
-		if filePath == "" {
-			pwd, err := os.Getwd()
-			if err != nil {
-				mizu.Log.Errorf("Failed to get PWD, %v (try using `mizu logs -f <full path dest zip file>)`", err)
-				return nil
-			}
-			filePath = path.Join(pwd, "mizu_logs.zip")
+		if validationErr := config.Config.Logs.Validate(); validationErr != nil {
+			return errormessage.FormatError(validationErr)
 		}
-		mizu.Log.Debugf("Using file path %s", filePath)
 
-		if err := fsUtils.DumpLogs(kubernetesProvider, ctx, filePath); err != nil {
-			mizu.Log.Errorf("Failed dump logs %v", err)
+		logger.Log.Debugf("Using file path %s", config.Config.Logs.FilePath())
+
+		if dumpLogsErr := fsUtils.DumpLogs(kubernetesProvider, ctx, config.Config.Logs.FilePath()); dumpLogsErr != nil {
+			logger.Log.Errorf("Failed dump logs %v", dumpLogsErr)
 		}
 
 		return nil
@@ -42,5 +42,9 @@ var logsCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(logsCmd)
-	logsCmd.Flags().StringVarP(&filePath, "file", "f", "", "Path for zip file (default current <pwd>\\mizu_logs.zip)")
+
+	defaultLogsConfig := configStructs.LogsConfig{}
+	defaults.Set(&defaultLogsConfig)
+
+	logsCmd.Flags().StringP(configStructs.FileLogsName, "f", defaultLogsConfig.FileStr, "Path for zip file (default current <pwd>\\mizu_logs.zip)")
 }

cli/cmd/root.go

@@ -2,9 +2,14 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/fsUtils"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/fsUtils"
+	"github.com/up9inc/mizu/cli/mizu/version"
+	"github.com/up9inc/mizu/cli/uiUtils"
 )
 
 var rootCmd = &cobra.Command{
@@ -13,24 +18,39 @@ var rootCmd = &cobra.Command{
 	Long: `A web traffic viewer for kubernetes
 Further info is available at https://github.com/up9inc/mizu`,
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-		if err := fsUtils.EnsureDir(mizu.GetMizuFolderPath()); err != nil {
-			mizu.Log.Errorf("Failed to use mizu folder, %v", err)
+		if err := config.InitConfig(cmd); err != nil {
+			logger.Log.Fatal(err)
 		}
-		mizu.InitLogger()
-		if err := mizu.InitConfig(cmd); err != nil {
-			mizu.Log.Fatal(err)
-		}
-
 		return nil
 	},
 }
 
 func init() {
-	rootCmd.PersistentFlags().StringSlice(mizu.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", mizu.SetCommandName))
+	defaultConfig := config.ConfigStruct{}
+	defaults.Set(&defaultConfig)
+
+	rootCmd.PersistentFlags().StringSlice(config.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", config.SetCommandName))
+	rootCmd.PersistentFlags().String(config.ConfigFilePathCommandName, defaultConfig.ConfigFilePath, fmt.Sprintf("Override config file path using --%s", config.ConfigFilePathCommandName))
+}
+
+func printNewVersionIfNeeded(versionChan chan string) {
+	versionMsg := <-versionChan
+	if versionMsg != "" {
+		logger.Log.Infof(uiUtils.Yellow, versionMsg)
+	}
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the tapCmd.
 func Execute() {
+	if err := fsUtils.EnsureDir(mizu.GetMizuFolderPath()); err != nil {
+		logger.Log.Errorf("Failed to use mizu folder, %v", err)
+	}
+	logger.InitLogger()
+
+	versionChan := make(chan string)
+	defer printNewVersionIfNeeded(versionChan)
+	go version.CheckNewerVersion(versionChan)
+
 	cobra.CheckErr(rootCmd.Execute())
 }

cli/cmd/tap.go

@@ -2,13 +2,15 @@ package cmd
 
 import (
 	"errors"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"os"
 
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/errormessage"
-	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/cli/mizu/configStructs"
 	"github.com/up9inc/mizu/cli/uiUtils"
 )
 
@@ -20,31 +22,27 @@ var tapCmd = &cobra.Command{
 	Long: `Record the ingoing traffic of a kubernetes pod.
 Supported protocols are HTTP and gRPC.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go mizu.ReportRun("tap", mizu.Config.Tap)
+		go telemetry.ReportRun("tap", config.Config.Tap)
 		RunMizuTap()
 		return nil
 	},
 	PreRunE: func(cmd *cobra.Command, args []string) error {
 		if len(args) == 1 {
-			mizu.Config.Tap.PodRegexStr = args[0]
+			config.Config.Tap.PodRegexStr = args[0]
 		} else if len(args) > 1 {
 			return errors.New("unexpected number of arguments")
 		}
 
-		if err := mizu.Config.Validate(); err != nil {
+		if err := config.Config.Tap.Validate(); err != nil {
 			return errormessage.FormatError(err)
 		}
 
-		if err := mizu.Config.Tap.Validate(); err != nil {
-			return errormessage.FormatError(err)
-		}
+		logger.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", config.Config.Tap.HumanMaxEntriesDBSize)
 
-		mizu.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", mizu.Config.Tap.HumanMaxEntriesDBSize)
-
-		if mizu.Config.Tap.Analysis {
-			mizu.Log.Infof(analysisMessageToConfirm)
+		if config.Config.Tap.Analysis {
+			logger.Log.Infof(analysisMessageToConfirm)
 			if !uiUtils.AskForConfirmation("Would you like to proceed [Y/n]: ") {
-				mizu.Log.Infof("You can always run mizu without analysis, aborting")
+				logger.Log.Infof("You can always run mizu without analysis, aborting")
 				os.Exit(0)
 			}
 		}
@@ -60,10 +58,10 @@ func init() {
 	defaults.Set(&defaultTapConfig)
 
 	tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
-	tapCmd.Flags().StringArrayP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")
+	tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")
 	tapCmd.Flags().Bool(configStructs.AnalysisTapName, defaultTapConfig.Analysis, "Uploads traffic to UP9 for further analysis (Beta)")
 	tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
-	tapCmd.Flags().StringArrayP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
+	tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
 	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
 	tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any")

cli/cmd/tapRunner.go

@@ -1,32 +1,28 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
-	"github.com/up9inc/mizu/cli/fsUtils"
-	"github.com/up9inc/mizu/cli/goUtils"
-	"github.com/up9inc/mizu/cli/mizu/configStructs"
-	"net/http"
-	"net/url"
-	"os"
-	"os/signal"
-	"path"
-	"regexp"
-	"strings"
-	"syscall"
-	"time"
-
+	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/fsUtils"
+	"github.com/up9inc/mizu/cli/mizu/goUtils"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
 	yaml "gopkg.in/yaml.v3"
 	core "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"path"
+	"regexp"
+	"strings"
+	"time"
 )
 
 const (
@@ -46,21 +42,20 @@ var state tapState
 func RunMizuTap() {
 	mizuApiFilteringOptions, err := getMizuApiFilteringOptions()
 	if err != nil {
-		mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error parsing regex-masking: %v", errormessage.FormatError(err)))
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error parsing regex-masking: %v", errormessage.FormatError(err)))
 		return
 	}
 	var mizuValidationRules string
-	if mizu.Config.Tap.EnforcePolicyFile != "" {
-		mizuValidationRules, err = readValidationRules(mizu.Config.Tap.EnforcePolicyFile)
+	if config.Config.Tap.EnforcePolicyFile != "" {
+		mizuValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile)
 		if err != nil {
-			mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading policy file: %v", errormessage.FormatError(err)))
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading policy file: %v", errormessage.FormatError(err)))
 			return
 		}
 	}
-
-	kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.KubeConfigPath)
+	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
 	if err != nil {
-		mizu.Log.Error(err)
+		logger.Log.Error(err)
 		return
 	}
 
@@ -69,41 +64,49 @@ func RunMizuTap() {
 
 	targetNamespaces := getNamespaces(kubernetesProvider)
 
+	if config.Config.IsNsRestrictedMode() {
+		if len(targetNamespaces) != 1 || !mizu.Contains(targetNamespaces, config.Config.MizuResourcesNamespace) {
+			logger.Log.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+
+				"You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, config.MizuResourcesNamespaceConfigName)
+			return
+		}
+	}
+
 	var namespacesStr string
-	if targetNamespaces[0] != mizu.K8sAllNamespaces {
+	if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
 		namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(targetNamespaces, "\", \""))
 	} else {
 		namespacesStr = "all namespaces"
 	}
-	mizu.CheckNewerVersion()
-	mizu.Log.Infof("Tapping pods in %s", namespacesStr)
+
+	logger.Log.Infof("Tapping pods in %s", namespacesStr)
 
 	if err, _ := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespaces); err != nil {
-		mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error getting pods by regex: %v", errormessage.FormatError(err)))
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error getting pods by regex: %v", errormessage.FormatError(err)))
 		return
 	}
 
 	if len(state.currentlyTappedPods) == 0 {
 		var suggestionStr string
-		if targetNamespaces[0] != mizu.K8sAllNamespaces {
+		if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
 			suggestionStr = ". Select a different namespace with -n or tap all namespaces with -A"
 		}
-		mizu.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr))
+		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr))
 	}
 
-	if mizu.Config.Tap.DryRun {
+	if config.Config.Tap.DryRun {
 		return
 	}
 
 	nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
 
-	defer cleanUpMizuResources(kubernetesProvider)
+	defer finishMizuExecution(kubernetesProvider)
 	if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil {
-		mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
 		return
 	}
 
-	go goUtils.HandleExcWrapper(createProxyToApiServerPod, ctx, kubernetesProvider, cancel)
+	go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel)
 	go goUtils.HandleExcWrapper(watchPodsForTapping, ctx, kubernetesProvider, targetNamespaces, cancel)
 
 	//block until exit signal or error
@@ -120,7 +123,7 @@ func readValidationRules(file string) (string, error) {
 }
 
 func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, mizuApiFilteringOptions *shared.TrafficFilteringOptions, mizuValidationRules string) error {
-	if !mizu.Config.IsNsRestrictedMode() {
+	if !config.Config.IsNsRestrictedMode() {
 		if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
 			return err
 		}
@@ -135,7 +138,7 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	}
 
 	if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules); err != nil {
-		mizu.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err)))
+		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err)))
 		state.doNotRemoveConfigMap = true
 	} else if mizuValidationRules == "" {
 		state.doNotRemoveConfigMap = true
@@ -145,12 +148,12 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro
 }
 
 func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, data string) error {
-	err := kubernetesProvider.CreateConfigMap(ctx, mizu.Config.MizuResourcesNamespace, mizu.ConfigMapName, data)
+	err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName, data)
 	return err
 }
 
 func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider) error {
-	_, err := kubernetesProvider.CreateNamespace(ctx, mizu.Config.MizuResourcesNamespace)
+	_, err := kubernetesProvider.CreateNamespace(ctx, config.Config.MizuResourcesNamespace)
 	return err
 }
 
@@ -159,7 +162,7 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro
 
 	state.mizuServiceAccountExists, err = createRBACIfNecessary(ctx, kubernetesProvider)
 	if err != nil {
-		mizu.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err)))
+		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err)))
 	}
 
 	var serviceAccountName string
@@ -170,25 +173,27 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	}
 
 	opts := &kubernetes.ApiServerOptions{
-		Namespace:               mizu.Config.MizuResourcesNamespace,
+		Namespace:               config.Config.MizuResourcesNamespace,
 		PodName:                 mizu.ApiServerPodName,
-		PodImage:                mizu.Config.AgentImage,
+		PodImage:                config.Config.AgentImage,
 		ServiceAccountName:      serviceAccountName,
-		IsNamespaceRestricted:   mizu.Config.IsNsRestrictedMode(),
+		IsNamespaceRestricted:   config.Config.IsNsRestrictedMode(),
 		MizuApiFilteringOptions: mizuApiFilteringOptions,
-		MaxEntriesDBSizeBytes:   mizu.Config.Tap.MaxEntriesDBSizeBytes(),
+		MaxEntriesDBSizeBytes:   config.Config.Tap.MaxEntriesDBSizeBytes(),
+		Resources:               config.Config.Tap.ApiServerResources,
+		ImagePullPolicy:         config.Config.ImagePullPolicy(),
 	}
 	_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, opts)
 	if err != nil {
 		return err
 	}
-	mizu.Log.Debugf("Successfully created API server pod: %s", mizu.ApiServerPodName)
+	logger.Log.Debugf("Successfully created API server pod: %s", mizu.ApiServerPodName)
 
-	state.apiServerService, err = kubernetesProvider.CreateService(ctx, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName, mizu.ApiServerPodName)
+	state.apiServerService, err = kubernetesProvider.CreateService(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName, mizu.ApiServerPodName)
 	if err != nil {
 		return err
 	}
-	mizu.Log.Debugf("Successfully created service: %s", mizu.ApiServerPodName)
+	logger.Log.Debugf("Successfully created service: %s", mizu.ApiServerPodName)
 
 	return nil
 }
@@ -196,9 +201,9 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro
 func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
 	var compiledRegexSlice []*shared.SerializableRegexp
 
-	if mizu.Config.Tap.PlainTextFilterRegexes != nil && len(mizu.Config.Tap.PlainTextFilterRegexes) > 0 {
+	if config.Config.Tap.PlainTextFilterRegexes != nil && len(config.Config.Tap.PlainTextFilterRegexes) > 0 {
 		compiledRegexSlice = make([]*shared.SerializableRegexp, 0)
-		for _, regexStr := range mizu.Config.Tap.PlainTextFilterRegexes {
+		for _, regexStr := range config.Config.Tap.PlainTextFilterRegexes {
 			compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr)
 			if err != nil {
 				return nil, err
@@ -209,8 +214,8 @@ func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
 
 	return &shared.TrafficFilteringOptions{
 		PlainTextMaskingRegexes:      compiledRegexSlice,
-		HealthChecksUserAgentHeaders: mizu.Config.Tap.HealthChecksUserAgentHeaders,
-		DisableRedaction:             mizu.Config.Tap.DisableRedaction,
+		HealthChecksUserAgentHeaders: config.Config.Tap.HealthChecksUserAgentHeaders,
+		DisableRedaction:             config.Config.Tap.DisableRedaction,
 	}, nil
 }
 
@@ -225,20 +230,22 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
 
 		if err := kubernetesProvider.ApplyMizuTapperDaemonSet(
 			ctx,
-			mizu.Config.MizuResourcesNamespace,
+			config.Config.MizuResourcesNamespace,
 			mizu.TapperDaemonSetName,
-			mizu.Config.AgentImage,
+			config.Config.AgentImage,
 			mizu.TapperPodName,
 			fmt.Sprintf("%s.%s.svc.cluster.local", state.apiServerService.Name, state.apiServerService.Namespace),
 			nodeToTappedPodIPMap,
 			serviceAccountName,
-			mizu.Config.Tap.TapOutgoing(),
+			config.Config.Tap.TapOutgoing(),
+			config.Config.Tap.TapperResources,
+			config.Config.ImagePullPolicy(),
 		); err != nil {
 			return err
 		}
-		mizu.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap))
+		logger.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap))
 	} else {
-		if err := kubernetesProvider.RemoveDaemonSet(ctx, mizu.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
+		if err := kubernetesProvider.RemoveDaemonSet(ctx, config.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
 			return err
 		}
 	}
@@ -246,70 +253,77 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
 	return nil
 }
 
-func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
-
+func finishMizuExecution(kubernetesProvider *kubernetes.Provider) {
+	telemetry.ReportAPICalls()
 	removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
 	defer cancel()
+	dumpLogsIfNeeded(kubernetesProvider, removalCtx)
+	cleanUpMizuResources(kubernetesProvider, removalCtx, cancel)
+}
 
-	if mizu.Config.DumpLogs {
-		mizuDir := mizu.GetMizuFolderPath()
-		filePath = path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
-		if err := fsUtils.DumpLogs(kubernetesProvider, removalCtx, filePath); err != nil {
-			mizu.Log.Errorf("Failed dump logs %v", err)
-		}
+func dumpLogsIfNeeded(kubernetesProvider *kubernetes.Provider, removalCtx context.Context) {
+	if !config.Config.DumpLogs {
+		return
 	}
+	mizuDir := mizu.GetMizuFolderPath()
+	filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
+	if err := fsUtils.DumpLogs(kubernetesProvider, removalCtx, filePath); err != nil {
+		logger.Log.Errorf("Failed dump logs %v", err)
+	}
+}
 
-	mizu.Log.Infof("\nRemoving mizu resources\n")
+func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider, removalCtx context.Context, cancel context.CancelFunc) {
+	logger.Log.Infof("\nRemoving mizu resources\n")
 
-	if !mizu.Config.IsNsRestrictedMode() {
-		if err := kubernetesProvider.RemoveNamespace(removalCtx, mizu.Config.MizuResourcesNamespace); err != nil {
-			mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Namespace %s: %v", mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+	if !config.Config.IsNsRestrictedMode() {
+		if err := kubernetesProvider.RemoveNamespace(removalCtx, config.Config.MizuResourcesNamespace); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Namespace %s: %v", config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
 			return
 		}
 	} else {
-		if err := kubernetesProvider.RemovePod(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
-			mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Pod %s in namespace %s: %v", mizu.ApiServerPodName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+		if err := kubernetesProvider.RemovePod(removalCtx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Pod %s in namespace %s: %v", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
 		}
 
-		if err := kubernetesProvider.RemoveService(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
-			mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service %s in namespace %s: %v", mizu.ApiServerPodName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+		if err := kubernetesProvider.RemoveService(removalCtx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service %s in namespace %s: %v", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
 		}
 
-		if err := kubernetesProvider.RemoveDaemonSet(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
-			mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing DaemonSet %s in namespace %s: %v", mizu.TapperDaemonSetName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+		if err := kubernetesProvider.RemoveDaemonSet(removalCtx, config.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing DaemonSet %s in namespace %s: %v", mizu.TapperDaemonSetName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
 		}
 
 		if !state.doNotRemoveConfigMap {
-			if err := kubernetesProvider.RemoveConfigMap(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.ConfigMapName); err != nil {
-				mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing ConfigMap %s in namespace %s: %v", mizu.ConfigMapName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			if err := kubernetesProvider.RemoveConfigMap(removalCtx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing ConfigMap %s in namespace %s: %v", mizu.ConfigMapName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
 			}
 		}
 
 	}
 
 	if state.mizuServiceAccountExists {
-		if !mizu.Config.IsNsRestrictedMode() {
+		if !config.Config.IsNsRestrictedMode() {
 			if err := kubernetesProvider.RemoveNonNamespacedResources(removalCtx, mizu.ClusterRoleName, mizu.ClusterRoleBindingName); err != nil {
-				mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing non-namespaced resources: %v", errormessage.FormatError(err)))
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing non-namespaced resources: %v", errormessage.FormatError(err)))
 				return
 			}
 		} else {
-			if err := kubernetesProvider.RemoveServicAccount(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.ServiceAccountName); err != nil {
-				mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service Account %s in namespace %s: %v", mizu.ServiceAccountName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			if err := kubernetesProvider.RemoveServicAccount(removalCtx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service Account %s in namespace %s: %v", mizu.ServiceAccountName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
 				return
 			}
 
-			if err := kubernetesProvider.RemoveRole(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.RoleName); err != nil {
-				mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Role %s in namespace %s: %v", mizu.RoleName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			if err := kubernetesProvider.RemoveRole(removalCtx, config.Config.MizuResourcesNamespace, mizu.RoleName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Role %s in namespace %s: %v", mizu.RoleName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
 			}
 
-			if err := kubernetesProvider.RemoveRoleBinding(removalCtx, mizu.Config.MizuResourcesNamespace, mizu.RoleBindingName); err != nil {
-				mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing RoleBinding %s in namespace %s: %v", mizu.RoleBindingName, mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			if err := kubernetesProvider.RemoveRoleBinding(removalCtx, config.Config.MizuResourcesNamespace, mizu.RoleBindingName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing RoleBinding %s in namespace %s: %v", mizu.RoleBindingName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
 			}
 		}
 	}
 
-	if !mizu.Config.IsNsRestrictedMode() {
+	if !config.Config.IsNsRestrictedMode() {
 		waitUntilNamespaceDeleted(removalCtx, cancel, kubernetesProvider)
 	}
 }
@@ -320,65 +334,44 @@ func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, k
 		waitForFinish(ctx, cancel)
 	}()
 
-	if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, mizu.Config.MizuResourcesNamespace); err != nil {
+	if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, config.Config.MizuResourcesNamespace); err != nil {
 		switch {
 		case ctx.Err() == context.Canceled:
-			// Do nothing. User interrupted the wait.
+			logger.Log.Debugf("Do nothing. User interrupted the wait")
 		case err == wait.ErrWaitTimeout:
-			mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", mizu.Config.MizuResourcesNamespace))
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", config.Config.MizuResourcesNamespace))
 		default:
-			mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error while waiting for Namespace %s to be deleted: %v", mizu.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
-		}
-	}
-}
-
-func reportTappedPods() {
-	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort)
-	tappedPodsUrl := fmt.Sprintf("http://%s/status/tappedPods", mizuProxiedUrl)
-
-	podInfos := make([]shared.PodInfo, 0)
-	for _, pod := range state.currentlyTappedPods {
-		podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
-	}
-	tapStatus := shared.TapStatus{Pods: podInfos}
-
-	if jsonValue, err := json.Marshal(tapStatus); err != nil {
-		mizu.Log.Debugf("[ERROR] failed Marshal the tapped pods %v", err)
-	} else {
-		if response, err := http.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
-			mizu.Log.Debugf("[ERROR] failed sending to API server the tapped pods %v", err)
-		} else if response.StatusCode != 200 {
-			mizu.Log.Debugf("[ERROR] failed sending to API server the tapped pods, response status code %v", response.StatusCode)
-		} else {
-			mizu.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos))
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error while waiting for Namespace %s to be deleted: %v", config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
 		}
 	}
 }
 
 func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, targetNamespaces []string, cancel context.CancelFunc) {
-	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, targetNamespaces, mizu.Config.Tap.PodRegex())
+	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, targetNamespaces, config.Config.Tap.PodRegex())
 
 	restartTappers := func() {
 		err, changeFound := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespaces)
 		if err != nil {
-			mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Failed to update currently tapped pods: %v", err))
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Failed to update currently tapped pods: %v", err))
 			cancel()
 		}
 
 		if !changeFound {
-			mizu.Log.Debugf("Nothing changed update tappers not needed")
+			logger.Log.Debugf("Nothing changed update tappers not needed")
 			return
 		}
 
-		reportTappedPods()
+		if err := apiserver.Provider.ReportTappedPods(state.currentlyTappedPods); err != nil {
+			logger.Log.Debugf("[Error] failed update tapped pods %v", err)
+		}
 
 		nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
 		if err != nil {
-			mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error building node to ips map: %v", errormessage.FormatError(err)))
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error building node to ips map: %v", errormessage.FormatError(err)))
 			cancel()
 		}
 		if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
-			mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating daemonset: %v", errormessage.FormatError(err)))
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating daemonset: %v", errormessage.FormatError(err)))
 			cancel()
 		}
 	}
@@ -387,13 +380,13 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	for {
 		select {
 		case pod := <-added:
-			mizu.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
+			logger.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
 			restartTappersDebouncer.SetOn()
 		case pod := <-removed:
-			mizu.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
+			logger.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
 			restartTappersDebouncer.SetOn()
 		case pod := <-modified:
-			mizu.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
+			logger.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
 			// Act only if the modified pod has already obtained an IP address.
 			// After filtering for IPs, on a normal pod restart this includes the following events:
 			// - Pod deletion
@@ -405,13 +398,13 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 			}
 
 		case err := <-errorChan:
-			mizu.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err)
+			logger.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err)
 			restartTappersDebouncer.Cancel()
 			// TODO: Does this also perform cleanup?
 			cancel()
 
 		case <-ctx.Done():
-			mizu.Log.Debugf("Watching pods loop, context done, stopping `restart tappers debouncer`")
+			logger.Log.Debugf("Watching pods loop, context done, stopping `restart tappers debouncer`")
 			restartTappersDebouncer.Cancel()
 			return
 		}
@@ -420,18 +413,18 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 
 func updateCurrentlyTappedPods(kubernetesProvider *kubernetes.Provider, ctx context.Context, targetNamespaces []string) (error, bool) {
 	changeFound := false
-	if matchingPods, err := kubernetesProvider.ListAllRunningPodsMatchingRegex(ctx, mizu.Config.Tap.PodRegex(), targetNamespaces); err != nil {
+	if matchingPods, err := kubernetesProvider.ListAllRunningPodsMatchingRegex(ctx, config.Config.Tap.PodRegex(), targetNamespaces); err != nil {
 		return err, false
 	} else {
 		podsToTap := excludeMizuPods(matchingPods)
 		addedPods, removedPods := getPodArrayDiff(state.currentlyTappedPods, podsToTap)
 		for _, addedPod := range addedPods {
 			changeFound = true
-			mizu.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", addedPod.Name))
+			logger.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", addedPod.Name))
 		}
 		for _, removedPod := range removedPods {
 			changeFound = true
-			mizu.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedPod.Name))
+			logger.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedPod.Name))
 		}
 		state.currentlyTappedPods = podsToTap
 	}
@@ -477,88 +470,73 @@ func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod {
 	return missingPods
 }
 
-func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName))
-	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{mizu.Config.MizuResourcesNamespace}, podExactRegex)
+	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
 	isPodReady := false
 	timeAfter := time.After(25 * time.Second)
 	for {
 		select {
 		case <-ctx.Done():
-			mizu.Log.Debugf("Watching API Server pod loop, ctx done")
+			logger.Log.Debugf("Watching API Server pod loop, ctx done")
 			return
 		case <-added:
-			mizu.Log.Debugf("Watching API Server pod loop, added")
+			logger.Log.Debugf("Watching API Server pod loop, added")
 			continue
 		case <-removed:
-			mizu.Log.Infof("%s removed", mizu.ApiServerPodName)
+			logger.Log.Infof("%s removed", mizu.ApiServerPodName)
 			cancel()
 			return
 		case modifiedPod := <-modified:
 			if modifiedPod == nil {
-				mizu.Log.Debugf("Watching API Server pod loop, modifiedPod with nil")
+				logger.Log.Debugf("Watching API Server pod loop, modifiedPod with nil")
 				continue
 			}
-			mizu.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
+			logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
 			if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
 				isPodReady = true
 				go startProxyReportErrorIfAny(kubernetesProvider, cancel)
-				mizu.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort))
-				time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
-				requestForAnalysis()
-				reportTappedPods()
+
+				if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl(), 20); err != nil {
+					logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, check logs")
+					cancel()
+					break
+				}
+				logger.Log.Infof("Mizu is available at %s\n", GetApiServerUrl())
+				requestForAnalysisIfNeeded()
+				if err := apiserver.Provider.ReportTappedPods(state.currentlyTappedPods); err != nil {
+					logger.Log.Debugf("[Error] failed update tapped pods %v", err)
+				}
 			}
 		case <-timeAfter:
 			if !isPodReady {
-				mizu.Log.Errorf(uiUtils.Error, "Mizu API server was not ready in time")
+				logger.Log.Errorf(uiUtils.Error, "Mizu API server was not ready in time")
 				cancel()
 			}
 		case <-errorChan:
-			mizu.Log.Debugf("[ERROR] Agent creation, watching %v namespace", mizu.Config.MizuResourcesNamespace)
+			logger.Log.Debugf("[ERROR] Agent creation, watching %v namespace", config.Config.MizuResourcesNamespace)
 			cancel()
 		}
 	}
 }
 
-func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
-	err := kubernetes.StartProxy(kubernetesProvider, mizu.Config.Tap.GuiPort, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
-	if err != nil {
-		mizu.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+
-			"Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName))
-		cancel()
-	}
-}
-
-func requestForAnalysis() {
-	if !mizu.Config.Tap.Analysis {
+func requestForAnalysisIfNeeded() {
+	if !config.Config.Tap.Analysis {
 		return
 	}
-
-	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort)
-	urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(mizu.Config.Tap.AnalysisDestination), mizu.Config.Tap.SleepIntervalSec)
-	u, parseErr := url.ParseRequestURI(urlPath)
-	if parseErr != nil {
-		mizu.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr)
-	}
-
-	mizu.Log.Debugf("Sending get request to %v", u.String())
-	if response, requestErr := http.Get(u.String()); requestErr != nil {
-		mizu.Log.Errorf("Failed to notify agent for analysis, err: %v", requestErr)
-	} else if response.StatusCode != 200 {
-		mizu.Log.Errorf("Failed to notify agent for analysis, status code: %v", response.StatusCode)
-	} else {
-		mizu.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis")
+	if err := apiserver.Provider.RequestAnalysis(config.Config.Tap.AnalysisDestination, config.Config.Tap.SleepIntervalSec); err != nil {
+		logger.Log.Debugf("[Error] failed requesting for analysis %v", err)
 	}
 }
 
 func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) (bool, error) {
-	if !mizu.Config.IsNsRestrictedMode() {
-		err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.ClusterRoleName, mizu.ClusterRoleBindingName, mizu.RBACVersion)
+	if !config.Config.IsNsRestrictedMode() {
+		err := kubernetesProvider.CreateMizuRBAC(ctx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.ClusterRoleName, mizu.ClusterRoleBindingName, mizu.RBACVersion)
 		if err != nil {
 			return false, err
 		}
 	} else {
-		err := kubernetesProvider.CreateMizuRBACNamespaceRestricted(ctx, mizu.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.RoleName, mizu.RoleBindingName, mizu.RBACVersion)
+		err := kubernetesProvider.CreateMizuRBACNamespaceRestricted(ctx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.RoleName, mizu.RoleBindingName, mizu.RBACVersion)
 		if err != nil {
 			return false, err
 		}
@@ -579,24 +557,11 @@ func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) map[string][]string {
 	return nodeToTappedPodIPMap
 }
 
-func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
-	sigChan := make(chan os.Signal, 1)
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
-
-	// block until ctx cancel is called or termination signal is received
-	select {
-	case <-ctx.Done():
-		break
-	case <-sigChan:
-		cancel()
-	}
-}
-
 func getNamespaces(kubernetesProvider *kubernetes.Provider) []string {
-	if mizu.Config.Tap.AllNamespaces {
+	if config.Config.Tap.AllNamespaces {
 		return []string{mizu.K8sAllNamespaces}
-	} else if len(mizu.Config.Tap.Namespaces) > 0 {
-		return mizu.Config.Tap.Namespaces
+	} else if len(config.Config.Tap.Namespaces) > 0 {
+		return mizu.Unique(config.Config.Tap.Namespaces)
 	} else {
 		return []string{kubernetesProvider.CurrentNamespace()}
 	}

cli/cmd/version.go

@@ -1,27 +1,31 @@
 package cmd
 
 import (
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"strconv"
 	"time"
 
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/cli/mizu/configStructs"
 )
 
 var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print version info",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go mizu.ReportRun("version", mizu.Config.Version)
-		if mizu.Config.Version.DebugInfo {
+		go telemetry.ReportRun("version", config.Config.Version)
+
+		if config.Config.Version.DebugInfo {
 			timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
-			mizu.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
-			mizu.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
+			logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
+			logger.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
 
 		} else {
-			mizu.Log.Infof("Version: %s (%s)", mizu.SemVer, mizu.Branch)
+			logger.Log.Infof("Version: %s (%s)", mizu.SemVer, mizu.Branch)
 		}
 		return nil
 	},

cli/cmd/view.go

@@ -3,15 +3,16 @@ package cmd
 import (
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/cli/mizu/configStructs"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/telemetry"
 )
 
 var viewCmd = &cobra.Command{
 	Use:   "view",
 	Short: "Open GUI in browser",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go mizu.ReportRun("view", mizu.Config.View)
+		go telemetry.ReportRun("view", config.Config.View)
 		runMizuView()
 		return nil
 	},
@@ -24,5 +25,4 @@ func init() {
 	defaults.Set(&defaultViewConfig)
 
 	viewCmd.Flags().Uint16P(configStructs.GuiPortViewName, "p", defaultViewConfig.GuiPort, "Provide a custom port for the web interface webserver")
-	viewCmd.Flags().StringP(configStructs.KubeConfigPathViewName, "k", defaultViewConfig.KubeConfigPath, "Path to kube-config file")
 }

cli/cmd/viewRunner.go

@@ -3,46 +3,54 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/version"
+	"github.com/up9inc/mizu/cli/uiUtils"
 	"net/http"
 )
 
 func runMizuView() {
-	kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.View.KubeConfigPath)
+	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
 	if err != nil {
-		mizu.Log.Error(err)
+		logger.Log.Error(err)
 		return
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	exists, err := kubernetesProvider.DoesServicesExist(ctx, mizu.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
+	exists, err := kubernetesProvider.DoesServicesExist(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
 	if err != nil {
-		mizu.Log.Errorf("Failed to found mizu service %v", err)
+		logger.Log.Errorf("Failed to found mizu service %v", err)
 		cancel()
 		return
 	}
 	if !exists {
-		mizu.Log.Infof("%s service not found, you should run `mizu tap` command first", mizu.ApiServerPodName)
+		logger.Log.Infof("%s service not found, you should run `mizu tap` command first", mizu.ApiServerPodName)
 		cancel()
 		return
 	}
 
-	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.View.GuiPort)
-	_, err = http.Get(fmt.Sprintf("http://%s/", mizuProxiedUrl))
-	if err == nil {
-		mizu.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, mizu.Config.View.GuiPort)
+	response, err := http.Get(fmt.Sprintf("%s/", GetApiServerUrl()))
+	if err == nil && response.StatusCode == 200 {
+		logger.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, config.Config.View.GuiPort)
 		return
 	}
-	mizu.Log.Debugf("Found service %s, creating k8s proxy", mizu.ApiServerPodName)
-
+	logger.Log.Infof("Establishing connection to k8s cluster...")
 	go startProxyReportErrorIfAny(kubernetesProvider, cancel)
 
-	mizu.Log.Infof("Mizu is available at  http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.View.GuiPort))
-	if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.View.GuiPort); err != nil {
-		mizu.Log.Errorf("Failed to check versions compatibility %v", err)
+	if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl(), 10); err != nil {
+		logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, check logs")
+		return
+	}
+
+	logger.Log.Infof("Mizu is available at %s\n", GetApiServerUrl())
+	if isCompatible, err := version.CheckVersionCompatibility(); err != nil {
+		logger.Log.Errorf("Failed to check versions compatibility %v", err)
 		cancel()
 		return
 	} else if !isCompatible {

cli/config/config.go

@@ -0,0 +1,327 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/creasty/defaults"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"gopkg.in/yaml.v3"
+)
+
+const (
+	Separator      = "="
+	SetCommandName = "set"
+	FieldNameTag   = "yaml"
+	ReadonlyTag    = "readonly"
+)
+
+var (
+	Config  = ConfigStruct{}
+	cmdName string
+)
+
+func InitConfig(cmd *cobra.Command) error {
+	cmdName = cmd.Name()
+
+	if err := defaults.Set(&Config); err != nil {
+		return err
+	}
+
+	configFilePath := cmd.Flags().Lookup(ConfigFilePathCommandName).Value.String()
+
+	if err := mergeConfigFile(configFilePath); err != nil {
+		return fmt.Errorf("invalid config, %w\n"+
+			"you can regenerate the file by removing it (%v) and using `mizu config -r`", err, configFilePath)
+	}
+
+	cmd.Flags().Visit(initFlag)
+
+	finalConfigPrettified, _ := uiUtils.PrettyJson(Config)
+	logger.Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified)
+
+	return nil
+}
+
+func GetConfigWithDefaults() (string, error) {
+	defaultConf := ConfigStruct{}
+	if err := defaults.Set(&defaultConf); err != nil {
+		return "", err
+	}
+
+	configElem := reflect.ValueOf(&defaultConf).Elem()
+	setZeroForReadonlyFields(configElem)
+
+	return uiUtils.PrettyYaml(defaultConf)
+}
+
+func mergeConfigFile(configFilePath string) error {
+	reader, openErr := os.Open(configFilePath)
+	if openErr != nil {
+		return nil
+	}
+
+	buf, readErr := ioutil.ReadAll(reader)
+	if readErr != nil {
+		return readErr
+	}
+
+	if err := yaml.Unmarshal(buf, &Config); err != nil {
+		return err
+	}
+	logger.Log.Debugf("Found config file, merged to default options")
+
+	return nil
+}
+
+func initFlag(f *pflag.Flag) {
+	configElemValue := reflect.ValueOf(&Config).Elem()
+
+	var flagPath []string
+	if mizu.Contains([]string{ConfigFilePathCommandName}, f.Name) {
+		flagPath = []string{f.Name}
+	} else {
+		flagPath = []string{cmdName, f.Name}
+	}
+
+	sliceValue, isSliceValue := f.Value.(pflag.SliceValue)
+	if !isSliceValue {
+		if err := mergeFlagValue(configElemValue, flagPath, strings.Join(flagPath, "."), f.Value.String()); err != nil {
+			logger.Log.Warningf(uiUtils.Warning, err)
+		}
+		return
+	}
+
+	if f.Name == SetCommandName {
+		if err := mergeSetFlag(configElemValue, sliceValue.GetSlice()); err != nil {
+			logger.Log.Warningf(uiUtils.Warning, err)
+		}
+		return
+	}
+
+	if err := mergeFlagValues(configElemValue, flagPath, strings.Join(flagPath, "."), sliceValue.GetSlice()); err != nil {
+		logger.Log.Warningf(uiUtils.Warning, err)
+	}
+}
+
+func mergeSetFlag(configElemValue reflect.Value, setValues []string) error {
+	var setErrors []string
+	setMap := map[string][]string{}
+
+	for _, setValue := range setValues {
+		if !strings.Contains(setValue, Separator) {
+			setErrors = append(setErrors, fmt.Sprintf("Ignoring set argument %s (set argument format: <flag name>=<flag value>)", setValue))
+			continue
+		}
+
+		split := strings.SplitN(setValue, Separator, 2)
+		argumentKey, argumentValue := split[0], split[1]
+
+		setMap[argumentKey] = append(setMap[argumentKey], argumentValue)
+	}
+
+	for argumentKey, argumentValues := range setMap {
+		flagPath := strings.Split(argumentKey, ".")
+
+		if len(argumentValues) > 1 {
+			if err := mergeFlagValues(configElemValue, flagPath, argumentKey, argumentValues); err != nil {
+				setErrors = append(setErrors, fmt.Sprintf("%v", err))
+			}
+		} else {
+			if err := mergeFlagValue(configElemValue, flagPath, argumentKey, argumentValues[0]); err != nil {
+				setErrors = append(setErrors, fmt.Sprintf("%v", err))
+			}
+		}
+	}
+
+	if len(setErrors) > 0 {
+		return fmt.Errorf(strings.Join(setErrors, "\n"))
+	}
+
+	return nil
+}
+
+func mergeFlagValue(configElemValue reflect.Value, flagPath []string, fullFlagName string, flagValue string) error {
+	mergeFunction := func(flagName string, currentFieldStruct reflect.StructField, currentFieldElemValue reflect.Value, currentElemValue reflect.Value) error {
+		currentFieldKind := currentFieldStruct.Type.Kind()
+
+		if currentFieldKind == reflect.Slice {
+			return mergeFlagValues(currentElemValue, []string{flagName}, fullFlagName, []string{flagValue})
+		}
+
+		parsedValue, err := getParsedValue(currentFieldKind, flagValue)
+		if err != nil {
+			return fmt.Errorf("invalid value %s for flag name %s, expected %s", flagValue, flagName, currentFieldKind)
+		}
+
+		currentFieldElemValue.Set(parsedValue)
+		return nil
+	}
+
+	return mergeFlag(configElemValue, flagPath, fullFlagName, mergeFunction)
+}
+
+func mergeFlagValues(configElemValue reflect.Value, flagPath []string, fullFlagName string, flagValues []string) error {
+	mergeFunction := func(flagName string, currentFieldStruct reflect.StructField, currentFieldElemValue reflect.Value, currentElemValue reflect.Value) error {
+		currentFieldKind := currentFieldStruct.Type.Kind()
+
+		if currentFieldKind != reflect.Slice {
+			return fmt.Errorf("invalid values %s for flag name %s, expected %s", strings.Join(flagValues, ","), flagName, currentFieldKind)
+		}
+
+		flagValueKind := currentFieldStruct.Type.Elem().Kind()
+
+		parsedValues := reflect.MakeSlice(reflect.SliceOf(currentFieldStruct.Type.Elem()), 0, 0)
+		for _, flagValue := range flagValues {
+			parsedValue, err := getParsedValue(flagValueKind, flagValue)
+			if err != nil {
+				return fmt.Errorf("invalid value %s for flag name %s, expected %s", flagValue, flagName, flagValueKind)
+			}
+
+			parsedValues = reflect.Append(parsedValues, parsedValue)
+		}
+
+		currentFieldElemValue.Set(parsedValues)
+		return nil
+	}
+
+	return mergeFlag(configElemValue, flagPath, fullFlagName, mergeFunction)
+}
+
+func mergeFlag(currentElemValue reflect.Value, currentFlagPath []string, fullFlagName string, mergeFunction func(flagName string, currentFieldStruct reflect.StructField, currentFieldElemValue reflect.Value, currentElemValue reflect.Value) error) error {
+	if len(currentFlagPath) == 0 {
+		return fmt.Errorf("flag \"%s\" not found", fullFlagName)
+	}
+
+	for i := 0; i < currentElemValue.NumField(); i++ {
+		currentFieldStruct := currentElemValue.Type().Field(i)
+		currentFieldElemValue := currentElemValue.FieldByName(currentFieldStruct.Name)
+
+		if currentFieldStruct.Type.Kind() == reflect.Struct && getFieldNameByTag(currentFieldStruct) == currentFlagPath[0] {
+			return mergeFlag(currentFieldElemValue, currentFlagPath[1:], fullFlagName, mergeFunction)
+		}
+
+		if len(currentFlagPath) > 1 || getFieldNameByTag(currentFieldStruct) != currentFlagPath[0] {
+			continue
+		}
+
+		return mergeFunction(currentFlagPath[0], currentFieldStruct, currentFieldElemValue, currentElemValue)
+	}
+
+	return fmt.Errorf("flag \"%s\" not found", fullFlagName)
+}
+
+func getFieldNameByTag(field reflect.StructField) string {
+	return strings.Split(field.Tag.Get(FieldNameTag), ",")[0]
+}
+
+func getParsedValue(kind reflect.Kind, value string) (reflect.Value, error) {
+	switch kind {
+	case reflect.String:
+		return reflect.ValueOf(value), nil
+	case reflect.Bool:
+		boolArgumentValue, err := strconv.ParseBool(value)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(boolArgumentValue), nil
+	case reflect.Int:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(int(intArgumentValue)), nil
+	case reflect.Int8:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 8)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(int8(intArgumentValue)), nil
+	case reflect.Int16:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 16)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(int16(intArgumentValue)), nil
+	case reflect.Int32:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 32)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(int32(intArgumentValue)), nil
+	case reflect.Int64:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(intArgumentValue), nil
+	case reflect.Uint:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uint(uintArgumentValue)), nil
+	case reflect.Uint8:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 8)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uint8(uintArgumentValue)), nil
+	case reflect.Uint16:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 16)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uint16(uintArgumentValue)), nil
+	case reflect.Uint32:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 32)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uint32(uintArgumentValue)), nil
+	case reflect.Uint64:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uintArgumentValue), nil
+	}
+
+	return reflect.ValueOf(nil), errors.New("value to parse does not match type")
+}
+
+func setZeroForReadonlyFields(currentElem reflect.Value) {
+	for i := 0; i < currentElem.NumField(); i++ {
+		currentField := currentElem.Type().Field(i)
+		currentFieldByName := currentElem.FieldByName(currentField.Name)
+
+		if currentField.Type.Kind() == reflect.Struct {
+			setZeroForReadonlyFields(currentFieldByName)
+			continue
+		}
+
+		if _, ok := currentField.Tag.Lookup(ReadonlyTag); ok {
+			currentFieldByName.Set(reflect.Zero(currentField.Type))
+		}
+	}
+}

cli/config/configStruct.go

@@ -0,0 +1,60 @@
+package config
+
+import (
+	"fmt"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/mizu"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/util/homedir"
+	"os"
+	"path"
+	"path/filepath"
+)
+
+const (
+	MizuResourcesNamespaceConfigName = "mizu-resources-namespace"
+	ConfigFilePathCommandName        = "config-path"
+)
+
+type ConfigStruct struct {
+	Tap                    configStructs.TapConfig     `yaml:"tap"`
+	Fetch                  configStructs.FetchConfig   `yaml:"fetch"`
+	Version                configStructs.VersionConfig `yaml:"version"`
+	View                   configStructs.ViewConfig    `yaml:"view"`
+	Logs                   configStructs.LogsConfig    `yaml:"logs"`
+	Config                 configStructs.ConfigConfig  `yaml:"config,omitempty"`
+	AgentImage             string                      `yaml:"agent-image,omitempty" readonly:""`
+	ImagePullPolicyStr     string                      `yaml:"image-pull-policy" default:"Always"`
+	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
+	Telemetry              bool                        `yaml:"telemetry" default:"true"`
+	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`
+	KubeConfigPathStr      string                      `yaml:"kube-config-path"`
+	ConfigFilePath         string                      `yaml:"config-path,omitempty" readonly:""`
+}
+
+func (config *ConfigStruct) SetDefaults() {
+	config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer)
+	config.ConfigFilePath = path.Join(mizu.GetMizuFolderPath(), "config.yaml")
+}
+
+func (config *ConfigStruct) ImagePullPolicy() v1.PullPolicy {
+	return v1.PullPolicy(config.ImagePullPolicyStr)
+}
+
+func (config *ConfigStruct) IsNsRestrictedMode() bool {
+	return config.MizuResourcesNamespace != "mizu" // Notice "mizu" string must match the default MizuResourcesNamespace
+}
+
+func (config *ConfigStruct) KubeConfigPath() string {
+	if config.KubeConfigPathStr != "" {
+		return config.KubeConfigPathStr
+	}
+
+	envKubeConfigPath := os.Getenv("KUBECONFIG")
+	if envKubeConfigPath != "" {
+		return envKubeConfigPath
+	}
+
+	home := homedir.HomeDir()
+	return filepath.Join(home, ".kube", "config")
+}

cli/config/configStructs/configConfig.go

@@ -0,0 +1,9 @@
+package configStructs
+
+const (
+	RegenerateConfigName = "regenerate"
+)
+
+type ConfigConfig struct {
+	Regenerate bool `yaml:"regenerate,omitempty" default:"false" readonly:""`
+}

cli/config/configStructs/logsConfig.go

@@ -0,0 +1,35 @@
+package configStructs
+
+import (
+	"fmt"
+	"os"
+	"path"
+)
+
+const (
+	FileLogsName = "file"
+)
+
+type LogsConfig struct {
+	FileStr string `yaml:"file"`
+}
+
+func (config *LogsConfig) Validate() error {
+	if config.FileStr == "" {
+		_, err := os.Getwd()
+		if err != nil {
+			return fmt.Errorf("failed to get PWD, %v (try using `mizu logs -f <full path dest zip file>)`", err)
+		}
+	}
+
+	return nil
+}
+
+func (config *LogsConfig) FilePath() string {
+	if config.FileStr == "" {
+		pwd, _ := os.Getwd()
+		return path.Join(pwd, "mizu_logs.zip")
+	}
+
+	return config.FileStr
+}

cli/config/configStructs/tapConfig.go

@@ -10,15 +10,12 @@ import (
 )
 
 const (
-	AnalysisDestinationTapName    = "dest"
-	SleepIntervalSecTapName       = "upload-interval"
 	GuiPortTapName                = "gui-port"
 	NamespacesTapName             = "namespaces"
 	AnalysisTapName               = "analysis"
 	AllNamespacesTapName          = "all-namespaces"
 	PlainTextFilterRegexesTapName = "regex-masking"
 	DisableRedactionTapName       = "no-redact"
-	IgnoredUserAgentsTapName      = "ignored-user-agents"
 	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
 	DirectionTapName              = "direction"
 	DryRunTapName                 = "dry-run"
@@ -26,20 +23,29 @@ const (
 )
 
 type TapConfig struct {
-	AnalysisDestination          string   `yaml:"dest" default:"up9.app"`
-	SleepIntervalSec             int      `yaml:"upload-interval" default:"10"`
-	PodRegexStr                  string   `yaml:"regex" default:".*"`
-	GuiPort                      uint16   `yaml:"gui-port" default:"8899"`
-	Namespaces                   []string `yaml:"namespaces"`
-	Analysis                     bool     `yaml:"analysis" default:"false"`
-	AllNamespaces                bool     `yaml:"all-namespaces" default:"false"`
-	PlainTextFilterRegexes       []string `yaml:"regex-masking"`
-	HealthChecksUserAgentHeaders []string `yaml:"ignored-user-agents" default:"[]"`
-	DisableRedaction             bool     `yaml:"no-redact" default:"false"`
-	HumanMaxEntriesDBSize        string   `yaml:"max-entries-db-size" default:"200MB"`
-	Direction                    string   `yaml:"direction" default:"in"`
-	DryRun                       bool     `yaml:"dry-run" default:"false"`
-	EnforcePolicyFile            string   `yaml:"test-rules"`
+	AnalysisDestination          string    `yaml:"dest" default:"up9.app"`
+	SleepIntervalSec             int       `yaml:"upload-interval" default:"10"`
+	PodRegexStr                  string    `yaml:"regex" default:".*"`
+	GuiPort                      uint16    `yaml:"gui-port" default:"8899"`
+	Namespaces                   []string  `yaml:"namespaces"`
+	Analysis                     bool      `yaml:"analysis" default:"false"`
+	AllNamespaces                bool      `yaml:"all-namespaces" default:"false"`
+	PlainTextFilterRegexes       []string  `yaml:"regex-masking"`
+	HealthChecksUserAgentHeaders []string  `yaml:"ignored-user-agents"`
+	DisableRedaction             bool      `yaml:"no-redact" default:"false"`
+	HumanMaxEntriesDBSize        string    `yaml:"max-entries-db-size" default:"200MB"`
+	Direction                    string    `yaml:"direction" default:"in"`
+	DryRun                       bool      `yaml:"dry-run" default:"false"`
+	EnforcePolicyFile            string    `yaml:"test-rules"`
+	ApiServerResources           Resources `yaml:"api-server-resources"`
+	TapperResources              Resources `yaml:"tapper-resources"`
+}
+
+type Resources struct {
+	CpuLimit       string `yaml:"cpu-limit" default:"750m"`
+	MemoryLimit    string `yaml:"memory-limit" default:"1Gi"`
+	CpuRequests    string `yaml:"cpu-requests" default:"50m"`
+	MemoryRequests string `yaml:"memory-requests" default:"50Mi"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {

cli/config/configStructs/viewConfig.go

@@ -0,0 +1,9 @@
+package configStructs
+
+const (
+	GuiPortViewName = "gui-port"
+)
+
+type ViewConfig struct {
+	GuiPort uint16 `yaml:"gui-port" default:"8899"`
+}

cli/config/config_internal_test.go

@@ -0,0 +1,385 @@
+package config
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+type ConfigMock struct {
+	SectionMock      SectionMock `yaml:"section"`
+	Test             string      `yaml:"test"`
+	StringField      string      `yaml:"string-field"`
+	IntField         int         `yaml:"int-field"`
+	BoolField        bool        `yaml:"bool-field"`
+	UintField        uint        `yaml:"uint-field"`
+	StringSliceField []string    `yaml:"string-slice-field"`
+	IntSliceField    []int       `yaml:"int-slice-field"`
+	BoolSliceField   []bool      `yaml:"bool-slice-field"`
+	UintSliceField   []uint      `yaml:"uint-slice-field"`
+}
+
+type SectionMock struct {
+	Test string `yaml:"test"`
+}
+
+type FieldSetValues struct {
+	SetValues  []string
+	FieldName  string
+	FieldValue interface{}
+}
+
+func TestMergeSetFlagNoSeparator(t *testing.T) {
+	tests := []struct {
+		Name      string
+		SetValues []string
+	}{
+		{Name: "empty value", SetValues: []string{""}},
+		{Name: "single char", SetValues: []string{"t"}},
+		{Name: "combine empty value and single char", SetValues: []string{"", "t"}},
+		{Name: "two values without separator", SetValues: []string{"test", "test:true"}},
+		{Name: "four values without separator", SetValues: []string{"test", "test:true", "testing!", "true"}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			configMock := ConfigMock{}
+			configMockElemValue := reflect.ValueOf(&configMock).Elem()
+
+			err := mergeSetFlag(configMockElemValue, test.SetValues)
+
+			if err == nil {
+				t.Errorf("unexpected unhandled error - SetValues: %v", test.SetValues)
+				return
+			}
+
+			for i := 0; i < configMockElemValue.NumField(); i++ {
+				currentField := configMockElemValue.Type().Field(i)
+				currentFieldByName := configMockElemValue.FieldByName(currentField.Name)
+
+				if !currentFieldByName.IsZero() {
+					t.Errorf("unexpected value with not default value - SetValues: %v", test.SetValues)
+				}
+			}
+		})
+	}
+}
+
+func TestMergeSetFlagInvalidFlagName(t *testing.T) {
+	tests := []struct {
+		Name      string
+		SetValues []string
+	}{
+		{Name: "invalid flag name", SetValues: []string{"invalid_flag=true"}},
+		{Name: "invalid flag name inside section struct", SetValues: []string{"section.invalid_flag=test"}},
+		{Name: "flag name is a struct", SetValues: []string{"section=test"}},
+		{Name: "empty flag name", SetValues: []string{"=true"}},
+		{Name: "four tests combined", SetValues: []string{"invalid_flag=true", "config.invalid_flag=test", "section=test", "=true"}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			configMock := ConfigMock{}
+			configMockElemValue := reflect.ValueOf(&configMock).Elem()
+
+			err := mergeSetFlag(configMockElemValue, test.SetValues)
+
+			if err == nil {
+				t.Errorf("unexpected unhandled error - SetValues: %v", test.SetValues)
+				return
+			}
+
+			for i := 0; i < configMockElemValue.NumField(); i++ {
+				currentField := configMockElemValue.Type().Field(i)
+				currentFieldByName := configMockElemValue.FieldByName(currentField.Name)
+
+				if !currentFieldByName.IsZero() {
+					t.Errorf("unexpected case - SetValues: %v", test.SetValues)
+				}
+			}
+		})
+	}
+}
+
+func TestMergeSetFlagInvalidFlagValue(t *testing.T) {
+	tests := []struct {
+		Name      string
+		SetValues []string
+	}{
+		{Name: "bool value to int field", SetValues: []string{"int-field=true"}},
+		{Name: "int value to bool field", SetValues: []string{"bool-field:5"}},
+		{Name: "int value to uint field", SetValues: []string{"uint-field=-1"}},
+		{Name: "bool value to int slice field", SetValues: []string{"int-slice-field=true"}},
+		{Name: "int value to bool slice field", SetValues: []string{"bool-slice-field=5"}},
+		{Name: "int value to uint slice field", SetValues: []string{"uint-slice-field=-1"}},
+		{Name: "int slice value to int field", SetValues: []string{"int-field=6", "int-field=66"}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			configMock := ConfigMock{}
+			configMockElemValue := reflect.ValueOf(&configMock).Elem()
+
+			err := mergeSetFlag(configMockElemValue, test.SetValues)
+
+			if err == nil {
+				t.Errorf("unexpected unhandled error - SetValues: %v", test.SetValues)
+				return
+			}
+
+			for i := 0; i < configMockElemValue.NumField(); i++ {
+				currentField := configMockElemValue.Type().Field(i)
+				currentFieldByName := configMockElemValue.FieldByName(currentField.Name)
+
+				if !currentFieldByName.IsZero() {
+					t.Errorf("unexpected case - SetValues: %v", test.SetValues)
+				}
+			}
+		})
+	}
+}
+
+func TestMergeSetFlagNotSliceValues(t *testing.T) {
+	tests := []struct {
+		Name            string
+		FieldsSetValues []FieldSetValues
+	}{
+		{Name: "string field", FieldsSetValues: []FieldSetValues{{SetValues: []string{"string-field=test"}, FieldName: "StringField", FieldValue: "test"}}},
+		{Name: "int field", FieldsSetValues: []FieldSetValues{{SetValues: []string{"int-field=6"}, FieldName: "IntField", FieldValue: 6}}},
+		{Name: "bool field", FieldsSetValues: []FieldSetValues{{SetValues: []string{"bool-field=true"}, FieldName: "BoolField", FieldValue: true}}},
+		{Name: "uint field", FieldsSetValues: []FieldSetValues{{SetValues: []string{"uint-field=6"}, FieldName: "UintField", FieldValue: uint(6)}}},
+		{Name: "four fields combined", FieldsSetValues: []FieldSetValues {
+			{SetValues: []string{"string-field=test"}, FieldName: "StringField", FieldValue: "test"},
+			{SetValues: []string{"int-field=6"}, FieldName: "IntField", FieldValue: 6},
+			{SetValues: []string{"bool-field=true"}, FieldName: "BoolField", FieldValue: true},
+			{SetValues: []string{"uint-field=6"}, FieldName: "UintField", FieldValue: uint(6)},
+		}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			configMock := ConfigMock{}
+			configMockElemValue := reflect.ValueOf(&configMock).Elem()
+
+			var setValues []string
+			for _, fieldSetValues := range test.FieldsSetValues {
+				setValues = append(setValues, fieldSetValues.SetValues...)
+			}
+
+			err := mergeSetFlag(configMockElemValue, setValues)
+
+			if err != nil {
+				t.Errorf("unexpected error result - err: %v", err)
+				return
+			}
+
+			for _, fieldSetValues := range test.FieldsSetValues {
+				fieldValue := configMockElemValue.FieldByName(fieldSetValues.FieldName).Interface()
+				if fieldValue != fieldSetValues.FieldValue {
+					t.Errorf("unexpected result - expected: %v, actual: %v", fieldSetValues.FieldValue, fieldValue)
+				}
+			}
+		})
+	}
+}
+
+func TestMergeSetFlagSliceValues(t *testing.T) {
+	tests := []struct {
+		Name            string
+		FieldsSetValues []FieldSetValues
+	}{
+		{Name: "string slice field single value", FieldsSetValues: []FieldSetValues{{SetValues: []string{"string-slice-field=test"}, FieldName: "StringSliceField", FieldValue: []string{"test"}}}},
+		{Name: "int slice field single value", FieldsSetValues: []FieldSetValues{{SetValues: []string{"int-slice-field=6"}, FieldName: "IntSliceField", FieldValue: []int{6}}}},
+		{Name: "bool slice field single value", FieldsSetValues: []FieldSetValues{{SetValues: []string{"bool-slice-field=true"}, FieldName: "BoolSliceField", FieldValue: []bool{true}}}},
+		{Name: "uint slice field single value", FieldsSetValues: []FieldSetValues{{SetValues: []string{"uint-slice-field=6"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6)}}}},
+		{Name: "four single value fields combined", FieldsSetValues: []FieldSetValues{
+			{SetValues: []string{"string-slice-field=test"}, FieldName: "StringSliceField", FieldValue: []string{"test"}},
+			{SetValues: []string{"int-slice-field=6"}, FieldName: "IntSliceField", FieldValue: []int{6}},
+			{SetValues: []string{"bool-slice-field=true"}, FieldName: "BoolSliceField", FieldValue: []bool{true}},
+			{SetValues: []string{"uint-slice-field=6"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6)}},
+		}},
+		{Name: "string slice field two values", FieldsSetValues: []FieldSetValues{{SetValues: []string{"string-slice-field=test", "string-slice-field=test2"}, FieldName: "StringSliceField", FieldValue: []string{"test", "test2"}}}},
+		{Name: "int slice field two values", FieldsSetValues: []FieldSetValues{{SetValues: []string{"int-slice-field=6", "int-slice-field=66"}, FieldName: "IntSliceField", FieldValue: []int{6, 66}}}},
+		{Name: "bool slice field two values", FieldsSetValues: []FieldSetValues{{SetValues: []string{"bool-slice-field=true", "bool-slice-field=false"}, FieldName: "BoolSliceField", FieldValue: []bool{true, false}}}},
+		{Name: "uint slice field two values", FieldsSetValues: []FieldSetValues{{SetValues: []string{"uint-slice-field=6", "uint-slice-field=66"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6), uint(66)}}}},
+		{Name: "four two values fields combined", FieldsSetValues: []FieldSetValues{
+			{SetValues: []string{"string-slice-field=test", "string-slice-field=test2"}, FieldName: "StringSliceField", FieldValue: []string{"test", "test2"}},
+			{SetValues: []string{"int-slice-field=6", "int-slice-field=66"}, FieldName: "IntSliceField", FieldValue: []int{6, 66}},
+			{SetValues: []string{"bool-slice-field=true", "bool-slice-field=false"}, FieldName: "BoolSliceField", FieldValue: []bool{true, false}},
+			{SetValues: []string{"uint-slice-field=6", "uint-slice-field=66"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6), uint(66)}},
+		}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			configMock := ConfigMock{}
+			configMockElemValue := reflect.ValueOf(&configMock).Elem()
+
+			var setValues []string
+			for _, fieldSetValues := range test.FieldsSetValues {
+				setValues = append(setValues, fieldSetValues.SetValues...)
+			}
+
+			err := mergeSetFlag(configMockElemValue, setValues)
+
+			if err != nil {
+				t.Errorf("unexpected error result - err: %v", err)
+				return
+			}
+
+			for _, fieldSetValues := range test.FieldsSetValues {
+				fieldValue := configMockElemValue.FieldByName(fieldSetValues.FieldName).Interface()
+				if !reflect.DeepEqual(fieldValue, fieldSetValues.FieldValue) {
+					t.Errorf("unexpected result - expected: %v, actual: %v", fieldSetValues.FieldValue, fieldValue)
+				}
+			}
+		})
+	}
+}
+
+func TestMergeSetFlagMixValues(t *testing.T) {
+	tests := []struct {
+		Name            string
+		FieldsSetValues []FieldSetValues
+	}{
+		{Name: "single value all fields", FieldsSetValues: []FieldSetValues{
+			{SetValues: []string{"string-slice-field=test"}, FieldName: "StringSliceField", FieldValue: []string{"test"}},
+			{SetValues: []string{"int-slice-field=6"}, FieldName: "IntSliceField", FieldValue: []int{6}},
+			{SetValues: []string{"bool-slice-field=true"}, FieldName: "BoolSliceField", FieldValue: []bool{true}},
+			{SetValues: []string{"uint-slice-field=6"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6)}},
+			{SetValues: []string{"string-field=test"}, FieldName: "StringField", FieldValue: "test"},
+			{SetValues: []string{"int-field=6"}, FieldName: "IntField", FieldValue: 6},
+			{SetValues: []string{"bool-field=true"}, FieldName: "BoolField", FieldValue: true},
+			{SetValues: []string{"uint-field=6"}, FieldName: "UintField", FieldValue: uint(6)},
+		}},
+		{Name: "two values slice fields and single value fields", FieldsSetValues: []FieldSetValues{
+			{SetValues: []string{"string-slice-field=test", "string-slice-field=test2"}, FieldName: "StringSliceField", FieldValue: []string{"test", "test2"}},
+			{SetValues: []string{"int-slice-field=6", "int-slice-field=66"}, FieldName: "IntSliceField", FieldValue: []int{6, 66}},
+			{SetValues: []string{"bool-slice-field=true", "bool-slice-field=false"}, FieldName: "BoolSliceField", FieldValue: []bool{true, false}},
+			{SetValues: []string{"uint-slice-field=6", "uint-slice-field=66"}, FieldName: "UintSliceField", FieldValue: []uint{uint(6), uint(66)}},
+			{SetValues: []string{"string-field=test"}, FieldName: "StringField", FieldValue: "test"},
+			{SetValues: []string{"int-field=6"}, FieldName: "IntField", FieldValue: 6},
+			{SetValues: []string{"bool-field=true"}, FieldName: "BoolField", FieldValue: true},
+			{SetValues: []string{"uint-field=6"}, FieldName: "UintField", FieldValue: uint(6)},
+		}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			configMock := ConfigMock{}
+			configMockElemValue := reflect.ValueOf(&configMock).Elem()
+
+			var setValues []string
+			for _, fieldSetValues := range test.FieldsSetValues {
+				setValues = append(setValues, fieldSetValues.SetValues...)
+			}
+
+			err := mergeSetFlag(configMockElemValue, setValues)
+
+			if err != nil {
+				t.Errorf("unexpected error result - err: %v", err)
+				return
+			}
+
+			for _, fieldSetValues := range test.FieldsSetValues {
+				fieldValue := configMockElemValue.FieldByName(fieldSetValues.FieldName).Interface()
+				if !reflect.DeepEqual(fieldValue, fieldSetValues.FieldValue) {
+					t.Errorf("unexpected result - expected: %v, actual: %v", fieldSetValues.FieldValue, fieldValue)
+				}
+			}
+		})
+	}
+}
+
+func TestGetParsedValueValidValue(t *testing.T) {
+	tests := []struct {
+		StringValue string
+		Kind        reflect.Kind
+		ActualValue interface{}
+	}{
+		{StringValue: "test", Kind: reflect.String, ActualValue: "test"},
+		{StringValue: "123", Kind: reflect.String, ActualValue: "123"},
+		{StringValue: "true", Kind: reflect.Bool, ActualValue: true},
+		{StringValue: "false", Kind: reflect.Bool, ActualValue: false},
+		{StringValue: "6", Kind: reflect.Int, ActualValue: 6},
+		{StringValue: "-6", Kind: reflect.Int, ActualValue: -6},
+		{StringValue: "6", Kind: reflect.Int8, ActualValue: int8(6)},
+		{StringValue: "-6", Kind: reflect.Int8, ActualValue: int8(-6)},
+		{StringValue: "6", Kind: reflect.Int16, ActualValue: int16(6)},
+		{StringValue: "-6", Kind: reflect.Int16, ActualValue: int16(-6)},
+		{StringValue: "6", Kind: reflect.Int32, ActualValue: int32(6)},
+		{StringValue: "-6", Kind: reflect.Int32, ActualValue: int32(-6)},
+		{StringValue: "6", Kind: reflect.Int64, ActualValue: int64(6)},
+		{StringValue: "-6", Kind: reflect.Int64, ActualValue: int64(-6)},
+		{StringValue: "6", Kind: reflect.Uint, ActualValue: uint(6)},
+		{StringValue: "66", Kind: reflect.Uint, ActualValue: uint(66)},
+		{StringValue: "6", Kind: reflect.Uint8, ActualValue: uint8(6)},
+		{StringValue: "66", Kind: reflect.Uint8, ActualValue: uint8(66)},
+		{StringValue: "6", Kind: reflect.Uint16, ActualValue: uint16(6)},
+		{StringValue: "66", Kind: reflect.Uint16, ActualValue: uint16(66)},
+		{StringValue: "6", Kind: reflect.Uint32, ActualValue: uint32(6)},
+		{StringValue: "66", Kind: reflect.Uint32, ActualValue: uint32(66)},
+		{StringValue: "6", Kind: reflect.Uint64, ActualValue: uint64(6)},
+		{StringValue: "66", Kind: reflect.Uint64, ActualValue: uint64(66)},
+	}
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%v %v", test.Kind, test.StringValue), func(t *testing.T) {
+			parsedValue, err := getParsedValue(test.Kind, test.StringValue)
+
+			if err != nil {
+				t.Errorf("unexpected error result - err: %v", err)
+				return
+			}
+
+			if parsedValue.Interface() != test.ActualValue {
+				t.Errorf("unexpected result - expected: %v, actual: %v", test.ActualValue, parsedValue)
+			}
+		})
+	}
+}
+
+func TestGetParsedValueInvalidValue(t *testing.T) {
+	tests := []struct {
+		StringValue string
+		Kind        reflect.Kind
+	}{
+		{StringValue: "test", Kind: reflect.Bool},
+		{StringValue: "123", Kind: reflect.Bool},
+		{StringValue: "test", Kind: reflect.Int},
+		{StringValue: "true", Kind: reflect.Int},
+		{StringValue: "test", Kind: reflect.Int8},
+		{StringValue: "true", Kind: reflect.Int8},
+		{StringValue: "test", Kind: reflect.Int16},
+		{StringValue: "true", Kind: reflect.Int16},
+		{StringValue: "test", Kind: reflect.Int32},
+		{StringValue: "true", Kind: reflect.Int32},
+		{StringValue: "test", Kind: reflect.Int64},
+		{StringValue: "true", Kind: reflect.Int64},
+		{StringValue: "test", Kind: reflect.Uint},
+		{StringValue: "-6", Kind: reflect.Uint},
+		{StringValue: "test", Kind: reflect.Uint8},
+		{StringValue: "-6", Kind: reflect.Uint8},
+		{StringValue: "test", Kind: reflect.Uint16},
+		{StringValue: "-6", Kind: reflect.Uint16},
+		{StringValue: "test", Kind: reflect.Uint32},
+		{StringValue: "-6", Kind: reflect.Uint32},
+		{StringValue: "test", Kind: reflect.Uint64},
+		{StringValue: "-6", Kind: reflect.Uint64},
+	}
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%v %v", test.Kind, test.StringValue), func(t *testing.T) {
+			parsedValue, err := getParsedValue(test.Kind, test.StringValue)
+
+			if err == nil {
+				t.Errorf("unexpected unhandled error - stringValue: %v, Kind: %v", test.StringValue, test.Kind)
+				return
+			}
+
+			if parsedValue != reflect.ValueOf(nil) {
+				t.Errorf("unexpected parsed value - parsedValue: %v", parsedValue)
+			}
+		})
+	}
+}

cli/config/config_test.go

@@ -0,0 +1,43 @@
+package config_test
+
+import (
+	"fmt"
+	"github.com/up9inc/mizu/cli/config"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestConfigWriteIgnoresReadonlyFields(t *testing.T) {
+	var readonlyFields []string
+
+	configElem := reflect.ValueOf(&config.ConfigStruct{}).Elem()
+	getFieldsWithReadonlyTag(configElem, &readonlyFields)
+
+	configWithDefaults, _ := config.GetConfigWithDefaults()
+	for _, readonlyField := range readonlyFields {
+		t.Run(readonlyField, func(t *testing.T) {
+			readonlyFieldToCheck := fmt.Sprintf("\n%s:", readonlyField)
+			if strings.Contains(configWithDefaults, readonlyFieldToCheck) {
+				t.Errorf("unexpected result - readonly field: %v, config: %v", readonlyField, configWithDefaults)
+			}
+		})
+	}
+}
+
+func getFieldsWithReadonlyTag(currentElem reflect.Value, readonlyFields *[]string) {
+	for i := 0; i < currentElem.NumField(); i++ {
+		currentField := currentElem.Type().Field(i)
+		currentFieldByName := currentElem.FieldByName(currentField.Name)
+
+		if currentField.Type.Kind() == reflect.Struct {
+			getFieldsWithReadonlyTag(currentFieldByName, readonlyFields)
+			continue
+		}
+
+		if _, ok := currentField.Tag.Lookup(config.ReadonlyTag); ok {
+			fieldNameByTag := strings.Split(currentField.Tag.Get(config.FieldNameTag), ",")[0]
+			*readonlyFields = append(*readonlyFields, fieldNameByTag)
+		}
+	}
+}

cli/errormessage/errormessage.go

@@ -3,9 +3,7 @@ package errormessage
 import (
 	"errors"
 	"fmt"
-
-	"github.com/up9inc/mizu/cli/mizu"
-
+	"github.com/up9inc/mizu/cli/config"
 	regexpsyntax "regexp/syntax"
 
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -20,9 +18,9 @@ func FormatError(err error) error {
 			"supply the required permission or control Mizu's access to namespaces by setting %s "+
 			"in the config file or setting the tapped namespace with --%s %s=<NAMEPSACE>",
 			err,
-			mizu.MizuResourcesNamespaceConfigName,
-			mizu.SetCommandName,
-			mizu.MizuResourcesNamespaceConfigName)
+			config.MizuResourcesNamespaceConfigName,
+			config.SetCommandName,
+			config.MizuResourcesNamespaceConfigName)
 	} else if syntaxError, isSyntaxError := asRegexSyntaxError(err); isSyntaxError {
 		errorNew = fmt.Errorf("regex %s is invalid: %w", syntaxError.Expr, err)
 	} else {

cli/fsUtils/mizuLogsUtils.go

@@ -1,58 +0,0 @@
-package fsUtils
-
-import (
-	"archive/zip"
-	"context"
-	"fmt"
-	"github.com/up9inc/mizu/cli/kubernetes"
-	"github.com/up9inc/mizu/cli/mizu"
-	"os"
-	"regexp"
-)
-
-func DumpLogs(provider *kubernetes.Provider, ctx context.Context, filePath string) error {
-	podExactRegex := regexp.MustCompile("^" + mizu.MizuResourcesPrefix)
-	pods, err := provider.ListAllPodsMatchingRegex(ctx, podExactRegex, []string{mizu.Config.MizuResourcesNamespace})
-	if err != nil {
-		return err
-	}
-
-	if len(pods) == 0 {
-		return fmt.Errorf("no mizu pods found in namespace %s", mizu.Config.MizuResourcesNamespace)
-	}
-
-	newZipFile, err := os.Create(filePath)
-	if err != nil {
-		return err
-	}
-	defer newZipFile.Close()
-	zipWriter := zip.NewWriter(newZipFile)
-	defer zipWriter.Close()
-
-	for _, pod := range pods {
-		logs, err := provider.GetPodLogs(pod.Namespace, pod.Name, ctx)
-		if err != nil {
-			mizu.Log.Errorf("Failed to get logs, %v", err)
-			continue
-		} else {
-			mizu.Log.Debugf("Successfully read log length %d for pod: %s.%s", len(logs), pod.Namespace, pod.Name)
-		}
-		if err := AddStrToZip(zipWriter, logs, fmt.Sprintf("%s.%s.log", pod.Namespace, pod.Name)); err != nil {
-			mizu.Log.Errorf("Failed write logs, %v", err)
-		} else {
-			mizu.Log.Infof("Successfully added log length %d from pod: %s.%s", len(logs), pod.Namespace, pod.Name)
-		}
-	}
-	if err := AddFileToZip(zipWriter, mizu.GetConfigFilePath()); err != nil {
-		mizu.Log.Debugf("Failed write file, %v", err)
-	} else {
-		mizu.Log.Infof("Successfully added file %s", mizu.GetConfigFilePath())
-	}
-	if err := AddFileToZip(zipWriter, mizu.GetLogFilePath()); err != nil {
-		mizu.Log.Debugf("Failed write file, %v", err)
-	} else {
-		mizu.Log.Infof("Successfully added file %s", mizu.GetLogFilePath())
-	}
-	mizu.Log.Infof("You can find the zip with all logs in %s\n", filePath)
-	return nil
-}

cli/go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/creasty/defaults v1.5.1
+	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/google/go-github/v37 v37.0.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7

cli/go.sum

@@ -88,6 +88,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
+github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
+github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
@@ -412,7 +414,6 @@ github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=

cli/kubernetes/provider.go

@@ -7,7 +7,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"os"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -36,7 +37,6 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	_ "k8s.io/client-go/tools/portforward"
 	watchtools "k8s.io/client-go/tools/watch"
-	"k8s.io/client-go/util/homedir"
 )
 
 type Provider struct {
@@ -55,13 +55,23 @@ func NewProvider(kubeConfigPath string) (*Provider, error) {
 	restClientConfig, err := kubernetesConfig.ClientConfig()
 	if err != nil {
 		if clientcmd.IsEmptyConfig(err) {
-			return nil, fmt.Errorf("Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'\n")
+			return nil, fmt.Errorf("couldn't find the kube config file, or file is empty (%s)\n" +
+				"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 		}
 		if clientcmd.IsConfigurationInvalid(err) {
-			return nil, fmt.Errorf("Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'\n")
+			return nil, fmt.Errorf("invalid kube config file (%s)\n" +
+				"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 		}
+
+		return nil, fmt.Errorf("error while using kube config (%s)\n" +
+			"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
+	}
+
+	clientSet, err := getClientSet(restClientConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error while using kube config (%s)\n" +
+			"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 	}
-	clientSet := getClientSet(restClientConfig)
 
 	return &Provider{
 		clientSet:        clientSet,
@@ -140,6 +150,8 @@ type ApiServerOptions struct {
 	IsNamespaceRestricted   bool
 	MizuApiFilteringOptions *shared.TrafficFilteringOptions
 	MaxEntriesDBSizeBytes   int64
+	Resources               configStructs.Resources
+	ImagePullPolicy         core.PullPolicy
 }
 
 func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiServerOptions) (*core.Pod, error) {
@@ -152,19 +164,19 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS
 	configMapOptional := true
 	configMapVolumeName.Optional = &configMapOptional
 
-	cpuLimit, err := resource.ParseQuantity("750m")
+	cpuLimit, err := resource.ParseQuantity(opts.Resources.CpuLimit)
 	if err != nil {
 		return nil, errors.New(fmt.Sprintf("invalid cpu limit for %s container", opts.PodName))
 	}
-	memLimit, err := resource.ParseQuantity("512Mi")
+	memLimit, err := resource.ParseQuantity(opts.Resources.MemoryLimit)
 	if err != nil {
 		return nil, errors.New(fmt.Sprintf("invalid memory limit for %s container", opts.PodName))
 	}
-	cpuRequests, err := resource.ParseQuantity("50m")
+	cpuRequests, err := resource.ParseQuantity(opts.Resources.CpuRequests)
 	if err != nil {
 		return nil, errors.New(fmt.Sprintf("invalid cpu request for %s container", opts.PodName))
 	}
-	memRequests, err := resource.ParseQuantity("50Mi")
+	memRequests, err := resource.ParseQuantity(opts.Resources.MemoryRequests)
 	if err != nil {
 		return nil, errors.New(fmt.Sprintf("invalid memory request for %s container", opts.PodName))
 	}
@@ -185,7 +197,7 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS
 				{
 					Name:            opts.PodName,
 					Image:           opts.PodImage,
-					ImagePullPolicy: core.PullAlways,
+					ImagePullPolicy: opts.ImagePullPolicy,
 					VolumeMounts: []core.VolumeMount{
 						{
 							Name:      mizu.ConfigMapName,
@@ -561,8 +573,8 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
 	return nil
 }
 
-func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool) error {
-	mizu.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
+func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool, resources configStructs.Resources, imagePullPolicy core.PullPolicy) error {
+	logger.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
 
 	if len(nodeToTappedPodIPMap) == 0 {
 		return fmt.Errorf("Daemon set %s must tap at least 1 pod", daemonSetName)
@@ -586,7 +598,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	agentContainer := applyconfcore.Container()
 	agentContainer.WithName(tapperPodName)
 	agentContainer.WithImage(podImage)
-	agentContainer.WithImagePullPolicy(core.PullAlways)
+	agentContainer.WithImagePullPolicy(imagePullPolicy)
 	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true))
 	agentContainer.WithCommand(mizuCmd...)
 	agentContainer.WithEnv(
@@ -600,19 +612,19 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 			),
 		),
 	)
-	cpuLimit, err := resource.ParseQuantity("500m")
+	cpuLimit, err := resource.ParseQuantity(resources.CpuLimit)
 	if err != nil {
 		return errors.New(fmt.Sprintf("invalid cpu limit for %s container", tapperPodName))
 	}
-	memLimit, err := resource.ParseQuantity("1Gi")
+	memLimit, err := resource.ParseQuantity(resources.MemoryLimit)
 	if err != nil {
 		return errors.New(fmt.Sprintf("invalid memory limit for %s container", tapperPodName))
 	}
-	cpuRequests, err := resource.ParseQuantity("50m")
+	cpuRequests, err := resource.ParseQuantity(resources.CpuRequests)
 	if err != nil {
 		return errors.New(fmt.Sprintf("invalid cpu request for %s container", tapperPodName))
 	}
-	memRequests, err := resource.ParseQuantity("50Mi")
+	memRequests, err := resource.ParseQuantity(resources.MemoryRequests)
 	if err != nil {
 		return errors.New(fmt.Sprintf("invalid memory request for %s container", tapperPodName))
 	}
@@ -727,25 +739,17 @@ func (provider *Provider) GetPodLogs(namespace string, podName string, ctx conte
 	return str, nil
 }
 
-func getClientSet(config *restclient.Config) *kubernetes.Clientset {
+func getClientSet(config *restclient.Config) (*kubernetes.Clientset, error) {
 	clientSet, err := kubernetes.NewForConfig(config)
 	if err != nil {
-		panic(err.Error())
+		return nil, err
 	}
-	return clientSet
+
+	return clientSet, nil
 }
 
 func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
-	if kubeConfigPath == "" {
-		kubeConfigPath = os.Getenv("KUBECONFIG")
-	}
-
-	if kubeConfigPath == "" {
-		home := homedir.HomeDir()
-		kubeConfigPath = filepath.Join(home, ".kube", "config")
-	}
-
-	mizu.Log.Debugf("Using kube config %s", kubeConfigPath)
+	logger.Log.Debugf("Using kube config %s", kubeConfigPath)
 	configPathList := filepath.SplitList(kubeConfigPath)
 	configLoadingRules := &clientcmd.ClientConfigLoadingRules{}
 	if len(configPathList) <= 1 {

cli/kubernetes/proxy.go

@@ -2,7 +2,7 @@ package kubernetes
 
 import (
 	"fmt"
-	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/logger"
 	"k8s.io/kubectl/pkg/proxy"
 	"net"
 	"net/http"
@@ -14,7 +14,7 @@ const k8sProxyApiPrefix = "/"
 const mizuServicePort = 80
 
 func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace string, mizuServiceName string) error {
-	mizu.Log.Debugf("Starting proxy. namespace: [%v], service name: [%s], port: [%v]", mizuNamespace, mizuServiceName, mizuPort)
+	logger.Log.Debugf("Starting proxy. namespace: [%v], service name: [%s], port: [%v]", mizuNamespace, mizuServiceName, mizuPort)
 	filter := &proxy.FilterServer{
 		AcceptPaths:   proxy.MakeRegexpArrayOrDie(proxy.DefaultPathAcceptRE),
 		RejectPaths:   proxy.MakeRegexpArrayOrDie(proxy.DefaultPathRejectRE),

cli/logger/logger.go

@@ -1,7 +1,8 @@
-package mizu
+package logger
 
 import (
 	"github.com/op/go-logging"
+	"github.com/up9inc/mizu/cli/mizu"
 	"os"
 	"path"
 )
@@ -13,7 +14,7 @@ var format = logging.MustStringFormatter(
 )
 
 func GetLogFilePath() string {
-	return path.Join(GetMizuFolderPath(), "mizu_cli.log")
+	return path.Join(mizu.GetMizuFolderPath(), "mizu_cli.log")
 }
 
 func InitLogger() {
@@ -34,5 +35,5 @@ func InitLogger() {
 	logging.SetBackend(backend1Leveled, backend2Formatter)
 
 	Log.Debugf("\n\n\n")
-	Log.Debugf("Running mizu version %v", SemVer)
+	Log.Debugf("Running mizu version %v", mizu.SemVer)
 }

cli/mizu.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"github.com/up9inc/mizu/cli/cmd"
-	"github.com/up9inc/mizu/cli/goUtils"
+	"github.com/up9inc/mizu/cli/mizu/goUtils"
 )
 
 func main() {

cli/mizu/config.go

@@ -1,284 +0,0 @@
-package mizu
-
-import (
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
-	"reflect"
-	"strconv"
-	"strings"
-
-	"github.com/creasty/defaults"
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
-	"github.com/up9inc/mizu/cli/mizu/configStructs"
-	"github.com/up9inc/mizu/cli/uiUtils"
-	"gopkg.in/yaml.v3"
-)
-
-const (
-	Separator      = "="
-	SetCommandName = "set"
-)
-
-var allowedSetFlags = []string{
-	AgentImageConfigName,
-	MizuResourcesNamespaceConfigName,
-	TelemetryConfigName,
-	DumpLogsConfigName,
-	KubeConfigPathName,
-	configStructs.AnalysisDestinationTapName,
-	configStructs.SleepIntervalSecTapName,
-	configStructs.IgnoredUserAgentsTapName,
-}
-
-var Config = ConfigStruct{}
-
-func (config *ConfigStruct) Validate() error {
-	if config.IsNsRestrictedMode() {
-		if config.Tap.AllNamespaces || len(config.Tap.Namespaces) != 1 || config.Tap.Namespaces[0] != config.MizuResourcesNamespace {
-			return fmt.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+
-				"You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, MizuResourcesNamespaceConfigName)
-		}
-	}
-
-	return nil
-}
-
-func InitConfig(cmd *cobra.Command) error {
-	if err := defaults.Set(&Config); err != nil {
-		return err
-	}
-
-	if err := mergeConfigFile(); err != nil {
-		return fmt.Errorf("invalid config %w\n"+
-			"you can regenerate the file using `mizu config -r` or just remove it %v", err, GetConfigFilePath())
-	}
-
-	cmd.Flags().Visit(initFlag)
-
-	finalConfigPrettified, _ := uiUtils.PrettyJson(Config)
-	Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified)
-
-	return nil
-}
-
-func GetConfigWithDefaults() (string, error) {
-	defaultConf := ConfigStruct{}
-	if err := defaults.Set(&defaultConf); err != nil {
-		return "", err
-	}
-
-	// TODO: change to generic solution
-	defaultConf.AgentImage = ""
-
-	return uiUtils.PrettyYaml(defaultConf)
-}
-
-func GetConfigFilePath() string {
-	return path.Join(GetMizuFolderPath(), "config.yaml")
-}
-
-func mergeConfigFile() error {
-	reader, openErr := os.Open(GetConfigFilePath())
-	if openErr != nil {
-		return nil
-	}
-
-	buf, readErr := ioutil.ReadAll(reader)
-	if readErr != nil {
-		return readErr
-	}
-
-	if err := yaml.Unmarshal(buf, &Config); err != nil {
-		return err
-	}
-	Log.Debugf("Found config file, merged to default options")
-
-	return nil
-}
-
-func initFlag(f *pflag.Flag) {
-	configElem := reflect.ValueOf(&Config).Elem()
-
-	sliceValue, isSliceValue := f.Value.(pflag.SliceValue)
-	if !isSliceValue {
-		mergeFlagValue(configElem, f.Name, f.Value.String())
-		return
-	}
-
-	if f.Name == SetCommandName {
-		mergeSetFlag(sliceValue.GetSlice())
-		return
-	}
-
-	mergeFlagValues(configElem, f.Name, sliceValue.GetSlice())
-}
-
-func mergeSetFlag(setValues []string) {
-	configElem := reflect.ValueOf(&Config).Elem()
-
-	for _, setValue := range setValues {
-		if !strings.Contains(setValue, Separator) {
-			Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument %s (set argument format: <flag name>=<flag value>)", setValue))
-		}
-
-		split := strings.SplitN(setValue, Separator, 2)
-		if len(split) != 2 {
-			Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument %s (set argument format: <flag name>=<flag value>)", setValue))
-		}
-
-		argumentKey, argumentValue := split[0], split[1]
-
-		if !Contains(allowedSetFlags, argumentKey) {
-			Log.Warningf(uiUtils.Warning, fmt.Sprintf("Ignoring set argument %s, flag name must be one of the following: \"%s\"", setValue, strings.Join(allowedSetFlags, "\", \"")))
-		}
-
-		mergeFlagValue(configElem, argumentKey, argumentValue)
-	}
-}
-
-func mergeFlagValue(currentElem reflect.Value, flagKey string, flagValue string) {
-	for i := 0; i < currentElem.NumField(); i++ {
-		currentField := currentElem.Type().Field(i)
-		currentFieldByName := currentElem.FieldByName(currentField.Name)
-
-		if currentField.Type.Kind() == reflect.Struct {
-			mergeFlagValue(currentFieldByName, flagKey, flagValue)
-			continue
-		}
-
-		if currentField.Tag.Get("yaml") != flagKey {
-			continue
-		}
-
-		flagValueKind := currentField.Type.Kind()
-
-		parsedValue, err := getParsedValue(flagValueKind, flagValue)
-		if err != nil {
-			Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for flag name %s, expected %s", flagValue, flagKey, flagValueKind))
-			return
-		}
-
-		currentFieldByName.Set(parsedValue)
-	}
-}
-
-func mergeFlagValues(currentElem reflect.Value, flagKey string, flagValues []string) {
-	for i := 0; i < currentElem.NumField(); i++ {
-		currentField := currentElem.Type().Field(i)
-		currentFieldByName := currentElem.FieldByName(currentField.Name)
-
-		if currentField.Type.Kind() == reflect.Struct {
-			mergeFlagValues(currentFieldByName, flagKey, flagValues)
-			continue
-		}
-
-		if currentField.Tag.Get("yaml") != flagKey {
-			continue
-		}
-
-		flagValueKind := currentField.Type.Elem().Kind()
-
-		parsedValues := reflect.MakeSlice(reflect.SliceOf(currentField.Type.Elem()), 0, 0)
-		for _, flagValue := range flagValues {
-			parsedValue, err := getParsedValue(flagValueKind, flagValue)
-			if err != nil {
-				Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for flag name %s, expected %s", flagValue, flagKey, flagValueKind))
-				return
-			}
-
-			parsedValues = reflect.Append(parsedValues, parsedValue)
-		}
-
-		currentFieldByName.Set(parsedValues)
-	}
-}
-
-func getParsedValue(kind reflect.Kind, value string) (reflect.Value, error) {
-	switch kind {
-	case reflect.String:
-		return reflect.ValueOf(value), nil
-	case reflect.Bool:
-		boolArgumentValue, err := strconv.ParseBool(value)
-		if err != nil {
-			break
-		}
-
-		return reflect.ValueOf(boolArgumentValue), nil
-	case reflect.Int:
-		intArgumentValue, err := strconv.ParseInt(value, 10, 64)
-		if err != nil {
-			break
-		}
-
-		return reflect.ValueOf(int(intArgumentValue)), nil
-	case reflect.Int8:
-		intArgumentValue, err := strconv.ParseInt(value, 10, 8)
-		if err != nil {
-			break
-		}
-
-		return reflect.ValueOf(int8(intArgumentValue)), nil
-	case reflect.Int16:
-		intArgumentValue, err := strconv.ParseInt(value, 10, 16)
-		if err != nil {
-			break
-		}
-
-		return reflect.ValueOf(int16(intArgumentValue)), nil
-	case reflect.Int32:
-		intArgumentValue, err := strconv.ParseInt(value, 10, 32)
-		if err != nil {
-			break
-		}
-
-		return reflect.ValueOf(int32(intArgumentValue)), nil
-	case reflect.Int64:
-		intArgumentValue, err := strconv.ParseInt(value, 10, 64)
-		if err != nil {
-			break
-		}
-
-		return reflect.ValueOf(intArgumentValue), nil
-	case reflect.Uint:
-		uintArgumentValue, err := strconv.ParseUint(value, 10, 64)
-		if err != nil {
-			break
-		}
-
-		return reflect.ValueOf(uint(uintArgumentValue)), nil
-	case reflect.Uint8:
-		uintArgumentValue, err := strconv.ParseUint(value, 10, 8)
-		if err != nil {
-			break
-		}
-
-		return reflect.ValueOf(uint8(uintArgumentValue)), nil
-	case reflect.Uint16:
-		uintArgumentValue, err := strconv.ParseUint(value, 10, 16)
-		if err != nil {
-			break
-		}
-
-		return reflect.ValueOf(uint16(uintArgumentValue)), nil
-	case reflect.Uint32:
-		uintArgumentValue, err := strconv.ParseUint(value, 10, 32)
-		if err != nil {
-			break
-		}
-
-		return reflect.ValueOf(uint32(uintArgumentValue)), nil
-	case reflect.Uint64:
-		uintArgumentValue, err := strconv.ParseUint(value, 10, 64)
-		if err != nil {
-			break
-		}
-
-		return reflect.ValueOf(uintArgumentValue), nil
-	}
-
-	return reflect.ValueOf(nil), errors.New("value to parse does not match type")
-}

cli/mizu/configStruct.go

@@ -1,35 +0,0 @@
-package mizu
-
-import (
-	"fmt"
-
-	"github.com/up9inc/mizu/cli/mizu/configStructs"
-)
-
-const (
-	AgentImageConfigName             = "agent-image"
-	MizuResourcesNamespaceConfigName = "mizu-resources-namespace"
-	TelemetryConfigName              = "telemetry"
-	DumpLogsConfigName               = "dump-logs"
-	KubeConfigPathName               = "kube-config-path"
-)
-
-type ConfigStruct struct {
-	Tap                    configStructs.TapConfig     `yaml:"tap"`
-	Fetch                  configStructs.FetchConfig   `yaml:"fetch"`
-	Version                configStructs.VersionConfig `yaml:"version"`
-	View                   configStructs.ViewConfig    `yaml:"view"`
-	AgentImage             string                      `yaml:"agent-image,omitempty"`
-	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
-	Telemetry              bool                        `yaml:"telemetry" default:"true"`
-	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`
-	KubeConfigPath         string                      `yaml:"kube-config-path" default:""`
-}
-
-func (config *ConfigStruct) SetDefaults() {
-	config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", Branch, SemVer)
-}
-
-func (config *ConfigStruct) IsNsRestrictedMode() bool {
-	return config.MizuResourcesNamespace != "mizu" // Notice "mizu" string must match the default MizuResourcesNamespace
-}

cli/mizu/configStructs/viewConfig.go

@@ -1,11 +0,0 @@
-package configStructs
-
-const (
-	GuiPortViewName        = "gui-port"
-	KubeConfigPathViewName = "kube-config"
-)
-
-type ViewConfig struct {
-	GuiPort        uint16 `yaml:"gui-port" default:"8899"`
-	KubeConfigPath string `yaml:"kube-config"`
-}

cli/mizu/fsUtils/mizuLogsUtils.go

@@ -0,0 +1,60 @@
+package fsUtils
+
+import (
+	"archive/zip"
+	"context"
+	"fmt"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu"
+	"os"
+	"regexp"
+)
+
+func DumpLogs(provider *kubernetes.Provider, ctx context.Context, filePath string) error {
+	podExactRegex := regexp.MustCompile("^" + mizu.MizuResourcesPrefix)
+	pods, err := provider.ListAllPodsMatchingRegex(ctx, podExactRegex, []string{config.Config.MizuResourcesNamespace})
+	if err != nil {
+		return err
+	}
+
+	if len(pods) == 0 {
+		return fmt.Errorf("no mizu pods found in namespace %s", config.Config.MizuResourcesNamespace)
+	}
+
+	newZipFile, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	defer newZipFile.Close()
+	zipWriter := zip.NewWriter(newZipFile)
+	defer zipWriter.Close()
+
+	for _, pod := range pods {
+		logs, err := provider.GetPodLogs(pod.Namespace, pod.Name, ctx)
+		if err != nil {
+			logger.Log.Errorf("Failed to get logs, %v", err)
+			continue
+		} else {
+			logger.Log.Debugf("Successfully read log length %d for pod: %s.%s", len(logs), pod.Namespace, pod.Name)
+		}
+		if err := AddStrToZip(zipWriter, logs, fmt.Sprintf("%s.%s.log", pod.Namespace, pod.Name)); err != nil {
+			logger.Log.Errorf("Failed write logs, %v", err)
+		} else {
+			logger.Log.Debugf("Successfully added log length %d from pod: %s.%s", len(logs), pod.Namespace, pod.Name)
+		}
+	}
+	if err := AddFileToZip(zipWriter, config.Config.ConfigFilePath); err != nil {
+		logger.Log.Debugf("Failed write file, %v", err)
+	} else {
+		logger.Log.Debugf("Successfully added file %s", config.Config.ConfigFilePath)
+	}
+	if err := AddFileToZip(zipWriter, logger.GetLogFilePath()); err != nil {
+		logger.Log.Debugf("Failed write file, %v", err)
+	} else {
+		logger.Log.Debugf("Successfully added file %s", logger.GetLogFilePath())
+	}
+	logger.Log.Infof("You can find the zip file with all logs in %s\n", filePath)
+	return nil
+}

cli/mizu/fsUtils/zipUtils.go

@@ -3,9 +3,11 @@ package fsUtils
 import (
 	"archive/zip"
 	"fmt"
+	"github.com/up9inc/mizu/cli/logger"
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 func AddFileToZip(zipWriter *zip.Writer, filename string) error {
@@ -53,3 +55,60 @@ func AddStrToZip(writer *zip.Writer, logs string, fileName string) error {
 	}
 	return nil
 }
+
+func Unzip(reader *zip.Reader, dest string) error {
+	dest, _ = filepath.Abs(dest)
+	_ = os.MkdirAll(dest, os.ModePerm)
+
+	// Closure to address file descriptors issue with all the deferred .Close() methods
+	extractAndWriteFile := func(f *zip.File) error {
+		rc, err := f.Open()
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if err := rc.Close(); err != nil {
+				panic(err)
+			}
+		}()
+
+		path := filepath.Join(dest, f.Name)
+
+		// Check for ZipSlip (Directory traversal)
+		if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) {
+			return fmt.Errorf("illegal file path: %s", path)
+		}
+
+		if f.FileInfo().IsDir() {
+			_ = os.MkdirAll(path, f.Mode())
+		} else {
+			_ = os.MkdirAll(filepath.Dir(path), f.Mode())
+			logger.Log.Infof("writing HAR file [ %v ]", path)
+			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				if err := f.Close(); err != nil {
+					panic(err)
+				}
+				logger.Log.Info(" done")
+			}()
+
+			_, err = io.Copy(f, rc)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	for _, f := range reader.File {
+		err := extractAndWriteFile(f)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cli/mizu/goUtils/funcWrappers.go

@@ -1,7 +1,7 @@
 package goUtils
 
 import (
-	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/logger"
 	"reflect"
 	"runtime/debug"
 )
@@ -10,7 +10,7 @@ func HandleExcWrapper(fn interface{}, params ...interface{}) (result []reflect.V
 	defer func() {
 		if panicMessage := recover(); panicMessage != nil {
 			stack := debug.Stack()
-			mizu.Log.Fatalf("Unhandled panic: %v\n stack: %s", panicMessage, stack)
+			logger.Log.Fatalf("Unhandled panic: %v\n stack: %s", panicMessage, stack)
 		}
 	}()
 	f := reflect.ValueOf(fn)

cli/mizu/sliceUtils.go

@@ -9,3 +9,17 @@ func Contains(slice []string, containsValue string) bool {
 
 	return false
 }
+
+func Unique(slice []string) []string {
+	keys := make(map[string]bool)
+	var list []string
+
+	for _, entry := range slice {
+		if _, value := keys[entry]; !value {
+			keys[entry] = true
+			list = append(list, entry)
+		}
+	}
+
+	return list
+}

cli/mizu/sliceUtils_test.go

@@ -0,0 +1,130 @@
+package mizu_test
+
+import (
+	"fmt"
+	"github.com/up9inc/mizu/cli/mizu"
+	"reflect"
+	"testing"
+)
+
+func TestContainsExists(t *testing.T) {
+	tests := []struct {
+		Slice         []string
+		ContainsValue string
+		Expected      bool
+	}{
+		{Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "apple", Expected: true},
+		{Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "orange", Expected: true},
+		{Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "banana", Expected: true},
+		{Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "grapes", Expected: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.ContainsValue, func(t *testing.T) {
+			actual := mizu.Contains(test.Slice, test.ContainsValue)
+			if actual != test.Expected {
+				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
+			}
+		})
+	}
+}
+
+func TestContainsNotExists(t *testing.T) {
+	tests := []struct {
+		Slice         []string
+		ContainsValue string
+		Expected      bool
+	}{
+		{Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "cat", Expected: false},
+		{Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "dog", Expected: false},
+		{Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "apples", Expected: false},
+		{Slice: []string{"apple", "orange", "banana", "grapes"}, ContainsValue: "rapes", Expected: false},
+	}
+
+	for _, test := range tests {
+		t.Run(test.ContainsValue, func(t *testing.T) {
+			actual := mizu.Contains(test.Slice, test.ContainsValue)
+			if actual != test.Expected {
+				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
+			}
+		})
+	}
+}
+
+func TestContainsEmptySlice(t *testing.T) {
+	tests := []struct {
+		Slice         []string
+		ContainsValue string
+		Expected      bool
+	}{
+		{Slice: []string{}, ContainsValue: "cat", Expected: false},
+		{Slice: []string{}, ContainsValue: "dog", Expected: false},
+	}
+
+	for _, test := range tests {
+		t.Run(test.ContainsValue, func(t *testing.T) {
+			actual := mizu.Contains(test.Slice, test.ContainsValue)
+			if actual != test.Expected {
+				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
+			}
+		})
+	}
+}
+
+func TestContainsNilSlice(t *testing.T) {
+	tests := []struct {
+		Slice         []string
+		ContainsValue string
+		Expected      bool
+	}{
+		{Slice: nil, ContainsValue: "cat", Expected: false},
+		{Slice: nil, ContainsValue: "dog", Expected: false},
+	}
+
+	for _, test := range tests {
+		t.Run(test.ContainsValue, func(t *testing.T) {
+			actual := mizu.Contains(test.Slice, test.ContainsValue)
+			if actual != test.Expected {
+				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
+			}
+		})
+	}
+}
+
+func TestUniqueNoDuplicateValues(t *testing.T) {
+	tests := []struct {
+		Slice         []string
+		Expected      []string
+	}{
+		{Slice: []string{"apple", "orange", "banana", "grapes"}, Expected: []string{"apple", "orange", "banana", "grapes"}},
+		{Slice: []string{"dog", "cat", "mouse"}, Expected: []string{"dog", "cat", "mouse"}},
+	}
+
+	for index, test := range tests {
+		t.Run(fmt.Sprintf("%v", index), func(t *testing.T) {
+			actual := mizu.Unique(test.Slice)
+			if !reflect.DeepEqual(test.Expected, actual) {
+				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
+			}
+		})
+	}
+}
+
+func TestUniqueDuplicateValues(t *testing.T) {
+	tests := []struct {
+		Slice         []string
+		Expected      []string
+	}{
+		{Slice: []string{"apple", "apple", "orange", "orange", "banana", "banana", "grapes", "grapes"}, Expected: []string{"apple", "orange", "banana", "grapes"}},
+		{Slice: []string{"dog", "cat", "cat", "mouse"}, Expected: []string{"dog", "cat", "mouse"}},
+	}
+
+	for index, test := range tests {
+		t.Run(fmt.Sprintf("%v", index), func(t *testing.T) {
+			actual := mizu.Unique(test.Slice)
+			if !reflect.DeepEqual(test.Expected, actual) {
+				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
+			}
+		})
+	}
+}

cli/mizu/telemetry.go

@@ -1,36 +0,0 @@
-package mizu
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"net/http"
-)
-
-const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
-
-func ReportRun(cmd string, args interface{}) {
-	if !Config.Telemetry {
-		Log.Debugf("not reporting due to config value")
-		return
-	}
-
-	argsBytes, _ := json.Marshal(args)
-	argsMap := map[string]string{
-		"telemetry_type": "execution",
-		"cmd":            cmd,
-		"args":           string(argsBytes),
-		"component":      "mizu_cli",
-		"BuildTimestamp": BuildTimestamp,
-		"Branch":         Branch,
-		"version":        SemVer}
-	argsMap["message"] = fmt.Sprintf("mizu %v - %v", argsMap["cmd"], string(argsBytes))
-
-	jsonValue, _ := json.Marshal(argsMap)
-
-	if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
-		Log.Debugf("error sending telemetry err: %v, response %v", err, resp)
-	} else {
-		Log.Debugf("Successfully reported telemetry")
-	}
-}

cli/mizu/version/versionCheck.go

@@ -0,0 +1,78 @@
+package version
+
+import (
+	"context"
+	"fmt"
+	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"github.com/google/go-github/v37/github"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/shared/semver"
+)
+
+func CheckVersionCompatibility() (bool, error) {
+	apiSemVer, err := apiserver.Provider.GetVersion()
+	if err != nil {
+		return false, err
+	}
+
+	if semver.SemVersion(apiSemVer).Major() == semver.SemVersion(mizu.SemVer).Major() &&
+		semver.SemVersion(apiSemVer).Minor() == semver.SemVersion(mizu.SemVer).Minor() {
+		return true, nil
+	}
+
+	logger.Log.Errorf(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", mizu.SemVer, apiSemVer))
+	return false, nil
+}
+
+func CheckNewerVersion(versionChan chan string) {
+	logger.Log.Debugf("Checking for newer version...")
+	start := time.Now()
+	client := github.NewClient(nil)
+	latestRelease, _, err := client.Repositories.GetLatestRelease(context.Background(), "up9inc", "mizu")
+	if err != nil {
+		logger.Log.Debugf("[ERROR] Failed to get latest release")
+		return
+	}
+
+	versionFileUrl := ""
+	for _, asset := range latestRelease.Assets {
+		if *asset.Name == "version.txt" {
+			versionFileUrl = *asset.BrowserDownloadURL
+			break
+		}
+	}
+	if versionFileUrl == "" {
+		logger.Log.Debugf("[ERROR] Version file not found in the latest release")
+		return
+	}
+
+	res, err := http.Get(versionFileUrl)
+	if err != nil {
+		logger.Log.Debugf("[ERROR] Failed to get the version file %v", err)
+		return
+	}
+
+	data, err := ioutil.ReadAll(res.Body)
+	res.Body.Close()
+	if err != nil {
+		logger.Log.Debugf("[ERROR] Failed to read the version file -> %v", err)
+		return
+	}
+	gitHubVersion := string(data)
+	gitHubVersion = gitHubVersion[:len(gitHubVersion)-1]
+
+	gitHubVersionSemVer := semver.SemVersion(gitHubVersion)
+	currentSemVer := semver.SemVersion(mizu.SemVer)
+	logger.Log.Debugf("Finished version validation, github version %v, current version %v, took %v", gitHubVersion, currentSemVer, time.Since(start))
+
+	if gitHubVersionSemVer.GreaterThan(currentSemVer) {
+		versionChan <- fmt.Sprintf("Update available! %v -> %v (%v)", mizu.SemVer, gitHubVersion, *latestRelease.HTMLURL)
+	}
+	versionChan <- ""
+}

cli/mizu/versionCheck.go

@@ -1,93 +0,0 @@
-package mizu
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"time"
-
-	"github.com/google/go-github/v37/github"
-	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/semver"
-)
-
-func getApiVersion(port uint16) (string, error) {
-	versionUrl, _ := url.Parse(fmt.Sprintf("http://localhost:%d/mizu/metadata/version", port))
-	req := &http.Request{
-		Method: http.MethodGet,
-		URL:    versionUrl,
-	}
-	statusResp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer statusResp.Body.Close()
-
-	versionResponse := &shared.VersionResponse{}
-	if err := json.NewDecoder(statusResp.Body).Decode(&versionResponse); err != nil {
-		return "", err
-	}
-
-	return versionResponse.SemVer, nil
-}
-
-func CheckVersionCompatibility(port uint16) (bool, error) {
-	apiSemVer, err := getApiVersion(port)
-	if err != nil {
-		return false, err
-	}
-
-	if semver.SemVersion(apiSemVer).Major() == semver.SemVersion(SemVer).Major() &&
-		semver.SemVersion(apiSemVer).Minor() == semver.SemVersion(SemVer).Minor() {
-		return true, nil
-	}
-
-	Log.Errorf(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", SemVer, apiSemVer))
-	return false, nil
-}
-
-func CheckNewerVersion() {
-	Log.Debugf("Checking for newer version...")
-	start := time.Now()
-	client := github.NewClient(nil)
-	latestRelease, _, err := client.Repositories.GetLatestRelease(context.Background(), "up9inc", "mizu")
-	if err != nil {
-		Log.Debugf("[ERROR] Failed to get latest release")
-		return
-	}
-
-	versionFileUrl := ""
-	for _, asset := range latestRelease.Assets {
-		if *asset.Name == "version.txt" {
-			versionFileUrl = *asset.BrowserDownloadURL
-			break
-		}
-	}
-	if versionFileUrl == "" {
-		Log.Debugf("[ERROR] Version file not found in the latest release")
-		return
-	}
-
-	res, err := http.Get(versionFileUrl)
-	if err != nil {
-		Log.Debugf("[ERROR] Failed to get the version file %v", err)
-		return
-	}
-
-	data, err := ioutil.ReadAll(res.Body)
-	res.Body.Close()
-	if err != nil {
-		Log.Debugf("[ERROR] Failed to read the version file -> %v", err)
-		return
-	}
-	gitHubVersion := string(data)
-	gitHubVersion = gitHubVersion[:len(gitHubVersion)-1]
-	Log.Debugf("Finished version validation, took %v", time.Since(start))
-	if SemVer < gitHubVersion {
-		Log.Infof(uiUtils.Yellow, fmt.Sprintf("Update available! %v -> %v (%v)", SemVer, gitHubVersion, *latestRelease.HTMLURL))
-	}
-}

cli/telemetry/telemetry.go

@@ -0,0 +1,93 @@
+package telemetry
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/denisbrodbeck/machineid"
+	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu"
+	"net/http"
+)
+
+const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
+
+func ReportRun(cmd string, args interface{}) {
+	if !shouldRunTelemetry() {
+		logger.Log.Debugf("not reporting telemetry")
+		return
+	}
+
+	argsBytes, _ := json.Marshal(args)
+	argsMap := map[string]interface{}{
+		"cmd":  cmd,
+		"args": string(argsBytes),
+	}
+
+	if err := sendTelemetry("Execution", argsMap); err != nil {
+		logger.Log.Debug(err)
+		return
+	}
+
+	logger.Log.Debugf("successfully reported telemetry for cmd %v", cmd)
+}
+
+func ReportAPICalls() {
+	if !shouldRunTelemetry() {
+		logger.Log.Debugf("not reporting telemetry")
+		return
+	}
+
+	generalStats, err := apiserver.Provider.GetGeneralStats()
+	if err != nil {
+		logger.Log.Debugf("[ERROR] failed get general stats from api server %v", err)
+		return
+	}
+
+	argsMap := map[string]interface{}{
+		"apiCallsCount":         generalStats["EntriesCount"],
+		"firstAPICallTimestamp": generalStats["FirstEntryTimestamp"],
+		"lastAPICallTimestamp":  generalStats["LastEntryTimestamp"],
+	}
+
+	if err := sendTelemetry("APICalls", argsMap); err != nil {
+		logger.Log.Debug(err)
+		return
+	}
+
+	logger.Log.Debugf("successfully reported telemetry of api calls")
+}
+
+func shouldRunTelemetry() bool {
+	if !config.Config.Telemetry {
+		return false
+	}
+
+	if mizu.Branch != "main" && mizu.Branch != "develop" {
+		return false
+	}
+
+	return true
+}
+
+func sendTelemetry(telemetryType string, argsMap map[string]interface{}) error {
+	argsMap["telemetryType"] = telemetryType
+	argsMap["component"] = "mizu_cli"
+	argsMap["buildTimestamp"] = mizu.BuildTimestamp
+	argsMap["branch"] = mizu.Branch
+	argsMap["version"] = mizu.SemVer
+
+	if machineId, err := machineid.ProtectedID("mizu"); err == nil {
+		argsMap["machineId"] = machineId
+	}
+
+	jsonValue, _ := json.Marshal(argsMap)
+
+	if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+		return fmt.Errorf("ERROR: failed sending telemetry, err: %v, response %v", err, resp)
+	}
+
+	return nil
+}

codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+  status:
+    project:
+      default:
+        threshold: 1%
+    patch:
+      default:
+        enabled: no

docs/POLICY_RULES.md

@@ -0,0 +1,78 @@
+
+# API rules validation
+
+This feature allows you to define set of simple rules, and test the API against them.
+Such validation may test response for specific JSON fields, headers, etc.
+
+## Examples
+
+
+Example 1: HTTP request (REST API call) that didn’t pass validation is highlighted in red
+
+![Simple UI](../assets/validation-example1.png)
+
+- - -
+
+
+Example 2: Details pane shows the validation rule details and whether it passed or failed
+
+![Simple UI](../assets/validation-example2.png)
+
+
+## How to use
+To use this feature - create simple rules file (see details below) and pass this file as parameter to `mizu tap` command. For example, if rules are stored in file named `rules.yaml` — run the following command:
+
+
+```shell
+mizu tap --test-rules rules.yaml PODNAME
+```
+
+
+
+## Rules file structure
+
+The structure of the test-rules-file is:
+
+* `name`: string, name of the rule
+* `type`: string, type of the rule, must be `json` or `header` or `latency`
+* `key`: string, [jsonpath](https://code.google.com/archive/p/jsonpath/wikis/Javascript.wiki) used only in `json` or `header` type
+* `value`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) used only in `json` or `header` type
+* `service`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) service name to filter
+* `path`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) URL path to filter
+* `latency`: integer, time in ms of the expected latency.
+
+
+### For example:
+
+```yaml
+rules:
+- name: holy-in-name-property
+  type: json
+  key: "$.name"
+  value: "Holy"
+  service: "catalogue.*"
+  path: "catalogue.*"
+- name: content-length-header
+  type: header
+  key: "Content-Le.*"
+  value: "(\\d+(?:\\.\\d+)?)"
+- name: latency-test
+  type: latency
+  latency: 1
+  service: "carts.*"
+```
+
+### Explanation:
+
+* First rule `holy-in-name-property`:
+
+  > This rule will be applied to all request made to `catalogue.*` services with `catalogue.*` on the URL path with a json response containing a `$.name` field. If the value of `$.name` is `Holy` than is marked as success, marked as failure otherwise.
+
+* Second rule `content-length-header`:
+
+  > This rule will be applied to all request that has `Content-Le.*` on header. If the value of `Content-Le.*` is `(\\d+(?:\\.\\d+)?)` (number), will be marked as success, marked as failure otherwise.
+
+* Third rule `latency-test`:
+
+  > This rule will be applied to all request made to `carts.*` services. If the latency of the response is greater than `1` will be marked as failure, marked as success otherwise.
+

shared/semver/semver.go

@@ -26,3 +26,23 @@ func (v SemVersion) Patch() string {
 	_, _, patch := v.Breakdown()
 	return patch
 }
+
+func (v SemVersion) GreaterThan(v2 SemVersion) bool {
+	if v.Major() > v2.Major() {
+		return true
+	} else if v.Major() < v2.Major() {
+		return false
+	}
+
+	if v.Minor() > v2.Minor() {
+		return true
+	} else if v.Minor() < v2.Minor() {
+		return false
+	}
+
+	if v.Patch() > v2.Patch() {
+		return true
+	}
+
+	return false
+}

tap/http_reader.go

@@ -79,7 +79,8 @@ func (h *httpReader) Read(p []byte) (int, error) {
 			clientHello := tlsx.ClientHello{}
 			err := clientHello.Unmarshall(msg.bytes)
 			if err == nil {
-				fmt.Printf("Detected TLS client hello with SNI %s\n", clientHello.SNI)
+				statsTracker.incTlsConnectionsCount()
+				Debug("Detected TLS client hello with SNI %s\n", clientHello.SNI)
 				numericPort, _ := strconv.Atoi(h.tcpID.dstPort)
 				h.outboundLinkWriter.WriteOutboundLink(h.tcpID.srcIP, h.tcpID.dstIP, numericPort, clientHello.SNI, TLSProtocol)
 			}
@@ -176,7 +177,7 @@ func (h *httpReader) handleHTTP2Stream() error {
 	}
 
 	if reqResPair != nil {
-		statsTracker.incMatchedMessages()
+		statsTracker.incMatchedPairs()
 
 		if h.harWriter != nil {
 			h.harWriter.WritePair(
@@ -215,7 +216,7 @@ func (h *httpReader) handleHTTP1ClientStream(b *bufio.Reader) error {
 	ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.srcIP, h.tcpID.dstIP, h.tcpID.srcPort, h.tcpID.dstPort, h.messageCount)
 	reqResPair := reqResMatcher.registerRequest(ident, req, h.captureTime)
 	if reqResPair != nil {
-		statsTracker.incMatchedMessages()
+		statsTracker.incMatchedPairs()
 
 		if h.harWriter != nil {
 			h.harWriter.WritePair(
@@ -281,7 +282,7 @@ func (h *httpReader) handleHTTP1ServerStream(b *bufio.Reader) error {
 	ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.dstIP, h.tcpID.srcIP, h.tcpID.dstPort, h.tcpID.srcPort, h.messageCount)
 	reqResPair := reqResMatcher.registerResponse(ident, res, h.captureTime)
 	if reqResPair != nil {
-		statsTracker.incMatchedMessages()
+		statsTracker.incMatchedPairs()
 
 		if h.harWriter != nil {
 			h.harWriter.WritePair(

tap/passive_tapper.go

@@ -10,9 +10,9 @@ package tap
 
 import (
 	"encoding/hex"
+	"encoding/json"
 	"flag"
 	"fmt"
-	"github.com/romana/rlog"
 	"log"
 	"os"
 	"os/signal"
@@ -23,6 +23,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/romana/rlog"
+
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/examples/util"
 	"github.com/google/gopacket/ip4defrag"
@@ -374,9 +376,7 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 			errorMapLen := len(errorsMap)
 			errorsSummery := fmt.Sprintf("%v", errorsMap)
 			errorsMapMutex.Unlock()
-			log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v) - Errors Summary: %s",
-				statsTracker.appStats.TotalPacketsCount,
-				statsTracker.appStats.TotalProcessedBytes,
+			log.Printf("%v (errors: %v, errTypes:%v) - Errors Summary: %s",
 				time.Since(statsTracker.appStats.StartTime),
 				nErrors,
 				errorMapLen,
@@ -395,14 +395,15 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 
 			// Since the last print
 			cleanStats := cleaner.dumpStats()
-			matchedMessages := statsTracker.dumpStats()
 			log.Printf(
-				"flushed connections %d, closed connections: %d, deleted messages: %d, matched messages: %d",
+				"cleaner - flushed connections: %d, closed connections: %d, deleted messages: %d",
 				cleanStats.flushed,
 				cleanStats.closed,
 				cleanStats.deleted,
-				matchedMessages,
 			)
+			currentAppStats := statsTracker.dumpStats()
+			appStatsJSON, _ := json.Marshal(currentAppStats)
+			log.Printf("app stats - %v", string(appStatsJSON))
 		}
 	}()
 
@@ -414,7 +415,7 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 		packetsCount := statsTracker.incPacketsCount()
 		rlog.Debugf("PACKET #%d", packetsCount)
 		data := packet.Data()
-		statsTracker.updateProcessedSize(int64(len(data)))
+		statsTracker.updateProcessedBytes(int64(len(data)))
 		if *hexdumppkt {
 			rlog.Debugf("Packet content (%d/0x%x) - %s", len(data), len(data), hex.Dump(data))
 		}
@@ -448,6 +449,7 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 
 		tcp := packet.Layer(layers.LayerTypeTCP)
 		if tcp != nil {
+			statsTracker.incTcpPacketsCount()
 			tcp := tcp.(*layers.TCP)
 			if *checksum {
 				err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer())
@@ -465,14 +467,14 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 			assemblerMutex.Unlock()
 		}
 
-		done := *maxcount > 0 && statsTracker.appStats.TotalPacketsCount >= *maxcount
+		done := *maxcount > 0 && statsTracker.appStats.PacketsCount >= *maxcount
 		if done {
 			errorsMapMutex.Lock()
 			errorMapLen := len(errorsMap)
 			errorsMapMutex.Unlock()
 			log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)",
-				statsTracker.appStats.TotalPacketsCount,
-				statsTracker.appStats.TotalProcessedBytes,
+				statsTracker.appStats.PacketsCount,
+				statsTracker.appStats.ProcessedBytes,
 				time.Since(statsTracker.appStats.StartTime),
 				nErrors,
 				errorMapLen)

tap/stats_tracker.go

@@ -6,50 +6,99 @@ import (
 )
 
 type AppStats struct {
-	StartTime            time.Time `json:"startTime"`
-	MatchedMessages      int       `json:"matchedMessages"`
-	TotalPacketsCount    int64     `json:"totalPacketsCount"`
-	TotalProcessedBytes  int64     `json:"totalProcessedBytes"`
-	TotalMatchedMessages int64     `json:"totalMatchedMessages"`
+	StartTime                   time.Time `json:"-"`
+	ProcessedBytes              int64     `json:"processedBytes"`
+	PacketsCount                int64     `json:"packetsCount"`
+	TcpPacketsCount             int64     `json:"tcpPacketsCount"`
+	ReassembledTcpPayloadsCount int64     `json:"reassembledTcpPayloadsCount"`
+	TlsConnectionsCount         int64     `json:"tlsConnectionsCount"`
+	MatchedPairs                int64     `json:"matchedPairs"`
 }
 
 type StatsTracker struct {
-	appStats                AppStats
-	matchedMessagesMutex    sync.Mutex
-	totalPacketsCountMutex  sync.Mutex
-	totalProcessedSizeMutex sync.Mutex
+	appStats                         AppStats
+	processedBytesMutex              sync.Mutex
+	packetsCountMutex                sync.Mutex
+	tcpPacketsCountMutex             sync.Mutex
+	reassembledTcpPayloadsCountMutex sync.Mutex
+	tlsConnectionsCountMutex         sync.Mutex
+	matchedPairsMutex                sync.Mutex
 }
 
-func (st *StatsTracker) incMatchedMessages() {
-	st.matchedMessagesMutex.Lock()
-	st.appStats.MatchedMessages++
-	st.appStats.TotalMatchedMessages++
-	st.matchedMessagesMutex.Unlock()
+func (st *StatsTracker) incMatchedPairs() {
+	st.matchedPairsMutex.Lock()
+	st.appStats.MatchedPairs++
+	st.matchedPairsMutex.Unlock()
 }
 
 func (st *StatsTracker) incPacketsCount() int64 {
-	st.totalPacketsCountMutex.Lock()
-	st.appStats.TotalPacketsCount++
-	currentPacketsCount := st.appStats.TotalPacketsCount
-	st.totalPacketsCountMutex.Unlock()
+	st.packetsCountMutex.Lock()
+	st.appStats.PacketsCount++
+	currentPacketsCount := st.appStats.PacketsCount
+	st.packetsCountMutex.Unlock()
 	return currentPacketsCount
 }
 
-func (st *StatsTracker) updateProcessedSize(size int64) {
-	st.totalProcessedSizeMutex.Lock()
-	st.appStats.TotalProcessedBytes += size
-	st.totalProcessedSizeMutex.Unlock()
+func (st *StatsTracker) incTcpPacketsCount() {
+	st.tcpPacketsCountMutex.Lock()
+	st.appStats.TcpPacketsCount++
+	st.tcpPacketsCountMutex.Unlock()
+}
+
+func (st *StatsTracker) incReassembledTcpPayloadsCount() {
+	st.reassembledTcpPayloadsCountMutex.Lock()
+	st.appStats.ReassembledTcpPayloadsCount++
+	st.reassembledTcpPayloadsCountMutex.Unlock()
+}
+
+func (st *StatsTracker) incTlsConnectionsCount() {
+	st.tlsConnectionsCountMutex.Lock()
+	st.appStats.TlsConnectionsCount++
+	st.tlsConnectionsCountMutex.Unlock()
+}
+
+func (st *StatsTracker) updateProcessedBytes(size int64) {
+	st.processedBytesMutex.Lock()
+	st.appStats.ProcessedBytes += size
+	st.processedBytesMutex.Unlock()
 }
 
 func (st *StatsTracker) setStartTime(startTime time.Time) {
 	st.appStats.StartTime = startTime
 }
 
-func (st *StatsTracker) dumpStats() int {
-	st.matchedMessagesMutex.Lock()
-	matchedMessages := st.appStats.MatchedMessages
-	st.appStats.MatchedMessages = 0
-	st.matchedMessagesMutex.Unlock()
+func (st *StatsTracker) dumpStats() *AppStats {
+	currentAppStats := &AppStats{StartTime: st.appStats.StartTime}
 
-	return matchedMessages
+	st.processedBytesMutex.Lock()
+	currentAppStats.ProcessedBytes = st.appStats.ProcessedBytes
+	st.appStats.ProcessedBytes = 0
+	st.processedBytesMutex.Unlock()
+
+	st.packetsCountMutex.Lock()
+	currentAppStats.PacketsCount = st.appStats.PacketsCount
+	st.appStats.PacketsCount = 0
+	st.packetsCountMutex.Unlock()
+
+	st.tcpPacketsCountMutex.Lock()
+	currentAppStats.TcpPacketsCount = st.appStats.TcpPacketsCount
+	st.appStats.TcpPacketsCount = 0
+	st.tcpPacketsCountMutex.Unlock()
+
+	st.reassembledTcpPayloadsCountMutex.Lock()
+	currentAppStats.ReassembledTcpPayloadsCount = st.appStats.ReassembledTcpPayloadsCount
+	st.appStats.ReassembledTcpPayloadsCount = 0
+	st.reassembledTcpPayloadsCountMutex.Unlock()
+
+	st.tlsConnectionsCountMutex.Lock()
+	currentAppStats.TlsConnectionsCount = st.appStats.TlsConnectionsCount
+	st.appStats.TlsConnectionsCount = 0
+	st.tlsConnectionsCountMutex.Unlock()
+
+	st.matchedPairsMutex.Lock()
+	currentAppStats.MatchedPairs = st.appStats.MatchedPairs
+	st.appStats.MatchedPairs = 0
+	st.matchedPairsMutex.Unlock()
+
+	return currentAppStats
 }

tap/tcp_stream.go

@@ -148,6 +148,7 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
 			}
 			// This is where we pass the reassembled information onwards
 			// This channel is read by an httpReader object
+			statsTracker.incReassembledTcpPayloadsCount()
 			if dir == reassembly.TCPDirClientToServer && !t.reversed {
 				t.client.msgQueue <- httpReaderDataMsg{data, ac.GetCaptureInfo().Timestamp}
 			} else {

ui/src/App.sass

@@ -1,4 +1,4 @@
-@import 'components/style/variables.module'
+@import 'src/variables.module'
 
 .mizuApp
   background-color: $main-background-color

ui/src/App.tsx

@@ -2,8 +2,8 @@ import React, {useEffect, useState} from 'react';
 import './App.sass';
 import logo from './components/assets/Mizu-logo.svg';
 import {Button, Snackbar} from "@material-ui/core";
-import {HarPage} from "./components/HarPage";
-import Tooltip from "./components/Tooltip";
+import {TrafficPage} from "./components/TrafficPage";
+import Tooltip from "./components/UI/Tooltip";
 import {makeStyles} from "@material-ui/core/styles";
 import MuiAlert from '@material-ui/lab/Alert';
 import Api from "./helpers/api";
@@ -38,6 +38,7 @@ const App = () => {
             }
 
         })();
+        // eslint-disable-next-line
     }, []);
 
     const onTLSDetected = (destAddress: string) => {
@@ -116,7 +117,7 @@ const App = () => {
                 </Tooltip>
                 }
             </div>
-            <HarPage setAnalyzeStatus={setAnalyzeStatus} onTLSDetected={onTLSDetected}/>
+            <TrafficPage setAnalyzeStatus={setAnalyzeStatus} onTLSDetected={onTLSDetected}/>
             <Snackbar open={showTLSWarning && !userDismissedTLSWarning}>
                 <MuiAlert elevation={6} variant="filled" onClose={() => setUserDismissedTLSWarning(true)} severity="warning">
                     Mizu is detecting TLS traffic{addressesWithTLS.size ? ` (directed to ${Array.from(addressesWithTLS).join(", ")})` : ''}, this type of traffic will not be displayed.

ui/src/components/EntriesList.tsx

@@ -1,16 +1,17 @@
-import {HarEntry} from "./HarEntry";
-import React, {useCallback, useEffect, useMemo, useState} from "react";
-import styles from './style/HarEntriesList.module.sass';
+import {EntryItem} from "./EntryListItem/EntryListItem";
+import React, {useCallback, useEffect, useMemo, useRef, useState} from "react";
+import styles from './style/EntriesList.module.sass';
 import spinner from './assets/spinner.svg';
 import ScrollableFeed from "react-scrollable-feed";
-import {StatusType} from "./HarFilters";
+import {StatusType} from "./Filters";
 import Api from "../helpers/api";
+import down from "./assets/downImg.svg";
 
 interface HarEntriesListProps {
     entries: any[];
     setEntries: (entries: any[]) => void;
-    focusedEntryId: string;
-    setFocusedEntryId: (id: string) => void;
+    focusedEntry: any;
+    setFocusedEntry: (entry: any) => void;
     connectionOpen: boolean;
     noMoreDataTop: boolean;
     setNoMoreDataTop: (flag: boolean) => void;
@@ -19,6 +20,9 @@ interface HarEntriesListProps {
     methodsFilter: Array<string>;
     statusFilter: Array<string>;
     pathFilter: string
+    listEntryREF: any;
+    onScrollEvent: (isAtBottom:boolean) => void;
+    scrollableList: boolean;
 }
 
 enum FetchOperator {
@@ -28,11 +32,12 @@ enum FetchOperator {
 
 const api = new Api();
 
-export const HarEntriesList: React.FC<HarEntriesListProps> = ({entries, setEntries, focusedEntryId, setFocusedEntryId, connectionOpen, noMoreDataTop, setNoMoreDataTop, noMoreDataBottom, setNoMoreDataBottom, methodsFilter, statusFilter, pathFilter}) => {
+export const EntriesList: React.FC<HarEntriesListProps> = ({entries, setEntries, focusedEntry, setFocusedEntry, connectionOpen, noMoreDataTop, setNoMoreDataTop, noMoreDataBottom, setNoMoreDataBottom, methodsFilter, statusFilter, pathFilter, listEntryREF, onScrollEvent, scrollableList}) => {
 
     const [loadMoreTop, setLoadMoreTop] = useState(false);
     const [isLoadingTop, setIsLoadingTop] = useState(false);
-
+    const scrollableRef = useRef(null);
+    
     useEffect(() => {
         const list = document.getElementById('list').firstElementChild;
         list.addEventListener('scroll', (e) => {
@@ -106,20 +111,25 @@ export const HarEntriesList: React.FC<HarEntriesListProps> = ({entries, setEntri
 
     return <>
             <div className={styles.list}>
-                <div id="list" className={styles.list}>
+                <div id="list" ref={listEntryREF} className={styles.list} >
                     {isLoadingTop && <div className={styles.spinnerContainer}>
                         <img alt="spinner" src={spinner} style={{height: 25}}/>
                     </div>}
-                    <ScrollableFeed>
+                    <ScrollableFeed ref={scrollableRef} onScroll={(isAtBottom) => onScrollEvent(isAtBottom)}>
                         {noMoreDataTop && !connectionOpen && <div id="noMoreDataTop" className={styles.noMoreDataAvailable}>No more data available</div>}
-                        {filteredEntries.map(entry => <HarEntry key={entry.id}
+                        {filteredEntries.map(entry => <EntryItem key={entry.id}
                                                      entry={entry}
-                                                     setFocusedEntryId={setFocusedEntryId}
-                                                     isSelected={focusedEntryId === entry.id}/>)}
+                                                     setFocusedEntry = {setFocusedEntry}
+                                                     isSelected={focusedEntry.id === entry.id}/>)}
                         {!connectionOpen && !noMoreDataBottom && <div className={styles.fetchButtonContainer}>
                             <div className={styles.styledButton} onClick={() => getNewEntries()}>Fetch more entries</div>
                         </div>}
                     </ScrollableFeed>
+                    <button type="button" 
+                        className={`${styles.btnLive} ${scrollableList ? styles.showButton : styles.hideButton}`} 
+                        onClick={(_) => scrollableRef.current.scrollToBottom()}>
+                        <img alt="down" src={down} />
+                    </button>
                 </div>
 
                 {entries?.length > 0 && <div className={styles.footer}>

ui/src/components/EntryDetailed/EntryDetailed.module.sass

@@ -0,0 +1,23 @@
+@import "src/variables.module"
+
+.content
+    font-family: "Source Sans Pro", Lucida Grande, Tahoma, sans-serif
+    height: calc(100% - 56px)
+    overflow-y: auto
+    width: 100%
+
+    .body
+        background: $main-background-color
+        color: $blue-gray
+        border-radius: 4px
+        padding: 10px
+        .bodyHeader
+            padding: 0 1rem
+        .endpointURL
+            font-size: .75rem
+            display: block
+            color: $blue-color
+            text-decoration: none
+            margin-bottom: .5rem
+            overflow-wrap: anywhere
+            padding: 5px 0

ui/src/components/EntryDetailed/EntryDetailed.tsx

@@ -0,0 +1,56 @@
+import React from "react";
+import styles from './EntryDetailed.module.sass';
+import {makeStyles} from "@material-ui/core";
+import {EntryType} from "../EntryListItem/EntryListItem";
+import {RestEntryDetailsTitle} from "./Rest/RestEntryDetailsTitle";
+import {KafkaEntryDetailsTitle} from "./Kafka/KafkaEntryDetailsTitle";
+import {RestEntryDetailsContent} from "./Rest/RestEntryDetailsContent";
+import {KafkaEntryDetailsContent} from "./Kafka/KafkaEntryDetailsContent";
+
+const useStyles = makeStyles(() => ({
+    entryTitle: {
+        display: 'flex',
+        minHeight: 46,
+        maxHeight: 46,
+        alignItems: 'center',
+        marginBottom: 8,
+        padding: 5,
+        paddingBottom: 0
+    }
+}));
+
+interface EntryDetailedProps {
+    entryData: any;
+    classes?: any;
+    entryType: string;
+}
+
+export const EntryDetailed: React.FC<EntryDetailedProps> = ({classes, entryData, entryType}) => {
+    const classesTitle = useStyles();
+
+    let title, content;
+
+    switch (entryType) {
+        case EntryType.Rest:
+            title = <RestEntryDetailsTitle entryData={entryData}/>;
+            content = <RestEntryDetailsContent entryData={entryData}/>;
+            break;
+        case EntryType.Kafka:
+            title = <KafkaEntryDetailsTitle entryData={entryData}/>;
+            content = <KafkaEntryDetailsContent entryData={entryData}/>;
+            break;
+        default:
+            title = <RestEntryDetailsTitle entryData={entryData}/>;
+            content = <RestEntryDetailsContent entryData={entryData}/>;
+            break;
+    }
+
+    return <>
+        <div className={classesTitle.entryTitle}>{title}</div>
+        <div className={styles.content}>
+            <div className={styles.body}>
+                {content}
+            </div>
+        </div>
+    </>
+};

ui/src/components/EntryDetailed/EntrySections.module.sass

@@ -1,4 +1,4 @@
-@import '../style/variables.module'
+@import 'src/variables.module'
 
 .title
     display: flex
@@ -92,3 +92,6 @@
         tr td:first-child
             white-space: nowrap
             padding-right: .5rem
+
+.noRules
+    padding: 0 1rem 1rem

ui/src/components/EntryDetailed/EntrySections.tsx

@@ -0,0 +1,213 @@
+import styles from "./EntrySections.module.sass";
+import React, {useState} from "react";
+import {SyntaxHighlighter} from "../UI/SyntaxHighlighter";
+import CollapsibleContainer from "../UI/CollapsibleContainer";
+import FancyTextDisplay from "../UI/FancyTextDisplay";
+import Checkbox from "../UI/Checkbox";
+import ProtobufDecoder from "protobuf-decoder";
+
+interface ViewLineProps {
+    label: string;
+    value: number | string;
+}
+
+const ViewLine: React.FC<ViewLineProps> = ({label, value}) => {
+    return (label && value && <tr className={styles.dataLine}>
+                <td className={styles.dataKey}>{label}</td>
+                <td>
+                    <FancyTextDisplay
+                        className={styles.dataValue}
+                        text={value}
+                        applyTextEllipsis={false}
+                        flipped={true}
+                        displayIconOnMouseOver={true}
+                    />
+                </td>
+            </tr>) || null;
+}
+
+interface SectionCollapsibleTitleProps {
+    title: string;
+    isExpanded: boolean;
+}
+
+const SectionCollapsibleTitle: React.FC<SectionCollapsibleTitleProps> = ({title, isExpanded}) => {
+    return <div className={styles.title}>
+        <span className={`${styles.button} ${isExpanded ? styles.expanded : ''}`}>
+            {isExpanded ? '-' : '+'}
+        </span>
+        <span>{title}</span>
+    </div>
+}
+
+interface SectionContainerProps {
+    title: string;
+}
+
+export const SectionContainer: React.FC<SectionContainerProps> = ({title, children}) => {
+    const [expanded, setExpanded] = useState(true);
+    return <CollapsibleContainer
+        className={styles.collapsibleContainer}
+        isExpanded={expanded}
+        onClick={() => setExpanded(!expanded)}
+        title={<SectionCollapsibleTitle title={title} isExpanded={expanded}/>}
+    >
+        {children}
+    </CollapsibleContainer>
+}
+
+interface BodySectionProps {
+    content: any;
+    encoding?: string;
+    contentType?: string;
+}
+
+export const BodySection: React.FC<BodySectionProps> = ({content, encoding, contentType}) => {
+    const MAXIMUM_BYTES_TO_HIGHLIGHT = 10000; // The maximum of chars to highlight in body, in case the response can be megabytes
+    const supportedLanguages = [['html', 'html'], ['json', 'json'], ['application/grpc', 'json']]; // [[indicator, languageToUse],...]
+    const jsonLikeFormats = ['json'];
+    const protobufFormats = ['application/grpc'];
+    const [isWrapped, setIsWrapped] = useState(false);
+
+    const formatTextBody = (body): string => {
+        const chunk = body.slice(0, MAXIMUM_BYTES_TO_HIGHLIGHT);
+        const bodyBuf = encoding === 'base64' ? atob(chunk) : chunk;
+
+        try {
+            if (jsonLikeFormats.some(format => content?.mimeType?.indexOf(format) > -1)) {
+                return JSON.stringify(JSON.parse(bodyBuf), null, 2);
+            } else if (protobufFormats.some(format => content?.mimeType?.indexOf(format) > -1)) {
+                // Replace all non printable characters (ASCII)
+                const protobufDecoder = new ProtobufDecoder(bodyBuf, true);
+                return JSON.stringify(protobufDecoder.decode().toSimple(), null, 2);
+            }
+        } catch (error) {
+            console.error(error);
+        }
+        return bodyBuf;
+    }
+
+    const getLanguage = (mimetype) => {
+        const chunk = content.text?.slice(0, 100);
+        if (chunk.indexOf('html') > 0 || chunk.indexOf('HTML') > 0) return supportedLanguages[0][1];
+        const language = supportedLanguages.find(el => (mimetype + contentType).indexOf(el[0]) > -1);
+        return language ? language[1] : 'default';
+    }
+
+    return <React.Fragment>
+        {content && content.text?.length > 0 && <SectionContainer title='Body'>
+            <table>
+                <tbody>
+                    <ViewLine label={'Mime type'} value={content?.mimeType}/>
+                    <ViewLine label={'Encoding'} value={encoding}/>
+                </tbody>
+            </table>
+
+            <div style={{display: 'flex', alignItems: 'center', alignContent: 'center', margin: "5px 0"}} onClick={() => setIsWrapped(!isWrapped)}>
+                <div style={{paddingTop: 3}}>
+                    <Checkbox checked={isWrapped} onToggle={() => {}}/>
+                </div>
+                <span style={{marginLeft: '.5rem'}}>Wrap text</span>
+            </div>
+
+            <SyntaxHighlighter
+                isWrapped={isWrapped}
+                code={formatTextBody(content.text)}
+                language={content?.mimeType ? getLanguage(content.mimeType) : 'default'}
+            />
+        </SectionContainer>}
+    </React.Fragment>
+}
+
+interface TableSectionProps {
+    title: string,
+    arrayToIterate: any[],
+}
+
+export const TableSection: React.FC<TableSectionProps> = ({title, arrayToIterate}) => {
+    return <React.Fragment>
+        {
+            arrayToIterate && arrayToIterate.length > 0 ?
+                <SectionContainer title={title}>
+                    <table>
+                        <tbody>
+                            {arrayToIterate.map(({name, value}, index) => <ViewLine key={index} label={name}
+                                                                                            value={value}/>)}
+                        </tbody>
+                    </table>
+                </SectionContainer> : <span/>
+        }
+    </React.Fragment>
+}
+
+interface HAREntryPolicySectionProps {
+    service: string,
+    title: string,
+    response: any,
+    latency?: number,
+    arrayToIterate: any[],
+}
+
+
+interface HAREntryPolicySectionCollapsibleTitleProps {
+    label: string;
+    matched: string;
+    isExpanded: boolean;
+}
+
+const HAREntryPolicySectionCollapsibleTitle: React.FC<HAREntryPolicySectionCollapsibleTitleProps> = ({label, matched, isExpanded}) => {
+    return <div className={styles.title}>
+        <span className={`${styles.button} ${isExpanded ? styles.expanded : ''}`}>
+            {isExpanded ? '-' : '+'}
+        </span>
+        <span>
+            <tr className={styles.dataLine}>
+            <td className={`${styles.dataKey} ${styles.rulesTitleSuccess}`}>{label}</td>
+            <td className={`${styles.dataKey} ${matched === 'Success' ? styles.rulesMatchedSuccess : styles.rulesMatchedFailure}`}>{matched}</td>
+            </tr>
+        </span>
+    </div>
+}
+
+interface HAREntryPolicySectionContainerProps {
+    label: string;
+    matched: string;
+    children?: any;
+}
+
+export const HAREntryPolicySectionContainer: React.FC<HAREntryPolicySectionContainerProps> = ({label, matched, children}) => {
+    const [expanded, setExpanded] = useState(false);
+    return <CollapsibleContainer
+        className={styles.collapsibleContainer}
+        isExpanded={expanded}
+        onClick={() => setExpanded(!expanded)}
+        title={<HAREntryPolicySectionCollapsibleTitle label={label} matched={matched} isExpanded={expanded}/>}
+    >
+        {children}
+    </CollapsibleContainer>
+}
+
+export const HAREntryTablePolicySection: React.FC<HAREntryPolicySectionProps> = ({service, title, response, latency, arrayToIterate}) => {
+    return <React.Fragment>
+        {arrayToIterate && arrayToIterate.length > 0 ? <>
+            <SectionContainer title={title}>
+                <table>
+                    <tbody>
+                        {arrayToIterate.map(({rule, matched}, index) => {
+                            return (<HAREntryPolicySectionContainer key={index} label={rule.Name} matched={matched && (rule.Type === 'latency' ? rule.Latency >= latency : true)? "Success" : "Failure"}>
+                                    <>
+                                        {rule.Key && <tr className={styles.dataValue}><td><b>Key</b>:</td><td>{rule.Key}</td></tr>}
+                                        {rule.Latency > 0 ? <tr className={styles.dataValue}><td><b>Latency</b>:</td><td>{rule.Latency}</td></tr> : ''}
+                                        {rule.Method && <tr className={styles.dataValue}><td><b>Method:</b></td> <td>{rule.Method}</td></tr>}
+                                        {rule.Path && <tr className={styles.dataValue}><td><b>Path:</b></td> <td>{rule.Path}</td></tr>}
+                                        {rule.Service && <tr className={styles.dataValue}><td><b>Service:</b></td> <td>{service}</td></tr>}
+                                        {rule.Type && <tr className={styles.dataValue}><td><b>Type:</b></td> <td>{rule.Type}</td></tr>}
+                                        {rule.Value && <tr className={styles.dataValue}><td><b>Value:</b></td> <td>{rule.Value}</td></tr>}
+                                    </>
+                                </HAREntryPolicySectionContainer>)})}
+                    </tbody>
+                </table>
+            </SectionContainer>
+        </> : <span className={styles.noRules}>No rules could be applied to this request.</span>}
+    </React.Fragment>
+}

ui/src/components/EntryDetailed/Kafka/KafkaEntryDetailsContent.tsx

@@ -0,0 +1,6 @@
+import React from "react";
+
+export const KafkaEntryDetailsContent: React.FC<any> = ({entryData}) => {
+
+    return <></>;
+}

ui/src/components/EntryDetailed/Kafka/KafkaEntryDetailsTitle.tsx

@@ -0,0 +1,6 @@
+import React from "react";
+
+export const KafkaEntryDetailsTitle: React.FC<any> = ({entryData}) => {
+
+    return <></>
+}

ui/src/components/EntryDetailed/Rest/RestEntryDetailsContent.tsx

@@ -0,0 +1,43 @@
+import React, {useState} from "react";
+import styles from "../EntryDetailed.module.sass";
+import Tabs from "../../UI/Tabs";
+import {BodySection, HAREntryTablePolicySection, TableSection} from "../EntrySections";
+import {singleEntryToHAR} from "../../../helpers/utils";
+
+const MIME_TYPE_KEY = 'mimeType';
+
+export const RestEntryDetailsContent: React.FC<any> = ({entryData}) => {
+
+    const har = singleEntryToHAR(entryData);
+    const {request, response, timings: {receive}} = har.log.entries[0].entry;
+    const rulesMatched = har.log.entries[0].rulesMatched
+    const TABS = [
+        {tab: 'request'},
+        {tab: 'response'},
+        {tab: 'Rules'},
+    ];
+
+    const [currentTab, setCurrentTab] = useState(TABS[0].tab);
+
+    return <>
+        <div className={styles.bodyHeader}>
+            <Tabs tabs={TABS} currentTab={currentTab} onChange={setCurrentTab} leftAligned/>
+            {request?.url && <a className={styles.endpointURL} href={request.url} target='_blank' rel="noreferrer">{request.url}</a>}
+        </div>
+        {currentTab === TABS[0].tab && <>
+                <TableSection title={'Headers'} arrayToIterate={request.headers}/>
+                <TableSection title={'Cookies'} arrayToIterate={request.cookies}/>
+                {request?.postData && <BodySection content={request.postData} encoding={request.postData.comment} contentType={request.postData[MIME_TYPE_KEY]}/>}
+                <TableSection title={'Query'} arrayToIterate={request.queryString}/>
+            </>
+        }
+        {currentTab === TABS[1].tab && <>
+            <TableSection title={'Headers'} arrayToIterate={response.headers}/>
+            <BodySection content={response.content} encoding={response.content?.encoding} contentType={response.content?.mimeType}/>
+            <TableSection title={'Cookies'} arrayToIterate={response.cookies}/>
+        </>}
+        {currentTab === TABS[2].tab && <>
+            <HAREntryTablePolicySection service={har.log.entries[0].service} title={'Rule'} latency={receive} response={response} arrayToIterate={rulesMatched ? rulesMatched : []}/>
+        </>}
+    </>;
+}

ui/src/components/EntryDetailed/Rest/RestEntryDetailsTitle.tsx

@@ -0,0 +1,26 @@
+import React from "react";
+import {singleEntryToHAR} from "../../../helpers/utils";
+import StatusCode from "../../UI/StatusCode";
+import {EndpointPath} from "../../UI/EndpointPath";
+
+const formatSize = (n: number) => n > 1000 ? `${Math.round(n / 1000)}KB` : `${n} B`;
+
+export const RestEntryDetailsTitle: React.FC<any> = ({entryData}) => {
+
+    const har = singleEntryToHAR(entryData);
+    const {log: {entries}} = har;
+    const {response, request, timings: {receive}} = entries[0].entry;
+    const {status, statusText, bodySize} = response;
+
+    return har && <>
+        {status && <div style={{marginRight: 8}}>
+            <StatusCode statusCode={status}/>
+        </div>}
+        <div style={{flexGrow: 1, overflow: 'hidden'}}>
+            <EndpointPath method={request?.method} path={request?.url}/>
+        </div>
+        <div style={{margin: "0 18px", opacity: 0.5}}>{formatSize(bodySize)}</div>
+        <div style={{marginRight: 18, opacity: 0.5}}>{status} {statusText}</div>
+        <div style={{marginRight: 18, opacity: 0.5}}>{Math.round(receive)}ms</div>
+    </>
+}

ui/src/components/EntryListItem/EntryListItem.module.sass

@@ -1,4 +1,4 @@
-@import 'variables.module'
+@import 'src/variables.module'
 
 .row
   display: flex
@@ -24,12 +24,40 @@
   margin-right: 3px
 
 .ruleSuccessRow
-  border: 1px $success-color solid
-  border-left: 5px $success-color solid
+  background: #E8FFF1
+
+.ruleSuccessRowSelected
+  border: 1px #6FCF97 solid
+  border-left: 5px #6FCF97 solid
+  margin-left: 10px
+  margin-right: 3px
 
 .ruleFailureRow
+  background: #FFE9EF
+
+.ruleFailureRowSelected
   border: 1px $failure-color solid
   border-left: 5px $failure-color solid
+  margin-left: 10px
+  margin-right: 3px
+
+.ruleNumberTextFailure
+  color: #DB2156
+  font-family: Source Sans Pro
+  font-style: normal
+  font-weight: 600
+  font-size: 12px
+  line-height: 15px
+  padding-right: 12px
+
+.ruleNumberTextSuccess
+  color: #219653
+  font-family: Source Sans Pro
+  font-style: normal
+  font-weight: 600
+  font-size: 12px
+  line-height: 15px
+  padding-right: 12px
 
 .service
   text-overflow: ellipsis
@@ -45,10 +73,11 @@
 .timestamp
   font-size: 12px
   color: $secondary-font-color
-  padding-left: 12px
   flex-shrink: 0
   width: 145px
   text-align: left
+  border-left: 1px solid $data-background-color
+  padding: 6px 0 6px 12px
 
 .endpointServiceContainer
   display: flex
@@ -60,6 +89,12 @@
 
 .directionContainer
   display: flex
-  border-right: 1px solid $data-background-color
-  padding: 4px
-  padding-right: 12px
+  padding: 4px 12px 4px 4px
+
+.icon
+  height: 14px
+  width: 50px
+  padding: 5px
+  background-color: white
+  border-radius: 15px
+  box-shadow: 1px 1px 9px -4px black

ui/src/components/EntryListItem/EntryListItem.tsx

@@ -0,0 +1,85 @@
+import React from "react";
+import styles from './EntryListItem.module.sass';
+import restIcon from '../assets/restIcon.svg';
+import kafkaIcon from '../assets/kafkaIcon.svg';
+import {RestEntry, RestEntryContent} from "./RestEntryContent";
+import {KafkaEntry, KafkaEntryContent} from "./KafkaEntryContent";
+
+export interface BaseEntry {
+    type: string;
+    timestamp: Date;
+    id: string;
+    rules: Rules;
+    latency: number;
+}
+
+interface Rules {
+    status: boolean;
+    latency: number;
+    numberOfRules: number;
+}
+
+interface EntryProps {
+    entry: RestEntry | KafkaEntry | any;
+    setFocusedEntry: (entry: RestEntry | KafkaEntry) => void;
+    isSelected?: boolean;
+}
+
+export enum EntryType {
+    Rest = "rest",
+    Kafka = "kafka"
+}
+
+export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntry, isSelected}) => {
+
+    let additionalRulesProperties = "";
+    let rule = 'latency' in entry.rules
+    if (rule) {
+        if (entry.rules.latency !== -1) {
+            if (entry.rules.latency >= entry.latency) {
+                additionalRulesProperties = styles.ruleSuccessRow
+            } else {
+                additionalRulesProperties = styles.ruleFailureRow
+            }
+            if (isSelected) {
+                additionalRulesProperties += ` ${entry.rules.latency >= entry.latency ? styles.ruleSuccessRowSelected : styles.ruleFailureRowSelected}`
+            }
+        } else {
+            if (entry.rules.status) {
+                additionalRulesProperties = styles.ruleSuccessRow
+            } else {
+                additionalRulesProperties = styles.ruleFailureRow
+            }
+            if (isSelected) {
+                additionalRulesProperties += ` ${entry.rules.status ? styles.ruleSuccessRowSelected : styles.ruleFailureRowSelected}`
+            }
+        }
+    }
+
+    let icon, content;
+
+    switch (entry.type) {
+        case EntryType.Rest:
+            content = <RestEntryContent entry={entry}/>;
+            icon = restIcon;
+            break;
+        case EntryType.Kafka:
+            content = <KafkaEntryContent entry={entry}/>;
+            icon = kafkaIcon;
+            break;
+        default:
+            content = <RestEntryContent entry={entry}/>;
+            icon = restIcon;
+            break;
+    }
+
+    return <>
+        <div id={entry.id} className={`${styles.row} ${isSelected && !rule ? styles.rowSelected : additionalRulesProperties}`}
+             onClick={() => setFocusedEntry(entry)}>
+            {icon && <div style={{width: 80}}>{<img className={styles.icon} alt="icon" src={icon}/>}</div>}
+            {content}
+            <div className={styles.timestamp}>{new Date(+entry.timestamp)?.toLocaleString()}</div>
+        </div>
+    </>
+};
+

ui/src/components/EntryListItem/KafkaEntryContent.tsx

@@ -0,0 +1,15 @@
+import {BaseEntry} from "./EntryListItem";
+import React from "react";
+
+export interface KafkaEntry extends BaseEntry{
+}
+
+interface KafkaEntryContentProps {
+    entry: KafkaEntry;
+}
+
+export const KafkaEntryContent: React.FC<KafkaEntryContentProps> = ({entry}) => {
+
+    return <>
+    </>
+}