Fixing bad conflict resolution

# Conflicts:
#	cli/config/configStruct.go
#	cli/mizu/config.go
#	tap/http_reader.go

.dockerignore

@@ -2,7 +2,7 @@
 .dockerignore
 .editorconfig
 .gitignore
-**/.env*
+.env.*
 Dockerfile
 Makefile
 LICENSE

.github/ISSUE_TEMPLATE/bug_report.md

@@ -12,9 +12,9 @@ A clear and concise description of what the bug is.
 
 **To Reproduce**
 Steps to reproduce the behavior:
-1. Run `mizu <command> ...`
-2. Click on '...'
-3. Scroll down to '...'
+1. Run mizu <command> '...'
+2. Click on '....'
+3. Scroll down to '....'
 4. See error
 
 **Expected behavior**
@@ -22,17 +22,17 @@ A clear and concise description of what you expected to happen.
 
 **Logs**
 Upload logs:
-1. Run the mizu command with `--set dump-logs=true` (e.g `mizu tap --set dump-logs=true`)
+1. Run the mizu command with `--set dump-logs=true` (e.g `mizu tap --set dump-logs=true`) 
 2. Try to reproduce the issue
-3. <kbd>CTRL</kbd>+<kbd>C</kbd> on terminal tab which runs `mizu`
-4. Upload the logs zip file from `~/.mizu/mizu_logs_**.zip`
+3. CNTRL+C on terminal tab which runs mizu
+4. Upload the logs zip file from ~/.mizu/mizu_logs_**.zip
 
 **Screenshots**
 If applicable, add screenshots to help explain your problem.
 
 **Desktop (please complete the following information):**
- - OS: [e.g. macOS]
- - Web Browser: [e.g. Google Chrome]
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome]
 
 **Additional context**
 Add any other context about the problem here.

.github/workflows/pr_validation.yml

@@ -1,15 +1,9 @@
 name: PR validation
-
 on:
   pull_request:
     branches:
       - 'develop'
       - 'main'
-
-concurrency:
-  group: mizu-pr-validation-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
   build-cli:
     name: Build CLI
@@ -18,7 +12,7 @@ jobs:
       - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: '1.16'
+          go-version: '^1.16'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
@@ -33,7 +27,7 @@ jobs:
       - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: '1.16'
+          go-version: '^1.16'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
@@ -44,3 +38,43 @@ jobs:
 
       - name: Build Agent
         run: make agent
+
+  run-tests-cli:
+    name: Run CLI tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: make test-cli
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2
+
+  run-tests-agent:
+    name: Run Agent tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - shell: bash
+        run: |
+          sudo apt-get install libpcap-dev
+
+      - name: Test
+        run: make test-agent
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2

.github/workflows/publish.yml

@@ -14,10 +14,6 @@ jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.16'
       - name: Checkout
         uses: actions/checkout@v2
       - name: Set up Cloud SDK

.github/workflows/tests_validation.yml

@@ -1,56 +0,0 @@
-name: tests validation
-
-on:
-  pull_request:
-    branches:
-      - 'develop'
-      - 'main'
-  push:
-    branches:
-      - 'develop'
-      - 'main'
-
-concurrency:
-  group: mizu-tests-validation-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  run-tests-cli:
-    name: Run CLI tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
-        with:
-          go-version: '^1.16'
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Test
-        run: make test-cli
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2
-
-  run-tests-agent:
-    name: Run Agent tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
-        with:
-          go-version: '^1.16'
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - shell: bash
-        run: |
-          sudo apt-get install libpcap-dev
-
-      - name: Test
-        run: make test-agent
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2

.gitignore

@@ -19,13 +19,3 @@ build
 
 # Mac OS
 .DS_Store
-.vscode/
-
-# Ignore the scripts that are created for development
-*dev.*
-
-# Environment variables
-.env
-
-# pprof
-pprof/*

CONTRIBUTE.md

@@ -1,20 +1,18 @@
-![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg)
-
-# Contributing to Mizu
-
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+# CONTRIBUTE
 We welcome code contributions from the community.
 Please read and follow the guidelines below.
 
 ## Communication
-
 * Before starting work on a major feature, please reach out to us via [GitHub](https://github.com/up9inc/mizu), [Slack](https://join.slack.com/share/zt-u6bbs3pg-X1zhQOXOH0yEoqILgH~csw), [email](mailto:mizu@up9.com), etc. We will make sure no one else is already working on it. A _major feature_ is defined as any change that is > 100 LOC altered (not including tests), or changes any user-facing behavior
 * Small patches and bug fixes don't need prior communication.
 
-## Contribution Requirements
-
+## Contribution requirements
 * Code style - most of the code is written in Go, please follow [these guidelines](https://golang.org/doc/effective_go)
-* Go-tools compatible (`go get`, `go test`, etc.)
-* Code coverage for unit tests must not decrease.
+* Go-tools compatible (`go get`, `go test`, etc)
+* Unit-test coverage can’t go down ..
 * Code must be usefully commented. Not only for developers on the project, but also for external users of these packages
 * When reviewing PRs, you are encouraged to use Golang's [code review comments page](https://github.com/golang/go/wiki/CodeReviewComments)
-* Project follows [Google JSON Style Guide](https://google.github.io/styleguide/jsoncstyleguide.xml) for the REST APIs that are provided.
+
+
+

Dockerfile

@@ -2,10 +2,8 @@ FROM node:14-slim AS site-build
 
 WORKDIR /app/ui-build
 
-COPY ui/package.json .
-COPY ui/package-lock.json .
-RUN npm i
 COPY ui .
+RUN npm i
 RUN npm run build
 
 
@@ -13,7 +11,7 @@ FROM golang:1.16-alpine AS builder
 # Set necessary environment variables needed for our image.
 ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 
-RUN apk add libpcap-dev gcc g++ make bash
+RUN apk add libpcap-dev gcc g++ make
 
 # Move to agent working directory (/agent-build).
 WORKDIR /app/agent-build
@@ -21,7 +19,6 @@ WORKDIR /app/agent-build
 COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
 COPY tap/go.mod tap/go.mod ../tap/
-COPY tap/api/go.* ../tap/api/
 RUN go mod download
 # cheap trick to make the build faster (As long as go.mod wasn't changes)
 RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
@@ -41,8 +38,6 @@ RUN go build -ldflags="-s -w \
      -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
      -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
-COPY devops/build_extensions.sh ..
-RUN cd .. && /bin/bash build_extensions.sh
 
 FROM alpine:3.13.5
 
@@ -51,7 +46,6 @@ WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
-COPY --from=builder ["/app/agent/build/extensions", "extensions"]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 
 # gin-gonic runs in debug mode without this

Makefile

@@ -23,7 +23,7 @@ export SEM_VER?=0.0.0
 
 ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
-	@ls -l ui/build
+	@ls -l ui/build  
 
 cli: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build
@@ -34,7 +34,6 @@ build-cli-ci: ## Build CLI for CI.
 agent: ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
-	${MAKE} extensions
 	@ls -l agent/build
 
 docker: ## Build and publish agent docker image.
@@ -44,15 +43,11 @@ push: push-docker push-cli ## Build and publish agent docker image & CLI.
 
 push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
-	devops/build-push-featurebranch.sh
-
-push-docker-debug:
-	@echo "publishing debug Docker image .. "
-	devops/build-push-featurebranch-debug.sh
+	./build-push-featurebranch.sh
 
 build-docker-ci: ## Build agent docker image for CI.
 	@echo "building docker image for ci"
-	devops/build-agent-ci.sh
+	./build-agent-ci.sh
 
 push-cli: ## Build and publish CLI.
 	@echo "publishing CLI .. "
@@ -76,9 +71,6 @@ clean-cli:  ## Clean CLI.
 clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
-extensions:
-	devops/build_extensions.sh
-
 test-cli:
 	@echo "running cli tests"; cd cli && $(MAKE) test
 

PERMISSIONS.md

@@ -1,4 +1,4 @@
-![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg)
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
 # Kubernetes permissions for MIZU  
 
 This document describes in details all permissions required for full and correct operation of Mizu

README.md

@@ -2,9 +2,7 @@
 
 # The API Traffic Viewer for Kubernetes
 
-A simple-yet-powerful API traffic viewer for Kubernetes enabling you to view all API communication between microservices to help your debug and troubleshoot regressions.
-
-Think TCPDump and Chrome Dev Tools combined.
+A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined
 
 ![Simple UI](assets/mizu-ui.png)
 
@@ -15,10 +13,6 @@ Think TCPDump and Chrome Dev Tools combined.
 - No installation or code instrumentation
 - Works completely on premises
 
-## Requirements
-
-A Kubernetes server version of 1.16.0 or higher is required.
-
 ## Download
 
 Download Mizu for your platform and operating system
@@ -44,13 +38,11 @@ SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/
 ### Development (unstable) Build
 Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page
 
-## Kubeconfig & Permissions
-While `mizu`most often works out of the box, you can influence its behavior:
-
-1. [OPTIONAL] Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
+## Prerequisites
+1. Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
 2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination)
 
-For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document
+For detailed list of k8s permissions see [PERMISSIONS](PERMISSIONS.md) document
 
 
 ## How to Run
@@ -147,29 +139,23 @@ Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
 User-agent filtering (like health checks) - can be configured using command-line options:
 
 ```shell
-$ mizu tap "^ca.*" --set tap.ignored-user-agents=kube-probe --set tap.ignored-user-agents=prometheus
+$ mizu tap "^ca.*" --set ignored-user-agents=kube-probe --set ignored-user-agents=prometheus
 +carts-66c77f5fbb-fq65r
 +catalogue-5f4cb7cf5-7zrmn
 Web interface is now available at http://localhost:8899
 ^C
 
 ```
+
 Any request that contains `User-Agent` header with one of the specified values (`kube-probe` or `prometheus`) will not be captured
 
-### Traffic validation rules
+### API Rules validation
 
-This feature allows you to define set of simple rules, and test the traffic against them.
+This feature allows you to define set of simple rules, and test the API against them.
 Such validation may test response for specific JSON fields, headers, etc.
 
-Please see [TRAFFIC RULES](docs/POLICY_RULES.md) page for more details and syntax.
+Please see [API RULES](docs/POLICY_RULES.md) page for more details and syntax.
 
-### OpenAPI Specification (OAS) Contract Monitoring
-
-An OAS/Swagger file can contain schemas under `parameters` and `responses` fields. With `--contract catalogue.yaml`
-CLI option, you can pass your API description to Mizu and the traffic will automatically be validated
-against the contracts.
-
-Please see [CONTRACT MONITORING](docs/CONTRACT_MONITORING.md) page for more details and syntax.
 
 ## How to Run local UI
 

TESTING.md

@@ -0,0 +1,15 @@
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+# TESTING
+Testing guidelines for Mizu project 
+
+## Unit-tests
+* TBD 
+* TBD
+* TBD
+
+
+
+## System tests
+* TBD 
+* TBD
+* TBD

acceptanceTests/Makefile

@@ -1,2 +1,2 @@
 test: ## Run acceptance tests.
-	@go test ./... -timeout 1h
+	@go test ./...

acceptanceTests/config_test.go

@@ -1,283 +0,0 @@
-package acceptanceTests
-
-import (
-	"fmt"
-	"gopkg.in/yaml.v3"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"testing"
-)
-
-type tapConfig struct {
-	GuiPort uint16 `yaml:"gui-port"`
-}
-
-type configStruct struct {
-	Tap tapConfig `yaml:"tap"`
-}
-
-func TestConfigRegenerate(t *testing.T) {
-	if testing.Short() {
-		t.Skip("ignored acceptance test")
-	}
-
-	cliPath, cliPathErr := getCliPath()
-	if cliPathErr != nil {
-		t.Errorf("failed to get cli path, err: %v", cliPathErr)
-		return
-	}
-
-	configPath, configPathErr := getConfigPath()
-	if configPathErr != nil {
-		t.Errorf("failed to get config path, err: %v", cliPathErr)
-		return
-	}
-
-	configCmdArgs := getDefaultConfigCommandArgs()
-
-	configCmdArgs = append(configCmdArgs, "-r")
-
-	configCmd := exec.Command(cliPath, configCmdArgs...)
-	t.Logf("running command: %v", configCmd.String())
-
-	t.Cleanup(func() {
-		if err := os.Remove(configPath); err != nil {
-			t.Logf("failed to delete config file, err: %v", err)
-		}
-	})
-
-	if err := configCmd.Start(); err != nil {
-		t.Errorf("failed to start config command, err: %v", err)
-		return
-	}
-
-	if err := configCmd.Wait(); err != nil {
-		t.Errorf("failed to wait config command, err: %v", err)
-		return
-	}
-
-	_, readFileErr := ioutil.ReadFile(configPath)
-	if readFileErr != nil {
-		t.Errorf("failed to read config file, err: %v", readFileErr)
-		return
-	}
-}
-
-func TestConfigGuiPort(t *testing.T) {
-	if testing.Short() {
-		t.Skip("ignored acceptance test")
-	}
-
-	tests := []uint16{8898}
-
-	for _, guiPort := range tests {
-		t.Run(fmt.Sprintf("%d", guiPort), func(t *testing.T) {
-			cliPath, cliPathErr := getCliPath()
-			if cliPathErr != nil {
-				t.Errorf("failed to get cli path, err: %v", cliPathErr)
-				return
-			}
-
-			configPath, configPathErr := getConfigPath()
-			if configPathErr != nil {
-				t.Errorf("failed to get config path, err: %v", cliPathErr)
-				return
-			}
-
-			config := configStruct{}
-			config.Tap.GuiPort = guiPort
-
-			configBytes, marshalErr := yaml.Marshal(config)
-			if marshalErr != nil {
-				t.Errorf("failed to marshal config, err: %v", marshalErr)
-				return
-			}
-
-			if writeErr := ioutil.WriteFile(configPath, configBytes, 0644); writeErr != nil {
-				t.Errorf("failed to write config to file, err: %v", writeErr)
-				return
-			}
-
-			tapCmdArgs := getDefaultTapCommandArgs()
-
-			tapNamespace := getDefaultTapNamespace()
-			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-
-			tapCmd := exec.Command(cliPath, tapCmdArgs...)
-			t.Logf("running command: %v", tapCmd.String())
-
-			t.Cleanup(func() {
-				if err := cleanupCommand(tapCmd); err != nil {
-					t.Logf("failed to cleanup tap command, err: %v", err)
-				}
-
-				if err := os.Remove(configPath); err != nil {
-					t.Logf("failed to delete config file, err: %v", err)
-				}
-			})
-
-			if err := tapCmd.Start(); err != nil {
-				t.Errorf("failed to start tap command, err: %v", err)
-				return
-			}
-
-			apiServerUrl := getApiServerUrl(guiPort)
-
-			if err := waitTapPodsReady(apiServerUrl); err != nil {
-				t.Errorf("failed to start tap pods on time, err: %v", err)
-				return
-			}
-		})
-	}
-}
-
-func TestConfigSetGuiPort(t *testing.T) {
-	if testing.Short() {
-		t.Skip("ignored acceptance test")
-	}
-
-	tests := []struct {
-		ConfigFileGuiPort uint16
-		SetGuiPort        uint16
-	}{
-		{ConfigFileGuiPort: 8898, SetGuiPort: 8897},
-	}
-
-	for _, guiPortStruct := range tests {
-		t.Run(fmt.Sprintf("%d", guiPortStruct.SetGuiPort), func(t *testing.T) {
-			cliPath, cliPathErr := getCliPath()
-			if cliPathErr != nil {
-				t.Errorf("failed to get cli path, err: %v", cliPathErr)
-				return
-			}
-
-			configPath, configPathErr := getConfigPath()
-			if configPathErr != nil {
-				t.Errorf("failed to get config path, err: %v", cliPathErr)
-				return
-			}
-
-			config := configStruct{}
-			config.Tap.GuiPort = guiPortStruct.ConfigFileGuiPort
-
-			configBytes, marshalErr := yaml.Marshal(config)
-			if marshalErr != nil {
-				t.Errorf("failed to marshal config, err: %v", marshalErr)
-				return
-			}
-
-			if writeErr := ioutil.WriteFile(configPath, configBytes, 0644); writeErr != nil {
-				t.Errorf("failed to write config to file, err: %v", writeErr)
-				return
-			}
-
-			tapCmdArgs := getDefaultTapCommandArgs()
-
-			tapNamespace := getDefaultTapNamespace()
-			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-
-			tapCmdArgs = append(tapCmdArgs, "--set", fmt.Sprintf("tap.gui-port=%v", guiPortStruct.SetGuiPort))
-
-			tapCmd := exec.Command(cliPath, tapCmdArgs...)
-			t.Logf("running command: %v", tapCmd.String())
-
-			t.Cleanup(func() {
-				if err := cleanupCommand(tapCmd); err != nil {
-					t.Logf("failed to cleanup tap command, err: %v", err)
-				}
-
-				if err := os.Remove(configPath); err != nil {
-					t.Logf("failed to delete config file, err: %v", err)
-				}
-			})
-
-			if err := tapCmd.Start(); err != nil {
-				t.Errorf("failed to start tap command, err: %v", err)
-				return
-			}
-
-			apiServerUrl := getApiServerUrl(guiPortStruct.SetGuiPort)
-
-			if err := waitTapPodsReady(apiServerUrl); err != nil {
-				t.Errorf("failed to start tap pods on time, err: %v", err)
-				return
-			}
-		})
-	}
-}
-
-func TestConfigFlagGuiPort(t *testing.T) {
-	if testing.Short() {
-		t.Skip("ignored acceptance test")
-	}
-
-	tests := []struct {
-		ConfigFileGuiPort uint16
-		FlagGuiPort       uint16
-	}{
-		{ConfigFileGuiPort: 8898, FlagGuiPort: 8896},
-	}
-
-	for _, guiPortStruct := range tests {
-		t.Run(fmt.Sprintf("%d", guiPortStruct.FlagGuiPort), func(t *testing.T) {
-			cliPath, cliPathErr := getCliPath()
-			if cliPathErr != nil {
-				t.Errorf("failed to get cli path, err: %v", cliPathErr)
-				return
-			}
-
-			configPath, configPathErr := getConfigPath()
-			if configPathErr != nil {
-				t.Errorf("failed to get config path, err: %v", cliPathErr)
-				return
-			}
-
-			config := configStruct{}
-			config.Tap.GuiPort = guiPortStruct.ConfigFileGuiPort
-
-			configBytes, marshalErr := yaml.Marshal(config)
-			if marshalErr != nil {
-				t.Errorf("failed to marshal config, err: %v", marshalErr)
-				return
-			}
-
-			if writeErr := ioutil.WriteFile(configPath, configBytes, 0644); writeErr != nil {
-				t.Errorf("failed to write config to file, err: %v", writeErr)
-				return
-			}
-
-			tapCmdArgs := getDefaultTapCommandArgs()
-
-			tapNamespace := getDefaultTapNamespace()
-			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-
-			tapCmdArgs = append(tapCmdArgs, "-p", fmt.Sprintf("%v", guiPortStruct.FlagGuiPort))
-
-			tapCmd := exec.Command(cliPath, tapCmdArgs...)
-			t.Logf("running command: %v", tapCmd.String())
-
-			t.Cleanup(func() {
-				if err := cleanupCommand(tapCmd); err != nil {
-					t.Logf("failed to cleanup tap command, err: %v", err)
-				}
-
-				if err := os.Remove(configPath); err != nil {
-					t.Logf("failed to delete config file, err: %v", err)
-				}
-			})
-
-			if err := tapCmd.Start(); err != nil {
-				t.Errorf("failed to start tap command, err: %v", err)
-				return
-			}
-
-			apiServerUrl := getApiServerUrl(guiPortStruct.FlagGuiPort)
-
-			if err := waitTapPodsReady(apiServerUrl); err != nil {
-				t.Errorf("failed to start tap pods on time, err: %v", err)
-				return
-			}
-		})
-	}
-}

acceptanceTests/go.mod

@@ -1,10 +1,3 @@
 module github.com/up9inc/mizu/tests
 
 go 1.16
-
-require (
-	github.com/up9inc/mizu/shared v0.0.0
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
-)
-
-replace github.com/up9inc/mizu/shared v0.0.0 => ../shared

acceptanceTests/go.sum

@@ -1,8 +0,0 @@
-github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0=
-github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

acceptanceTests/logs_test.go

@@ -1,196 +0,0 @@
-package acceptanceTests
-
-import (
-	"archive/zip"
-	"os/exec"
-	"testing"
-)
-
-func TestLogs(t *testing.T) {
-	if testing.Short() {
-		t.Skip("ignored acceptance test")
-	}
-
-	cliPath, cliPathErr := getCliPath()
-	if cliPathErr != nil {
-		t.Errorf("failed to get cli path, err: %v", cliPathErr)
-		return
-	}
-
-	tapCmdArgs := getDefaultTapCommandArgs()
-
-	tapNamespace := getDefaultTapNamespace()
-	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-
-	tapCmd := exec.Command(cliPath, tapCmdArgs...)
-	t.Logf("running command: %v", tapCmd.String())
-
-	t.Cleanup(func() {
-		if err := cleanupCommand(tapCmd); err != nil {
-			t.Logf("failed to cleanup tap command, err: %v", err)
-		}
-	})
-
-	if err := tapCmd.Start(); err != nil {
-		t.Errorf("failed to start tap command, err: %v", err)
-		return
-	}
-
-	apiServerUrl := getApiServerUrl(defaultApiServerPort)
-
-	if err := waitTapPodsReady(apiServerUrl); err != nil {
-		t.Errorf("failed to start tap pods on time, err: %v", err)
-		return
-	}
-
-	logsCmdArgs := getDefaultLogsCommandArgs()
-
-	logsCmd := exec.Command(cliPath, logsCmdArgs...)
-	t.Logf("running command: %v", logsCmd.String())
-
-	if err := logsCmd.Start(); err != nil {
-		t.Errorf("failed to start logs command, err: %v", err)
-		return
-	}
-
-	if err := logsCmd.Wait(); err != nil {
-		t.Errorf("failed to wait logs command, err: %v", err)
-		return
-	}
-
-	logsPath, logsPathErr := getLogsPath()
-	if logsPathErr != nil {
-		t.Errorf("failed to get logs path, err: %v", logsPathErr)
-		return
-	}
-
-	zipReader, zipError := zip.OpenReader(logsPath)
-	if zipError != nil {
-		t.Errorf("failed to get zip reader, err: %v", zipError)
-		return
-	}
-
-	t.Cleanup(func() {
-		if err := zipReader.Close(); err != nil {
-			t.Logf("failed to close zip reader, err: %v", err)
-		}
-	})
-
-	var logsFileNames []string
-	for _, file := range zipReader.File {
-		logsFileNames = append(logsFileNames, file.Name)
-	}
-
-	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
-		t.Errorf("api server logs not found")
-		return
-	}
-
-	if !Contains(logsFileNames, "mizu_cli.log") {
-		t.Errorf("cli logs not found")
-		return
-	}
-
-	if !Contains(logsFileNames, "mizu_events.log") {
-		t.Errorf("events logs not found")
-		return
-	}
-
-	if !ContainsPartOfValue(logsFileNames, "mizu.mizu-tapper-daemon-set") {
-		t.Errorf("tapper logs not found")
-		return
-	}
-}
-
-func TestLogsPath(t *testing.T) {
-	if testing.Short() {
-		t.Skip("ignored acceptance test")
-	}
-
-	cliPath, cliPathErr := getCliPath()
-	if cliPathErr != nil {
-		t.Errorf("failed to get cli path, err: %v", cliPathErr)
-		return
-	}
-
-	tapCmdArgs := getDefaultTapCommandArgs()
-
-	tapNamespace := getDefaultTapNamespace()
-	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-
-	tapCmd := exec.Command(cliPath, tapCmdArgs...)
-	t.Logf("running command: %v", tapCmd.String())
-
-	t.Cleanup(func() {
-		if err := cleanupCommand(tapCmd); err != nil {
-			t.Logf("failed to cleanup tap command, err: %v", err)
-		}
-	})
-
-	if err := tapCmd.Start(); err != nil {
-		t.Errorf("failed to start tap command, err: %v", err)
-		return
-	}
-
-	apiServerUrl := getApiServerUrl(defaultApiServerPort)
-
-	if err := waitTapPodsReady(apiServerUrl); err != nil {
-		t.Errorf("failed to start tap pods on time, err: %v", err)
-		return
-	}
-
-	logsCmdArgs := getDefaultLogsCommandArgs()
-
-	logsPath := "../logs.zip"
-	logsCmdArgs = append(logsCmdArgs, "-f", logsPath)
-
-	logsCmd := exec.Command(cliPath, logsCmdArgs...)
-	t.Logf("running command: %v", logsCmd.String())
-
-	if err := logsCmd.Start(); err != nil {
-		t.Errorf("failed to start logs command, err: %v", err)
-		return
-	}
-
-	if err := logsCmd.Wait(); err != nil {
-		t.Errorf("failed to wait logs command, err: %v", err)
-		return
-	}
-
-	zipReader, zipError := zip.OpenReader(logsPath)
-	if zipError != nil {
-		t.Errorf("failed to get zip reader, err: %v", zipError)
-		return
-	}
-
-	t.Cleanup(func() {
-		if err := zipReader.Close(); err != nil {
-			t.Logf("failed to close zip reader, err: %v", err)
-		}
-	})
-
-	var logsFileNames []string
-	for _, file := range zipReader.File {
-		logsFileNames = append(logsFileNames, file.Name)
-	}
-
-	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
-		t.Errorf("api server logs not found")
-		return
-	}
-
-	if !Contains(logsFileNames, "mizu_cli.log") {
-		t.Errorf("cli logs not found")
-		return
-	}
-
-	if !Contains(logsFileNames, "mizu_events.log") {
-		t.Errorf("events logs not found")
-		return
-	}
-
-	if !ContainsPartOfValue(logsFileNames, "mizu.mizu-tapper-daemon-set") {
-		t.Errorf("tapper logs not found")
-		return
-	}
-}

acceptanceTests/setup.sh

@@ -26,21 +26,14 @@ fi
 echo "Starting minikube..."
 minikube start
 
-echo "Creating mizu tests namespaces"
+echo "Creating mizu tests namespace"
 kubectl create namespace mizu-tests
-kubectl create namespace mizu-tests2
 
-echo "Creating httpbin deployments"
+echo "Creating httpbin deployment"
 kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests
-kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests
 
-kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2
-
-echo "Creating httpbin services"
+echo "Creating httpbin service"
 kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
-kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests
-
-kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2
 
 echo "Starting proxy"
 kubectl proxy --port=8080 &

acceptanceTests/testsUtils.go

@@ -1,31 +1,18 @@
 package acceptanceTests
 
 import (
-	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
 	"path"
-	"strings"
 	"syscall"
-	"time"
-
-	"github.com/up9inc/mizu/shared"
 )
 
-const (
-	longRetriesCount     = 100
-	shortRetriesCount    = 10
-	defaultApiServerPort = shared.DefaultApiServerPort
-	defaultNamespaceName = "mizu-tests"
-	defaultServiceName   = "httpbin"
-	defaultEntriesCount  = 50
-)
-
-func getCliPath() (string, error) {
+func GetCliPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {
 		return "", filePathErr
@@ -35,120 +22,34 @@ func getCliPath() (string, error) {
 	return cliPath, nil
 }
 
-func getMizuFolderPath() (string, error) {
-	home, homeDirErr := os.UserHomeDir()
-	if homeDirErr != nil {
-		return "", homeDirErr
-	}
-
-	return path.Join(home, ".mizu"), nil
-}
-
-func getConfigPath() (string, error) {
-	mizuFolderPath, mizuPathError := getMizuFolderPath()
-	if mizuPathError != nil {
-		return "", mizuPathError
-	}
-
-	return path.Join(mizuFolderPath, "config.yaml"), nil
-}
-
-func getProxyUrl(namespace string, service string) string {
-	return fmt.Sprintf("http://localhost:8080/api/v1/namespaces/%v/services/%v/proxy", namespace, service)
-}
-
-func getApiServerUrl(port uint16) string {
-	return fmt.Sprintf("http://localhost:%v/mizu", port)
-}
-
-func getDefaultCommandArgs() []string {
+func GetDefaultCommandArgs() []string {
 	setFlag := "--set"
 	telemetry := "telemetry=false"
+
+	return []string{setFlag, telemetry}
+}
+
+func GetDefaultTapCommandArgs() []string {
+	tapCommand := "tap"
+	setFlag := "--set"
+	namespaces := "tap.namespaces=mizu-tests"
 	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
 	imagePullPolicy := "image-pull-policy=Never"
 
-	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy}
+	defaultCmdArgs := GetDefaultCommandArgs()
+
+	return append([]string{tapCommand, setFlag, namespaces, setFlag, agentImage, setFlag, imagePullPolicy}, defaultCmdArgs...)
 }
 
-func getDefaultTapCommandArgs() []string {
-	tapCommand := "tap"
-	defaultCmdArgs := getDefaultCommandArgs()
+func GetDefaultFetchCommandArgs() []string {
+	tapCommand := "fetch"
+
+	defaultCmdArgs := GetDefaultCommandArgs()
 
 	return append([]string{tapCommand}, defaultCmdArgs...)
 }
 
-func getDefaultTapCommandArgsWithRegex(regex string) []string {
-	tapCommand := "tap"
-	defaultCmdArgs := getDefaultCommandArgs()
-
-	return append([]string{tapCommand, regex}, defaultCmdArgs...)
-}
-
-func getDefaultLogsCommandArgs() []string {
-	logsCommand := "logs"
-	defaultCmdArgs := getDefaultCommandArgs()
-
-	return append([]string{logsCommand}, defaultCmdArgs...)
-}
-
-func getDefaultTapNamespace() []string {
-	return []string{"-n", "mizu-tests"}
-}
-
-func getDefaultConfigCommandArgs() []string {
-	configCommand := "config"
-	defaultCmdArgs := getDefaultCommandArgs()
-
-	return append([]string{configCommand}, defaultCmdArgs...)
-}
-
-func retriesExecute(retriesCount int, executeFunc func() error) error {
-	var lastError interface{}
-
-	for i := 0; i < retriesCount; i++ {
-		if err := tryExecuteFunc(executeFunc); err != nil {
-			lastError = err
-
-			time.Sleep(1 * time.Second)
-			continue
-		}
-
-		return nil
-	}
-
-	return fmt.Errorf("reached max retries count, retries count: %v, last err: %v", retriesCount, lastError)
-}
-
-func tryExecuteFunc(executeFunc func() error) (err interface{}) {
-	defer func() {
-		if panicErr := recover(); panicErr != nil {
-			err = panicErr
-		}
-	}()
-
-	return executeFunc()
-}
-
-func waitTapPodsReady(apiServerUrl string) error {
-	resolvingUrl := fmt.Sprintf("%v/status/tappersCount", apiServerUrl)
-	tapPodsReadyFunc := func() error {
-		requestResult, requestErr := executeHttpGetRequest(resolvingUrl)
-		if requestErr != nil {
-			return requestErr
-		}
-
-		tappersCount := requestResult.(float64)
-		if tappersCount == 0 {
-			return fmt.Errorf("no tappers running")
-		}
-
-		return nil
-	}
-
-	return retriesExecute(longRetriesCount, tapPodsReadyFunc)
-}
-
-func jsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
+func JsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
 	var result interface{}
 	if parseErr := json.Unmarshal(jsonBytes, &result); parseErr != nil {
 		return nil, parseErr
@@ -157,54 +58,23 @@ func jsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
 	return result, nil
 }
 
-func executeHttpRequest(response *http.Response, requestErr error) (interface{}, error) {
+func ExecuteHttpRequest(url string) (interface{}, error) {
+	response, requestErr := http.Get(url)
 	if requestErr != nil {
 		return nil, requestErr
 	} else if response.StatusCode != 200 {
 		return nil, fmt.Errorf("invalid status code %v", response.StatusCode)
 	}
 
-	defer func() { response.Body.Close() }()
-
 	data, readErr := ioutil.ReadAll(response.Body)
 	if readErr != nil {
 		return nil, readErr
 	}
 
-	return jsonBytesToInterface(data)
+	return JsonBytesToInterface(data)
 }
 
-func executeHttpGetRequestWithHeaders(url string, headers map[string]string) (interface{}, error) {
-	request, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	for headerKey, headerValue := range headers {
-		request.Header.Add(headerKey, headerValue)
-	}
-
-	client := &http.Client{}
-	response, requestErr := client.Do(request)
-	return executeHttpRequest(response, requestErr)
-}
-
-func executeHttpGetRequest(url string) (interface{}, error) {
-	response, requestErr := http.Get(url)
-	return executeHttpRequest(response, requestErr)
-}
-
-func executeHttpPostRequest(url string, body interface{}) (interface{}, error) {
-	requestBody, jsonErr := json.Marshal(body)
-	if jsonErr != nil {
-		return nil, jsonErr
-	}
-
-	response, requestErr := http.Post(url, "application/json", bytes.NewBuffer(requestBody))
-	return executeHttpRequest(response, requestErr)
-}
-
-func cleanupCommand(cmd *exec.Cmd) error {
+func CleanupCommand(cmd *exec.Cmd) error {
 	if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
 		return err
 	}
@@ -216,44 +86,28 @@ func cleanupCommand(cmd *exec.Cmd) error {
 	return nil
 }
 
-func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) {
-	tapStatus := tapStatusInterface.(map[string]interface{})
-	podsInterface := tapStatus["pods"].([]interface{})
-
-	var pods []map[string]interface{}
-	for _, podInterface := range podsInterface {
-		pods = append(pods, podInterface.(map[string]interface{}))
+func GetEntriesFromHarBytes(harBytes []byte) ([]interface{}, error){
+	harInterface, convertErr := JsonBytesToInterface(harBytes)
+	if convertErr != nil {
+		return nil, convertErr
 	}
 
-	return pods, nil
-}
-
-func getLogsPath() (string, error) {
-	dir, filePathErr := os.Getwd()
-	if filePathErr != nil {
-		return "", filePathErr
+	har, ok := harInterface.(map[string]interface{})
+	if !ok {
+		return nil, errors.New("invalid har type")
 	}
 
-	logsPath := path.Join(dir, "mizu_logs.zip")
-	return logsPath, nil
-}
-
-func Contains(slice []string, containsValue string) bool {
-	for _, sliceValue := range slice {
-		if sliceValue == containsValue {
-			return true
-		}
+	harLogInterface := har["log"]
+	harLog, ok :=  harLogInterface.(map[string]interface{})
+	if !ok {
+		return nil, errors.New("invalid har log type")
 	}
 
-	return false
-}
-
-func ContainsPartOfValue(slice []string, containsValue string) bool {
-	for _, sliceValue := range slice {
-		if strings.Contains(sliceValue, containsValue) {
-			return true
-		}
+	harEntriesInterface := harLog["entries"]
+	harEntries, ok := harEntriesInterface.([]interface{})
+	if !ok {
+		return nil, errors.New("invalid har entries type")
 	}
 
-	return false
+	return harEntries, nil
 }

agent/README.md

@@ -1,6 +1,7 @@
 # mizu agent
 Agent for MIZU (API server and tapper)
 Basic APIs:
+* /fetch - retrieve traffic data
 * /stats - retrieve statistics of collected data
 * /viewer - web ui
 
@@ -12,8 +13,7 @@ Basic APIs:
    `docker build . -t  gcr.io/up9-docker-hub/mizu/debug:latest -f debug.Dockerfile && docker push gcr.io/up9-docker-hub/mizu/debug:latest`
 
 ### Connecting
-1. Start mizu using the cli with the debug
-   image `mizu tap --set agent-image=gcr.io/up9-docker-hub/mizu/debug:latest {tapped_pod_name}`
+1. Start mizu using the cli with the debug image `mizu tap --mizu-image gcr.io/up9-docker-hub/mizu/debug:latest {tapped_pod_name}`
 2. Forward the debug port using `kubectl port-forward -n default mizu-api-server 2345:2345`
 3. Run the run/debug configuration you've created earlier in Intellij.
 

agent/go.mod

@@ -3,9 +3,9 @@ module mizuserver
 go 1.16
 
 require (
+	github.com/beevik/etree v1.1.0
 	github.com/djherbis/atime v1.0.0
 	github.com/fsnotify/fsnotify v1.4.9
-	github.com/getkin/kin-openapi v0.76.0
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.7.2
 	github.com/go-playground/locales v0.13.0
@@ -13,14 +13,13 @@ require (
 	github.com/go-playground/validator/v10 v10.5.0
 	github.com/google/martian v2.1.0+incompatible
 	github.com/gorilla/websocket v1.4.2
-	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
-	github.com/up9inc/mizu/tap/api v0.0.0
 	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
-	go.mongodb.org/mongo-driver v1.7.1
+	go.mongodb.org/mongo-driver v1.5.1
 	gorm.io/driver/sqlite v1.1.4
 	gorm.io/gorm v1.21.8
 	k8s.io/api v0.21.0
@@ -31,5 +30,3 @@ require (
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 
 replace github.com/up9inc/mizu/tap v0.0.0 => ../tap
-
-replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api

agent/go.sum

@@ -42,6 +42,9 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
+github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
+github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
 github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M=
 github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -68,10 +71,6 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/getkin/kin-openapi v0.76.0 h1:j77zg3Ec+k+r+GA3d8hBoXpAc6KX9TbBPrwQGBIy2sY=
-github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
-github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-contrib/static v0.0.1 h1:JVxuvHPuUfkoul12N7dtQw7KRn/pSMq7Ue1Va9Swm1U=
@@ -79,8 +78,6 @@ github.com/gin-contrib/static v0.0.1/go.mod h1:CSxeF+wep05e0kCOsqWdAWbSszmc31zTI
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/gin-gonic/gin v1.7.2 h1:Tg03T9yM2xa8j6I3Z3oqLaQRSmKvxPd6g/2HJ6zICFA=
 github.com/gin-gonic/gin v1.7.2/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
-github.com/go-errors/errors v1.4.1 h1:IvVlgbzSsaUNudsw5dcXSzF3EWyXTi5XrAdngnuhRyg=
-github.com/go-errors/errors v1.4.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -89,13 +86,10 @@ github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
-github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
@@ -107,6 +101,7 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E=
 github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
 github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
@@ -134,8 +129,6 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0=
-github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -187,8 +180,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -203,6 +194,8 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr
 github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI=
 github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -227,7 +220,6 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
@@ -253,8 +245,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
@@ -270,6 +260,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
+github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -285,7 +277,6 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
@@ -301,8 +292,8 @@ github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmv
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.mongodb.org/mongo-driver v1.7.1 h1:jwqTeEM3x6L9xDXrCxN0Hbg7vdGfPBOTIkr0+/LYZDA=
-go.mongodb.org/mongo-driver v1.7.1/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
+go.mongodb.org/mongo-driver v1.5.1 h1:9nOVLGDfOaZ9R0tBumx/BcuqkbFpyTCU2r/Po7A2azI=
+go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -371,8 +362,9 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210421230115-4e50805a0758 h1:aEpZnXcAmXkd6AvLb2OPt+EN1Zu/8Ne3pCqPjja5PXY=
+golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -418,8 +410,9 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M=
 golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ=
+golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
@@ -430,8 +423,9 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -544,7 +538,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

agent/main.go

@@ -4,29 +4,21 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"github.com/gin-contrib/static"
+	"github.com/gin-gonic/gin"
+	"github.com/gorilla/websocket"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap"
 	"mizuserver/pkg/api"
-	"mizuserver/pkg/controllers"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/routes"
-	"mizuserver/pkg/up9"
+	"mizuserver/pkg/sensitiveDataFiltering"
 	"mizuserver/pkg/utils"
 	"net/http"
 	"os"
 	"os/signal"
-	"path"
-	"path/filepath"
-	"plugin"
-	"sort"
-
-	"github.com/gin-contrib/static"
-	"github.com/gin-gonic/gin"
-	"github.com/gorilla/websocket"
-	"github.com/op/go-logging"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/tap"
-	tapApi "github.com/up9inc/mizu/tap/api"
+	"strings"
 )
 
 var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API")
@@ -37,36 +29,28 @@ var namespace = flag.String("namespace", "", "Resolve IPs if they belong to reso
 var harsReaderMode = flag.Bool("hars-read", false, "Run in hars-read mode")
 var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
 
-var extensions []*tapApi.Extension             // global
-var extensionsMap map[string]*tapApi.Extension // global
-
 func main() {
-	logLevel := determineLogLevel()
-	logger.InitLoggerStderrOnly(logLevel)
 	flag.Parse()
-	loadExtensions()
+	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+	tapOpts := &tap.TapOpts{HostMode: hostMode}
 
-	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode {
+
+	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode{
 		panic("One of the flags --tap, --api or --standalone or --hars-read must be provided")
 	}
 
 	if *standaloneMode {
 		api.StartResolving(*namespace)
 
-		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
-		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+		harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts)
+		filteredHarChannel := make(chan *tap.OutputChannelItem)
 
-		filteringOptions := getTrafficFilteringOptions()
-		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-		tapOpts := &tap.TapOpts{HostMode: hostMode}
-		tap.StartPassiveTapper(tapOpts, outputItemsChannel, extensions, filteringOptions)
-
-		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
-		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
+		go filterHarItems(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions())
+		go api.StartReadingEntries(filteredHarChannel, nil)
+		go api.StartReadingOutbound(outboundLinkOutputChannel)
 
 		hostApi(nil)
 	} else if *tapperMode {
-		logger.Log.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
 		if *apiServerAddress == "" {
 			panic("API server address must be provided with --api-server-address when using --tap")
 		}
@@ -74,45 +58,34 @@ func main() {
 		tapTargets := getTapTargets()
 		if tapTargets != nil {
 			tap.SetFilterAuthorities(tapTargets)
-			logger.Log.Infof("Filtering for the following authorities: %v", tap.GetFilterIPs())
+			rlog.Infof("Filtering for the following authorities: %v", tap.GetFilterIPs())
 		}
 
-		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+		harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts)
 
-		filteringOptions := getTrafficFilteringOptions()
-		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-		tapOpts := &tap.TapOpts{HostMode: hostMode}
-		tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions)
-		socketConnection, _, err := websocket.DefaultDialer.Dial(*apiServerAddress, nil)
+		socketConnection, err := shared.ConnectToSocketServer(*apiServerAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false)
 		if err != nil {
 			panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
 		}
-		logger.Log.Infof("Connected successfully to websocket %s", *apiServerAddress)
 
-		go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
+		go pipeTapChannelToSocket(socketConnection, harOutputChannel)
+		go pipeOutboundLinksChannelToSocket(socketConnection, outboundLinkOutputChannel)
 	} else if *apiServerMode {
 		api.StartResolving(*namespace)
 
-		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
-		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+		socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000)
+		filteredHarChannel := make(chan *tap.OutputChannelItem)
 
-		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
-		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
+		go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions())
+		go api.StartReadingEntries(filteredHarChannel, nil)
 
-		syncEntriesConfig := getSyncEntriesConfig()
-		if syncEntriesConfig != nil {
-			if err := up9.SyncEntries(syncEntriesConfig); err != nil {
-				panic(fmt.Sprintf("Error syncing entries, err: %v", err))
-			}
-		}
-
-		hostApi(outputItemsChannel)
+		hostApi(socketHarOutChannel)
 	} else if *harsReaderMode {
-		outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
-		filteredHarChannel := make(chan *tapApi.OutputChannelItem)
+		socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000)
+		filteredHarChannel := make(chan *tap.OutputChannelItem)
 
-		go filterItems(outputItemsChannel, filteredHarChannel)
-		go api.StartReadingEntries(filteredHarChannel, harsDir, extensionsMap)
+		go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions())
+		go api.StartReadingEntries(filteredHarChannel, harsDir)
 		hostApi(nil)
 	}
 
@@ -120,53 +93,10 @@ func main() {
 	signal.Notify(signalChan, os.Interrupt)
 	<-signalChan
 
-	logger.Log.Info("Exiting")
+	rlog.Info("Exiting")
 }
 
-func loadExtensions() {
-	dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
-	extensionsDir := path.Join(dir, "./extensions/")
-
-	files, err := ioutil.ReadDir(extensionsDir)
-	if err != nil {
-		logger.Log.Fatal(err)
-	}
-	extensions = make([]*tapApi.Extension, len(files))
-	extensionsMap = make(map[string]*tapApi.Extension)
-	for i, file := range files {
-		filename := file.Name()
-		logger.Log.Infof("Loading extension: %s\n", filename)
-		extension := &tapApi.Extension{
-			Path: path.Join(extensionsDir, filename),
-		}
-		plug, _ := plugin.Open(extension.Path)
-		extension.Plug = plug
-		symDissector, err := plug.Lookup("Dissector")
-
-		var dissector tapApi.Dissector
-		var ok bool
-		dissector, ok = symDissector.(tapApi.Dissector)
-		if err != nil || !ok {
-			panic(fmt.Sprintf("Failed to load the extension: %s\n", extension.Path))
-		}
-		dissector.Register(extension)
-		extension.Dissector = dissector
-		extensions[i] = extension
-		extensionsMap[extension.Protocol.Name] = extension
-	}
-
-	sort.Slice(extensions, func(i, j int) bool {
-		return extensions[i].Protocol.Priority < extensions[j].Protocol.Priority
-	})
-
-	for _, extension := range extensions {
-		logger.Log.Infof("Extension Properties: %+v\n", extension)
-	}
-
-	controllers.InitExtensionsMap(extensionsMap)
-}
-
-func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
+func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
 	app := gin.Default()
 
 	app.GET("/echo", func(c *gin.Context) {
@@ -174,10 +104,9 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 	})
 
 	eventHandlers := api.RoutesEventHandlers{
-		SocketOutChannel: socketHarOutputChannel,
+		SocketHarOutChannel: socketHarOutputChannel,
 	}
 
-	app.Use(DisableRootStaticCache())
 	app.Use(static.ServeRoot("/", "./site"))
 	app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
 
@@ -190,17 +119,6 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 	utils.StartServer(app)
 }
 
-func DisableRootStaticCache() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		if c.Request.RequestURI == "/" {
-			// Disable cache only for the main static route
-			c.Writer.Header().Set("Cache-Control", "no-store")
-		}
-
-		c.Next()
-	}
-}
-
 func CORSMiddleware() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
@@ -217,55 +135,63 @@ func CORSMiddleware() gin.HandlerFunc {
 	}
 }
 
-func parseEnvVar(env string) map[string][]string {
-	var mapOfList map[string][]string
-
-	val, present := os.LookupEnv(env)
-
-	if !present {
-		return mapOfList
-	}
-
-	err := json.Unmarshal([]byte(val), &mapOfList)
-	if err != nil {
-		panic(fmt.Sprintf("env var %s's value of %s is invalid! must be map[string][]string %v", env, mapOfList, err))
-	}
-	return mapOfList
-}
-
 func getTapTargets() []string {
 	nodeName := os.Getenv(shared.NodeNameEnvVar)
-	tappedAddressesPerNodeDict := parseEnvVar(shared.TappedAddressesPerNodeDictEnvVar)
+	var tappedAddressesPerNodeDict map[string][]string
+	err := json.Unmarshal([]byte(os.Getenv(shared.TappedAddressesPerNodeDictEnvVar)), &tappedAddressesPerNodeDict)
+	if err != nil {
+		panic(fmt.Sprintf("env var %s's value of %s is invalid! must be map[string][]string %v", shared.TappedAddressesPerNodeDictEnvVar, tappedAddressesPerNodeDict, err))
+	}
 	return tappedAddressesPerNodeDict[nodeName]
 }
 
-func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
+func getTrafficFilteringOptions() *shared.TrafficFilteringOptions {
 	filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar)
 	if filteringOptionsJson == "" {
-		return &tapApi.TrafficFilteringOptions{
-			IgnoredUserAgents: []string{},
-		}
+		return nil
 	}
-	var filteringOptions tapApi.TrafficFilteringOptions
+	var filteringOptions shared.TrafficFilteringOptions
 	err := json.Unmarshal([]byte(filteringOptionsJson), &filteringOptions)
 	if err != nil {
-		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the api.TrafficFilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err))
+		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.TrafficFilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err))
 	}
 
 	return &filteringOptions
 }
 
-func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
+func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) {
 	for message := range inChannel {
 		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
 			continue
 		}
+		// TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441
+		if isHealthCheckByUserAgent(message, filterOptions.HealthChecksUserAgentHeaders) {
+			continue
+		}
+
+		if !filterOptions.DisableRedaction {
+			sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions)
+		}
 
 		outChannel <- message
 	}
 }
 
-func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tapApi.OutputChannelItem) {
+func isHealthCheckByUserAgent(message *tap.OutputChannelItem, userAgentsToIgnore []string) bool {
+	for _, header := range message.HarEntry.Request.Headers {
+		if strings.ToLower(header.Name) == "user-agent" {
+			for _, userAgent := range userAgentsToIgnore {
+				if strings.Contains(strings.ToLower(header.Value), strings.ToLower(userAgent)) {
+					return true
+				}
+			}
+			return false
+		}
+	}
+	return false
+}
+
+func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) {
 	if connection == nil {
 		panic("Websocket connection is nil")
 	}
@@ -277,39 +203,32 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
 	for messageData := range messageDataChannel {
 		marshaledData, err := models.CreateWebsocketTappedEntryMessage(messageData)
 		if err != nil {
-			logger.Log.Errorf("error converting message to json %v, err: %s, (%v,%+v)", messageData, err, err, err)
+			rlog.Infof("error converting message to json %s, (%v,%+v)\n", err, err, err)
 			continue
 		}
 
-		// NOTE: This is where the `*tapApi.OutputChannelItem` leaves the code
-		// and goes into the intermediate WebSocket.
 		err = connection.WriteMessage(websocket.TextMessage, marshaledData)
 		if err != nil {
-			logger.Log.Errorf("error sending message through socket server %v, err: %s, (%v,%+v)", messageData, err, err, err)
+			rlog.Infof("error sending message through socket server %s, (%v,%+v)\n", err, err, err)
 			continue
 		}
 	}
 }
 
-func getSyncEntriesConfig() *shared.SyncEntriesConfig {
-	syncEntriesConfigJson := os.Getenv(shared.SyncEntriesConfigEnvVar)
-	if syncEntriesConfigJson == "" {
-		return nil
-	}
+func pipeOutboundLinksChannelToSocket(connection *websocket.Conn, outboundLinkChannel <-chan *tap.OutboundLink) {
+	for outboundLink := range outboundLinkChannel {
+		if outboundLink.SuggestedProtocol == tap.TLSProtocol {
+			marshaledData, err := models.CreateWebsocketOutboundLinkMessage(outboundLink)
+			if err != nil {
+				rlog.Infof("Error converting outbound link to json %s, (%v,%+v)", err, err, err)
+				continue
+			}
 
-	var syncEntriesConfig = &shared.SyncEntriesConfig{}
-	err := json.Unmarshal([]byte(syncEntriesConfigJson), syncEntriesConfig)
-	if err != nil {
-		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.SyncEntriesConfig struct, err: %v", shared.SyncEntriesConfigEnvVar, syncEntriesConfigJson, err))
+			err = connection.WriteMessage(websocket.TextMessage, marshaledData)
+			if err != nil {
+				rlog.Infof("error sending outbound link message through socket server %s, (%v,%+v)", err, err, err)
+				continue
+			}
+		}
 	}
-
-	return syncEntriesConfig
-}
-
-func determineLogLevel() (logLevel logging.Level) {
-	logLevel = logging.INFO
-	if os.Getenv(shared.DebugModeEnvVar) == "1" {
-		logLevel = logging.DEBUG
-	}
-	return
 }

agent/pkg/api/contract_validation.go

@@ -1,110 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-
-	"github.com/getkin/kin-openapi/openapi3"
-	"github.com/getkin/kin-openapi/openapi3filter"
-	"github.com/getkin/kin-openapi/routers"
-	legacyrouter "github.com/getkin/kin-openapi/routers/legacy"
-
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/tap/api"
-)
-
-const (
-	ContractNotApplicable api.ContractStatus = 0
-	ContractPassed        api.ContractStatus = 1
-	ContractFailed        api.ContractStatus = 2
-)
-
-func loadOAS(ctx context.Context) (doc *openapi3.T, contractContent string, router routers.Router, err error) {
-	path := fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.ContractFileName)
-	bytes, err := ioutil.ReadFile(path)
-	if err != nil {
-		logger.Log.Error(err.Error())
-		return
-	}
-	contractContent = string(bytes)
-	loader := &openapi3.Loader{Context: ctx}
-	doc, _ = loader.LoadFromData(bytes)
-	err = doc.Validate(ctx)
-	if err != nil {
-		logger.Log.Error(err.Error())
-		return
-	}
-	router, _ = legacyrouter.NewRouter(doc)
-	return
-}
-
-func validateOAS(ctx context.Context, doc *openapi3.T, router routers.Router, req *http.Request, res *http.Response) (isValid bool, reqErr error, resErr error) {
-	isValid = true
-	reqErr = nil
-	resErr = nil
-
-	// Find route
-	route, pathParams, err := router.FindRoute(req)
-	if err != nil {
-		return
-	}
-
-	// Validate request
-	requestValidationInput := &openapi3filter.RequestValidationInput{
-		Request:    req,
-		PathParams: pathParams,
-		Route:      route,
-	}
-	if reqErr = openapi3filter.ValidateRequest(ctx, requestValidationInput); reqErr != nil {
-		isValid = false
-	}
-
-	responseValidationInput := &openapi3filter.ResponseValidationInput{
-		RequestValidationInput: requestValidationInput,
-		Status:                 res.StatusCode,
-		Header:                 res.Header,
-	}
-
-	if res.Body != nil {
-		body, _ := ioutil.ReadAll(res.Body)
-		res.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-		responseValidationInput.SetBodyBytes(body)
-	}
-
-	// Validate response.
-	if resErr = openapi3filter.ValidateResponse(ctx, responseValidationInput); resErr != nil {
-		isValid = false
-	}
-
-	return
-}
-
-func handleOAS(ctx context.Context, doc *openapi3.T, router routers.Router, req *http.Request, res *http.Response, contractContent string) (contract api.Contract) {
-	contract = api.Contract{
-		Content: contractContent,
-		Status:  ContractNotApplicable,
-	}
-
-	isValid, reqErr, resErr := validateOAS(ctx, doc, router, req, res)
-	if isValid {
-		contract.Status = ContractPassed
-	} else {
-		contract.Status = ContractFailed
-		if reqErr != nil {
-			contract.RequestReason = reqErr.Error()
-		} else {
-			contract.RequestReason = ""
-		}
-		if resErr != nil {
-			contract.ResponseReason = resErr.Error()
-		} else {
-			contract.ResponseReason = ""
-		}
-	}
-
-	return
-}

agent/pkg/api/main.go

@@ -5,21 +5,21 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
+	"net/url"
 	"os"
 	"path"
 	"sort"
 	"strings"
 	"time"
 
+	"github.com/google/martian/har"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/tap"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 
-	"github.com/google/martian/har"
-	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
-
+	"mizuserver/pkg/database"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/resolver"
 	"mizuserver/pkg/utils"
@@ -31,7 +31,7 @@ func StartResolving(namespace string) {
 	errOut := make(chan error, 100)
 	res, err := resolver.NewFromInCluster(errOut, namespace)
 	if err != nil {
-		logger.Log.Infof("error creating k8s resolver %s", err)
+		rlog.Infof("error creating k8s resolver %s", err)
 		return
 	}
 	ctx := context.Background()
@@ -40,7 +40,7 @@ func StartResolving(namespace string) {
 		for {
 			select {
 			case err := <-errOut:
-				logger.Log.Infof("name resolving error %s", err)
+				rlog.Infof("name resolving error %s", err)
 			}
 		}
 	}()
@@ -49,19 +49,17 @@ func StartResolving(namespace string) {
 	holder.SetResolver(res)
 }
 
-func StartReadingEntries(harChannel <-chan *tapApi.OutputChannelItem, workingDir *string, extensionsMap map[string]*tapApi.Extension) {
+func StartReadingEntries(harChannel <-chan *tap.OutputChannelItem, workingDir *string) {
 	if workingDir != nil && *workingDir != "" {
 		startReadingFiles(*workingDir)
 	} else {
-		startReadingChannel(harChannel, extensionsMap)
+		startReadingChannel(harChannel)
 	}
 }
 
 func startReadingFiles(workingDir string) {
-	if err := os.MkdirAll(workingDir, os.ModePerm); err != nil {
-		logger.Log.Errorf("Failed to make dir: %s, err: %v", workingDir, err)
-		return
-	}
+	err := os.MkdirAll(workingDir, os.ModePerm)
+	utils.CheckErr(err)
 
 	for true {
 		dir, _ := os.Open(workingDir)
@@ -76,7 +74,7 @@ func startReadingFiles(workingDir string) {
 		sort.Sort(utils.ByModTime(harFiles))
 
 		if len(harFiles) == 0 {
-			logger.Log.Infof("Waiting for new files\n")
+			rlog.Infof("Waiting for new files\n")
 			time.Sleep(3 * time.Second)
 			continue
 		}
@@ -89,66 +87,53 @@ func startReadingFiles(workingDir string) {
 		decErr := json.NewDecoder(bufio.NewReader(file)).Decode(&inputHar)
 		utils.CheckErr(decErr)
 
+		for _, entry := range inputHar.Log.Entries {
+			time.Sleep(time.Millisecond * 250)
+			connectionInfo := &tap.ConnectionInfo{
+				ClientIP:   fileInfo.Name(),
+				ClientPort: "",
+				ServerIP:   "",
+				ServerPort: "",
+				IsOutgoing: false,
+			}
+			saveHarToDb(entry, connectionInfo)
+		}
 		rmErr := os.Remove(inputFilePath)
 		utils.CheckErr(rmErr)
 	}
 }
 
-func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extensionsMap map[string]*tapApi.Extension) {
+func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) {
 	if outputItems == nil {
 		panic("Channel of captured messages is nil")
 	}
 
-	disableOASValidation := false
-	ctx := context.Background()
-	doc, contractContent, router, err := loadOAS(ctx)
-	if err != nil {
-		logger.Log.Infof("Disabled OAS validation: %s\n", err.Error())
-		disableOASValidation = true
-	}
-
 	for item := range outputItems {
 		providers.EntryAdded()
-
-		extension := extensionsMap[item.Protocol.Name]
-		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
-		mizuEntry := extension.Dissector.Analyze(item, primitive.NewObjectID().Hex(), resolvedSource, resolvedDestionation)
-		baseEntry := extension.Dissector.Summarize(mizuEntry)
-		mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
-		if extension.Protocol.Name == "http" {
-			if !disableOASValidation {
-				var httpPair tapApi.HTTPRequestResponsePair
-				json.Unmarshal([]byte(mizuEntry.Entry), &httpPair)
-
-				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
-				baseEntry.ContractStatus = contract.Status
-				mizuEntry.ContractStatus = contract.Status
-				mizuEntry.ContractRequestReason = contract.RequestReason
-				mizuEntry.ContractResponseReason = contract.ResponseReason
-				mizuEntry.ContractContent = contract.Content
-			}
-
-			var pair tapApi.RequestResponsePair
-			json.Unmarshal([]byte(mizuEntry.Entry), &pair)
-			harEntry, err := utils.NewEntry(&pair)
-			if err == nil {
-				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Service)
-				baseEntry.Rules = rules
-			}
-		}
-		database.CreateEntry(mizuEntry)
-
-		baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(baseEntry)
-		BroadcastToBrowserClients(baseEntryBytes)
+		saveHarToDb(item.HarEntry, item.ConnectionInfo)
 	}
 }
 
-func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, resolvedDestination string) {
+func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) {
+	// tcpStreamFactory will block on write to channel. Empty channel to unblock.
+	// TODO: Make write to channel optional.
+	for range outboundLinkChannel {
+	}
+}
+
+func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
+	entryBytes, _ := json.Marshal(entry)
+	serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL)
+	entryId := primitive.NewObjectID().Hex()
+	var (
+		resolvedSource      string
+		resolvedDestination string
+	)
 	if k8sResolver != nil {
 		unresolvedSource := connectionInfo.ClientIP
 		resolvedSource = k8sResolver.Resolve(unresolvedSource)
 		if resolvedSource == "" {
-			logger.Log.Debugf("Cannot find resolved name to source: %s\n", unresolvedSource)
+			rlog.Debugf("Cannot find resolved name to source: %s\n", unresolvedSource)
 			if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
 				return
 			}
@@ -156,24 +141,52 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re
 		unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
 		resolvedDestination = k8sResolver.Resolve(unresolvedDestination)
 		if resolvedDestination == "" {
-			logger.Log.Debugf("Cannot find resolved name to dest: %s\n", unresolvedDestination)
+			rlog.Debugf("Cannot find resolved name to dest: %s\n", unresolvedDestination)
 			if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
 				return
 			}
 		}
 	}
-	return resolvedSource, resolvedDestination
+
+	mizuEntry := models.MizuEntry{
+		EntryId:             entryId,
+		Entry:               string(entryBytes), // simple way to store it and not convert to bytes
+		Service:             serviceName,
+		Url:                 entry.Request.URL,
+		Path:                urlPath,
+		Method:              entry.Request.Method,
+		Status:              entry.Response.Status,
+		RequestSenderIp:     connectionInfo.ClientIP,
+		Timestamp:           entry.StartedDateTime.UnixNano() / int64(time.Millisecond),
+		ResolvedSource:      resolvedSource,
+		ResolvedDestination: resolvedDestination,
+		IsOutgoing:          connectionInfo.IsOutgoing,
+	}
+	mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
+	database.CreateEntry(&mizuEntry)
+
+	baseEntry := models.BaseEntryDetails{}
+	if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil {
+		return
+	}
+	baseEntry.Rules = models.RunValidationRulesState(*entry, serviceName)
+	baseEntry.Latency = entry.Timings.Receive
+	baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry)
+	BroadcastToBrowserClients(baseEntryBytes)
+}
+
+func getServiceNameFromUrl(inputUrl string) (string, string) {
+	parsed, err := url.Parse(inputUrl)
+	utils.CheckErr(err)
+	return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path
 }
 
 func CheckIsServiceIP(address string) bool {
-	if k8sResolver == nil {
-		return false
-	}
 	return k8sResolver.CheckIsServiceIP(address)
 }
 
 // gives a rough estimate of the size this will take up in the db, good enough for maintaining db size limit accurately
-func getEstimatedEntrySizeBytes(mizuEntry *tapApi.MizuEntry) int {
+func getEstimatedEntrySizeBytes(mizuEntry models.MizuEntry) int {
 	sizeBytes := len(mizuEntry.Entry)
 	sizeBytes += len(mizuEntry.EntryId)
 	sizeBytes += len(mizuEntry.Service)

agent/pkg/api/socket_routes.go

@@ -2,14 +2,13 @@ package api
 
 import (
 	"errors"
+	"github.com/gin-gonic/gin"
+	"github.com/gorilla/websocket"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared/debounce"
 	"net/http"
 	"sync"
 	"time"
-
-	"github.com/gin-gonic/gin"
-	"github.com/gorilla/websocket"
-	"github.com/up9inc/mizu/shared/debounce"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 type EventHandlers interface {
@@ -51,7 +50,7 @@ func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
 func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) {
 	conn, err := websocketUpgrader.Upgrade(w, r, nil)
 	if err != nil {
-		logger.Log.Errorf("Failed to set websocket upgrade: %v", err)
+		rlog.Errorf("Failed to set websocket upgrade: %v", err)
 		return
 	}
 
@@ -72,7 +71,7 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 	for {
 		_, msg, err := conn.ReadMessage()
 		if err != nil {
-			logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
+			rlog.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
 			break
 		}
 		eventHandlers.WebSocketMessage(socketId, msg)
@@ -82,7 +81,7 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 func socketCleanup(socketId int, socketConnection *SocketConnection) {
 	err := socketConnection.connection.Close()
 	if err != nil {
-		logger.Log.Errorf("Error closing socket connection for socket id %d: %v\n", socketId, err)
+		rlog.Errorf("Error closing socket connection for socket id %d: %v\n", socketId, err)
 	}
 
 	websocketIdsLock.Lock()
@@ -93,7 +92,7 @@ func socketCleanup(socketId int, socketConnection *SocketConnection) {
 }
 
 var db = debounce.NewDebouncer(time.Second*5, func() {
-	logger.Log.Error("Successfully sent to socket")
+	rlog.Error("Successfully sent to socket")
 })
 
 func SendToSocket(socketId int, message []byte) error {
@@ -105,7 +104,7 @@ func SendToSocket(socketId int, message []byte) error {
 	var sent = false
 	time.AfterFunc(time.Second*5, func() {
 		if !sent {
-			logger.Log.Error("Socket timed out")
+			rlog.Error("Socket timed out")
 			socketCleanup(socketId, socketObj)
 		}
 	})

agent/pkg/api/socket_server_handlers.go

@@ -8,10 +8,9 @@ import (
 	"mizuserver/pkg/up9"
 	"sync"
 
-	tapApi "github.com/up9inc/mizu/tap/api"
-
+	"github.com/romana/rlog"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/tap"
 )
 
 var browserClientSocketUUIDs = make([]int, 0)
@@ -19,7 +18,7 @@ var socketListLock = sync.Mutex{}
 
 type RoutesEventHandlers struct {
 	EventHandlers
-	SocketOutChannel chan<- *tapApi.OutputChannelItem
+	SocketHarOutChannel chan<- *tap.OutputChannelItem
 }
 
 func init() {
@@ -28,10 +27,9 @@ func init() {
 
 func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 	if isTapper {
-		logger.Log.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
-		providers.TapperAdded()
+		rlog.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
 	} else {
-		logger.Log.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
+		rlog.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
 		socketListLock.Lock()
 		browserClientSocketUUIDs = append(browserClientSocketUUIDs, socketId)
 		socketListLock.Unlock()
@@ -40,10 +38,9 @@ func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 
 func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 	if isTapper {
-		logger.Log.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
-		providers.TapperRemoved()
+		rlog.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
 	} else {
-		logger.Log.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
+		rlog.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
 		socketListLock.Lock()
 		removeSocketUUIDFromBrowserSlice(socketId)
 		socketListLock.Unlock()
@@ -55,7 +52,7 @@ func BroadcastToBrowserClients(message []byte) {
 		go func(socketId int) {
 			err := SendToSocket(socketId, message)
 			if err != nil {
-				logger.Log.Errorf("error sending message to socket ID %d: %v", socketId, err)
+				rlog.Errorf("error sending message to socket ID %d: %v", socketId, err)
 			}
 		}(socketId)
 	}
@@ -65,23 +62,22 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 	var socketMessageBase shared.WebSocketMessageMetadata
 	err := json.Unmarshal(message, &socketMessageBase)
 	if err != nil {
-		logger.Log.Infof("Could not unmarshal websocket message %v\n", err)
+		rlog.Infof("Could not unmarshal websocket message %v\n", err)
 	} else {
 		switch socketMessageBase.MessageType {
 		case shared.WebSocketMessageTypeTappedEntry:
 			var tappedEntryMessage models.WebSocketTappedEntryMessage
 			err := json.Unmarshal(message, &tappedEntryMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
 			} else {
-				// NOTE: This is where the message comes back from the intermediate WebSocket to code.
-				h.SocketOutChannel <- tappedEntryMessage.Data
+				h.SocketHarOutChannel <- tappedEntryMessage.Data
 			}
 		case shared.WebSocketMessageTypeUpdateStatus:
 			var statusMessage shared.WebSocketStatusMessage
 			err := json.Unmarshal(message, &statusMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
 			} else {
 				providers.TapStatus.Pods = statusMessage.TappingStatus.Pods
 				BroadcastToBrowserClients(message)
@@ -90,12 +86,12 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 			var outboundLinkMessage models.WebsocketOutboundLinkMessage
 			err := json.Unmarshal(message, &outboundLinkMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
 			} else {
 				handleTLSLink(outboundLinkMessage)
 			}
 		default:
-			logger.Log.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
+			rlog.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
 		}
 	}
 }
@@ -116,9 +112,9 @@ func handleTLSLink(outboundLinkMessage models.WebsocketOutboundLinkMessage) {
 	}
 	marshaledMessage, err := json.Marshal(outboundLinkMessage)
 	if err != nil {
-		logger.Log.Errorf("Error marshaling outbound link message for broadcasting: %v", err)
+		rlog.Errorf("Error marshaling outbound link message for broadcasting: %v", err)
 	} else {
-		logger.Log.Errorf("Broadcasting outboundlink message %s", string(marshaledMessage))
+		rlog.Errorf("Broadcasting outboundlink message %s", string(marshaledMessage))
 		BroadcastToBrowserClients(marshaledMessage)
 	}
 }

agent/pkg/controllers/entries_controller.go

@@ -3,21 +3,21 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/gin-gonic/gin"
-	tapApi "github.com/up9inc/mizu/tap/api"
 	"mizuserver/pkg/database"
 	"mizuserver/pkg/models"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/up9"
 	"mizuserver/pkg/utils"
 	"mizuserver/pkg/validation"
 	"net/http"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/google/martian/har"
+	"github.com/romana/rlog"
 )
 
-var extensionsMap map[string]*tapApi.Extension // global
-
-func InitExtensionsMap(ref map[string]*tapApi.Extension) {
-	extensionsMap = ref
-}
-
 func GetEntries(c *gin.Context) {
 	entriesFilter := &models.EntriesFilter{}
 
@@ -31,66 +31,227 @@ func GetEntries(c *gin.Context) {
 
 	order := database.OperatorToOrderMapping[entriesFilter.Operator]
 	operatorSymbol := database.OperatorToSymbolMapping[entriesFilter.Operator]
-	var entries []tapApi.MizuEntry
+	var entries []models.MizuEntry
 	database.GetEntriesTable().
 		Order(fmt.Sprintf("timestamp %s", order)).
 		Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)).
+		Omit("entry"). // remove the "big" entry field
 		Limit(entriesFilter.Limit).
 		Find(&entries)
 
 	if len(entries) > 0 && order == database.OrderDesc {
-		// the entries always order from oldest to newest - we should reverse
+		// the entries always order from oldest to newest so we should revers
 		utils.ReverseSlice(entries)
 	}
 
-	baseEntries := make([]tapApi.BaseEntryDetails, 0)
-	for _, entry := range entries {
-		baseEntryDetails := tapApi.BaseEntryDetails{}
-		if err := models.GetEntry(&entry, &baseEntryDetails); err != nil {
+	baseEntries := make([]models.BaseEntryDetails, 0)
+	for _, data := range entries {
+		harEntry := models.BaseEntryDetails{}
+		if err := models.GetEntry(&data, &harEntry); err != nil {
 			continue
 		}
-
-		var pair tapApi.RequestResponsePair
-		json.Unmarshal([]byte(entry.Entry), &pair)
-		harEntry, err := utils.NewEntry(&pair)
-		if err == nil {
-			rules, _, _ := models.RunValidationRulesState(*harEntry, entry.Service)
-			baseEntryDetails.Rules = rules
-		}
-
-		baseEntries = append(baseEntries, baseEntryDetails)
+		baseEntries = append(baseEntries, harEntry)
 	}
 
 	c.JSON(http.StatusOK, baseEntries)
 }
 
+func GetHARs(c *gin.Context) {
+	entriesFilter := &models.HarFetchRequestQuery{}
+	order := database.OrderDesc
+	if err := c.BindQuery(entriesFilter); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+	err := validation.Validate(entriesFilter)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+
+	var timestampFrom, timestampTo int64
+
+	if entriesFilter.From < 0 {
+		timestampFrom = 0
+	} else {
+		timestampFrom = entriesFilter.From
+	}
+	if entriesFilter.To <= 0 {
+		timestampTo = time.Now().UnixNano() / int64(time.Millisecond)
+	} else {
+		timestampTo = entriesFilter.To
+	}
+
+	var entries []models.MizuEntry
+	database.GetEntriesTable().
+		Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)).
+		Order(fmt.Sprintf("timestamp %s", order)).
+		Find(&entries)
+
+	if len(entries) > 0 {
+		// the entries always order from oldest to newest so we should revers
+		utils.ReverseSlice(entries)
+	}
+
+	harsObject := map[string]*models.ExtendedHAR{}
+
+	for _, entryData := range entries {
+		var harEntry har.Entry
+		_ = json.Unmarshal([]byte(entryData.Entry), &harEntry)
+		if entryData.ResolvedDestination != "" {
+			harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entryData.ResolvedDestination)
+		}
+
+		var fileName string
+		sourceOfEntry := entryData.ResolvedSource
+		if sourceOfEntry != "" {
+			// naively assumes the proper service source is http
+			sourceOfEntry = fmt.Sprintf("http://%s", sourceOfEntry)
+			//replace / from the file name cause they end up creating a corrupted folder
+			fileName = fmt.Sprintf("%s.har", strings.ReplaceAll(sourceOfEntry, "/", "_"))
+		} else {
+			fileName = "unknown_source.har"
+		}
+		if harOfSource, ok := harsObject[fileName]; ok {
+			harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry)
+		} else {
+			var entriesHar []*har.Entry
+			entriesHar = append(entriesHar, &harEntry)
+			harsObject[fileName] = &models.ExtendedHAR{
+				Log: &models.ExtendedLog{
+					Version: "1.2",
+					Creator: &models.ExtendedCreator{
+						Creator: &har.Creator{
+							Name:    "mizu",
+							Version: "0.0.2",
+						},
+					},
+					Entries: entriesHar,
+				},
+			}
+			// leave undefined when no source is present, otherwise modeler assumes source is empty string ""
+			if sourceOfEntry != "" {
+				harsObject[fileName].Log.Creator.Source = &sourceOfEntry
+			}
+		}
+	}
+
+	retObj := map[string][]byte{}
+	for k, v := range harsObject {
+		bytesData, _ := json.Marshal(v)
+		retObj[k] = bytesData
+	}
+	buffer := utils.ZipData(retObj)
+	c.Data(http.StatusOK, "application/octet-stream", buffer.Bytes())
+}
+
+func UploadEntries(c *gin.Context) {
+	rlog.Infof("Upload entries - started\n")
+
+	uploadParams := &models.UploadEntriesRequestQuery{}
+	if err := c.BindQuery(uploadParams); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+	if err := validation.Validate(uploadParams); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+	if up9.GetAnalyzeInfo().IsAnalyzing {
+		c.String(http.StatusBadRequest, "Cannot analyze, mizu is already analyzing")
+		return
+	}
+
+	rlog.Infof("Upload entries - creating token. dest %s\n", uploadParams.Dest)
+	token, err := up9.CreateAnonymousToken(uploadParams.Dest)
+	if err != nil {
+		c.String(http.StatusServiceUnavailable, "Cannot analyze, mizu is already analyzing")
+		return
+	}
+	rlog.Infof("Upload entries - uploading. token: %s model: %s\n", token.Token, token.Model)
+	go up9.UploadEntriesImpl(token.Token, token.Model, uploadParams.Dest, uploadParams.SleepIntervalSec)
+	c.String(http.StatusOK, "OK")
+}
+
+func GetFullEntries(c *gin.Context) {
+	entriesFilter := &models.HarFetchRequestQuery{}
+	if err := c.BindQuery(entriesFilter); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+	err := validation.Validate(entriesFilter)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+
+	var timestampFrom, timestampTo int64
+
+	if entriesFilter.From < 0 {
+		timestampFrom = 0
+	} else {
+		timestampFrom = entriesFilter.From
+	}
+	if entriesFilter.To <= 0 {
+		timestampTo = time.Now().UnixNano() / int64(time.Millisecond)
+	} else {
+		timestampTo = entriesFilter.To
+	}
+
+	entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
+	result := make([]models.FullEntryDetails, 0)
+	for _, data := range entriesArray {
+		harEntry := models.FullEntryDetails{}
+		if err := models.GetEntry(&data, &harEntry); err != nil {
+			continue
+		}
+		result = append(result, harEntry)
+	}
+	c.JSON(http.StatusOK, result)
+}
+
 func GetEntry(c *gin.Context) {
-	var entryData tapApi.MizuEntry
+	var entryData models.MizuEntry
 	database.GetEntriesTable().
 		Where(map[string]string{"entryId": c.Param("entryId")}).
 		First(&entryData)
 
-	extension := extensionsMap[entryData.ProtocolName]
-	protocol, representation, bodySize, _ := extension.Dissector.Represent(&entryData)
-
-	var rules []map[string]interface{}
-	var isRulesEnabled bool
-	if entryData.ProtocolName == "http" {
-		var pair tapApi.RequestResponsePair
-		json.Unmarshal([]byte(entryData.Entry), &pair)
-		harEntry, _ := utils.NewEntry(&pair)
-		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entryData.Service)
-		isRulesEnabled = _isRulesEnabled
-		inrec, _ := json.Marshal(rulesMatched)
-		json.Unmarshal(inrec, &rules)
+	fullEntry := models.FullEntryDetails{}
+	if err := models.GetEntry(&entryData, &fullEntry); err != nil {
+		c.JSON(http.StatusInternalServerError, map[string]interface{}{
+			"error": true,
+			"msg":   "Can't get entry details",
+		})
 	}
-
-	c.JSON(http.StatusOK, tapApi.MizuEntryWrapper{
-		Protocol:       protocol,
-		Representation: string(representation),
-		BodySize:       bodySize,
-		Data:           entryData,
-		Rules:          rules,
-		IsRulesEnabled: isRulesEnabled,
-	})
+	fullEntryWithPolicy := models.FullEntryWithPolicy{}
+	if err := models.GetEntry(&entryData, &fullEntryWithPolicy); err != nil {
+		c.JSON(http.StatusInternalServerError, map[string]interface{}{
+			"error": true,
+			"msg":   "Can't get entry details",
+		})
+	}
+	c.JSON(http.StatusOK, fullEntryWithPolicy)
+}
+
+func DeleteAllEntries(c *gin.Context) {
+	database.GetEntriesTable().
+		Where("1 = 1").
+		Delete(&models.MizuEntry{})
+
+	c.JSON(http.StatusOK, map[string]string{
+		"msg": "Success",
+	})
+
+}
+
+func GetGeneralStats(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.GetGeneralStats())
+}
+
+func GetTappingStatus(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.TapStatus)
+}
+
+func AnalyzeInformation(c *gin.Context) {
+	c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
+}
+
+func GetRecentTLSLinks(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
 }

agent/pkg/controllers/resolving_controller.go

@@ -0,0 +1,12 @@
+package controllers
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/holder"
+	"net/http"
+)
+
+func GetCurrentResolvingInformation(c *gin.Context) {
+	c.JSON(http.StatusOK, holder.GetResolver().GetMap())
+}
+

agent/pkg/controllers/status_controller.go

@@ -2,16 +2,13 @@ package controllers
 
 import (
 	"encoding/json"
+	"github.com/gin-gonic/gin"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared"
 	"mizuserver/pkg/api"
-	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
-	"mizuserver/pkg/up9"
 	"mizuserver/pkg/validation"
 	"net/http"
-
-	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 func PostTappedPods(c *gin.Context) {
@@ -24,46 +21,12 @@ func PostTappedPods(c *gin.Context) {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
-	logger.Log.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods))
+	rlog.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods))
 	providers.TapStatus.Pods = tapStatus.Pods
 	message := shared.CreateWebSocketStatusMessage(*tapStatus)
 	if jsonBytes, err := json.Marshal(message); err != nil {
-		logger.Log.Errorf("Could not Marshal message %v\n", err)
+		rlog.Errorf("Could not Marshal message %v\n", err)
 	} else {
 		api.BroadcastToBrowserClients(jsonBytes)
 	}
 }
-
-func GetTappersCount(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.TappersCount)
-}
-
-func GetAuthStatus(c *gin.Context) {
-	authStatus, err := providers.GetAuthStatus()
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, err)
-		return
-	}
-
-	c.JSON(http.StatusOK, authStatus)
-}
-
-func GetTappingStatus(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.TapStatus)
-}
-
-func AnalyzeInformation(c *gin.Context) {
-	c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
-}
-
-func GetGeneralStats(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.GetGeneralStats())
-}
-
-func GetRecentTLSLinks(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
-}
-
-func GetCurrentResolvingInformation(c *gin.Context) {
-	c.JSON(http.StatusOK, holder.GetResolver().GetMap())
-}

agent/pkg/database/main.go

@@ -2,18 +2,16 @@ package database
 
 import (
 	"fmt"
-	"mizuserver/pkg/utils"
-	"time"
-
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
-
-	tapApi "github.com/up9inc/mizu/tap/api"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/utils"
+	"time"
 )
 
 const (
-	DBPath    = "./entries.db"
+	DBPath = "./entries.db"
 	OrderDesc = "desc"
 	OrderAsc  = "asc"
 	LT        = "lt"
@@ -21,8 +19,8 @@ const (
 )
 
 var (
-	DB                      *gorm.DB
-	IsDBLocked              = false
+	DB *gorm.DB
+	IsDBLocked = false
 	OperatorToSymbolMapping = map[string]string{
 		LT: "<",
 		GT: ">",
@@ -42,7 +40,7 @@ func GetEntriesTable() *gorm.DB {
 	return DB.Table("mizu_entries")
 }
 
-func CreateEntry(entry *tapApi.MizuEntry) {
+func CreateEntry(entry *models.MizuEntry) {
 	if IsDBLocked {
 		return
 	}
@@ -53,20 +51,15 @@ func initDataBase(databasePath string) *gorm.DB {
 	temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{
 		Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond},
 	})
-	_ = temp.AutoMigrate(&tapApi.MizuEntry{}) // this will ensure table is created
+	_ = temp.AutoMigrate(&models.MizuEntry{}) // this will ensure table is created
 	return temp
 }
 
-func GetEntriesFromDb(timestampFrom int64, timestampTo int64, protocolName *string) []tapApi.MizuEntry {
-	order := OrderDesc
-	protocolNameCondition := "1 = 1"
-	if protocolName != nil {
-		protocolNameCondition = fmt.Sprintf("protocolName = '%s'", *protocolName)
-	}
 
-	var entries []tapApi.MizuEntry
+func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []models.MizuEntry {
+	order := OrderDesc
+	var entries []models.MizuEntry
 	GetEntriesTable().
-		Where(protocolNameCondition).
 		Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)).
 		Order(fmt.Sprintf("timestamp %s", order)).
 		Find(&entries)
@@ -77,3 +70,4 @@ func GetEntriesFromDb(timestampFrom int64, timestampTo int64, protocolName *stri
 	}
 	return entries
 }
+

agent/pkg/database/size_enforcer.go

@@ -1,16 +1,16 @@
 package database
 
 import (
+	"github.com/fsnotify/fsnotify"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/debounce"
+	"github.com/up9inc/mizu/shared/units"
+	"log"
+	"mizuserver/pkg/models"
 	"os"
 	"strconv"
 	"time"
-
-	"github.com/fsnotify/fsnotify"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/debounce"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/shared/units"
-	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 const percentageOfMaxSizeBytesToPrune = 15
@@ -19,13 +19,13 @@ const defaultMaxDatabaseSizeBytes int64 = 200 * 1000 * 1000
 func StartEnforcingDatabaseSize() {
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
-		logger.Log.Fatalf("Error creating filesystem watcher for db size enforcement: %v\n", err)
+		log.Fatalf("Error creating filesystem watcher for db size enforcement: %v\n", err)
 		return
 	}
 
 	maxEntriesDBByteSize, err := getMaxEntriesDBByteSize()
 	if err != nil {
-		logger.Log.Fatalf("Error parsing max db size: %v\n", err)
+		log.Fatalf("Error parsing max db size: %v\n", err)
 		return
 	}
 
@@ -47,14 +47,14 @@ func StartEnforcingDatabaseSize() {
 				if !ok {
 					return // closed channel
 				}
-				logger.Log.Errorf("filesystem watcher encountered error:%v", err)
+				rlog.Errorf("filesystem watcher encountered error:%v", err)
 			}
 		}
 	}()
 
 	err = watcher.Add(DBPath)
 	if err != nil {
-		logger.Log.Fatalf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err)
+		log.Fatalf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err)
 	}
 }
 
@@ -72,7 +72,7 @@ func getMaxEntriesDBByteSize() (int64, error) {
 func checkFileSize(maxSizeBytes int64) {
 	fileStat, err := os.Stat(DBPath)
 	if err != nil {
-		logger.Log.Errorf("Error checking %s file size: %v", DBPath, err)
+		rlog.Errorf("Error checking %s file size: %v", DBPath, err)
 	} else {
 		if fileStat.Size() > maxSizeBytes {
 			pruneOldEntries(fileStat.Size())
@@ -89,7 +89,7 @@ func pruneOldEntries(currentFileSize int64) {
 
 	rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()
 	if err != nil {
-		logger.Log.Errorf("Error getting 10000 first db rows: %v", err)
+		rlog.Errorf("Error getting 10000 first db rows: %v", err)
 		return
 	}
 
@@ -99,10 +99,10 @@ func pruneOldEntries(currentFileSize int64) {
 		if bytesToBeRemoved >= amountOfBytesToTrim {
 			break
 		}
-		var entry tapApi.MizuEntry
+		var entry models.MizuEntry
 		err = DB.ScanRows(rows, &entry)
 		if err != nil {
-			logger.Log.Errorf("Error scanning db row: %v", err)
+			rlog.Errorf("Error scanning db row: %v", err)
 			continue
 		}
 
@@ -111,11 +111,11 @@ func pruneOldEntries(currentFileSize int64) {
 	}
 
 	if len(entryIdsToRemove) > 0 {
-		GetEntriesTable().Where(entryIdsToRemove).Delete(tapApi.MizuEntry{})
+		GetEntriesTable().Where(entryIdsToRemove).Delete(models.MizuEntry{})
 		// VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this
 		DB.Exec("VACUUM")
-		logger.Log.Errorf("Removed %d rows and cleared %s", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
+		rlog.Errorf("Removed %d rows and cleared %s", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
 	} else {
-		logger.Log.Error("Found no rows to remove when pruning")
+		rlog.Error("Found no rows to remove when pruning")
 	}
 }

agent/pkg/models/models.go

@@ -3,33 +3,147 @@ package models
 import (
 	"encoding/json"
 
-	tapApi "github.com/up9inc/mizu/tap/api"
-
 	"mizuserver/pkg/rules"
+	"mizuserver/pkg/utils"
+	"time"
 
 	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
 )
 
-func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error {
+type DataUnmarshaler interface {
+	UnmarshalData(*MizuEntry) error
+}
+
+func GetEntry(r *MizuEntry, v DataUnmarshaler) error {
 	return v.UnmarshalData(r)
 }
 
+type MizuEntry struct {
+	ID                  uint `gorm:"primarykey"`
+	CreatedAt           time.Time
+	UpdatedAt           time.Time
+	Entry               string `json:"entry,omitempty" gorm:"column:entry"`
+	EntryId             string `json:"entryId" gorm:"column:entryId"`
+	Url                 string `json:"url" gorm:"column:url"`
+	Method              string `json:"method" gorm:"column:method"`
+	Status              int    `json:"status" gorm:"column:status"`
+	RequestSenderIp     string `json:"requestSenderIp" gorm:"column:requestSenderIp"`
+	Service             string `json:"service" gorm:"column:service"`
+	Timestamp           int64  `json:"timestamp" gorm:"column:timestamp"`
+	Path                string `json:"path" gorm:"column:path"`
+	ResolvedSource      string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
+	ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
+	IsOutgoing          bool   `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"`
+	EstimatedSizeBytes  int    `json:"-" gorm:"column:estimatedSizeBytes"`
+}
+
+type BaseEntryDetails struct {
+	Id              string          `json:"id,omitempty"`
+	Url             string          `json:"url,omitempty"`
+	RequestSenderIp string          `json:"requestSenderIp,omitempty"`
+	Service         string          `json:"service,omitempty"`
+	Path            string          `json:"path,omitempty"`
+	StatusCode      int             `json:"statusCode,omitempty"`
+	Method          string          `json:"method,omitempty"`
+	Timestamp       int64           `json:"timestamp,omitempty"`
+	IsOutgoing      bool            `json:"isOutgoing,omitempty"`
+	Latency         int64           `json:"latency,omitempty"`
+	Rules           ApplicableRules `json:"rules,omitempty"`
+}
+
+type ApplicableRules struct {
+	Latency int64 `json:"latency,omitempty"`
+	Status  bool  `json:"status,omitempty"`
+	NumberOfRules int `json:"numberOfRules,omitempty"`
+}
+
+func NewApplicableRules(status bool, latency int64, number int) ApplicableRules {
+	ar := ApplicableRules{}
+	ar.Status = status
+	ar.Latency = latency
+	ar.NumberOfRules = number
+	return ar
+}
+
+type FullEntryDetails struct {
+	har.Entry
+}
+
+type FullEntryDetailsExtra struct {
+	har.Entry
+}
+
+func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
+	entryUrl := entry.Url
+	service := entry.Service
+	if entry.ResolvedDestination != "" {
+		entryUrl = utils.SetHostname(entryUrl, entry.ResolvedDestination)
+		service = utils.SetHostname(service, entry.ResolvedDestination)
+	}
+	bed.Id = entry.EntryId
+	bed.Url = entryUrl
+	bed.Service = service
+	bed.Path = entry.Path
+	bed.StatusCode = entry.Status
+	bed.Method = entry.Method
+	bed.Timestamp = entry.Timestamp
+	bed.RequestSenderIp = entry.RequestSenderIp
+	bed.IsOutgoing = entry.IsOutgoing
+	return nil
+}
+
+func (fed *FullEntryDetails) UnmarshalData(entry *MizuEntry) error {
+	if err := json.Unmarshal([]byte(entry.Entry), &fed.Entry); err != nil {
+		return err
+	}
+
+	if entry.ResolvedDestination != "" {
+		fed.Entry.Request.URL = utils.SetHostname(fed.Entry.Request.URL, entry.ResolvedDestination)
+	}
+	return nil
+}
+
+func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *MizuEntry) error {
+	if err := json.Unmarshal([]byte(entry.Entry), &fedex.Entry); err != nil {
+		return err
+	}
+
+	if entry.ResolvedSource != "" {
+		fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.ResolvedSource})
+	}
+	if entry.ResolvedDestination != "" {
+		fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.ResolvedDestination})
+		fedex.Entry.Request.URL = utils.SetHostname(fedex.Entry.Request.URL, entry.ResolvedDestination)
+	}
+	return nil
+}
+
 type EntriesFilter struct {
 	Limit     int    `form:"limit" validate:"required,min=1,max=200"`
 	Operator  string `form:"operator" validate:"required,oneof='lt' 'gt'"`
 	Timestamp int64  `form:"timestamp" validate:"required,min=1"`
 }
 
+type UploadEntriesRequestQuery struct {
+	Dest             string `form:"dest"`
+	SleepIntervalSec int    `form:"interval"`
+}
+
+type HarFetchRequestQuery struct {
+	From int64 `form:"from"`
+	To   int64 `form:"to"`
+}
+
 type WebSocketEntryMessage struct {
 	*shared.WebSocketMessageMetadata
-	Data *tapApi.BaseEntryDetails `json:"data,omitempty"`
+	Data *BaseEntryDetails `json:"data,omitempty"`
 }
 
 type WebSocketTappedEntryMessage struct {
 	*shared.WebSocketMessageMetadata
-	Data *tapApi.OutputChannelItem
+	Data *tap.OutputChannelItem
 }
 
 type WebsocketOutboundLinkMessage struct {
@@ -37,12 +151,7 @@ type WebsocketOutboundLinkMessage struct {
 	Data *tap.OutboundLink
 }
 
-type AuthStatus struct {
-	Email string `json:"email"`
-	Model string `json:"model"`
-}
-
-func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntryDetails) ([]byte, error) {
+func CreateBaseEntryWebSocketMessage(base *BaseEntryDetails) ([]byte, error) {
 	message := &WebSocketEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
 			MessageType: shared.WebSocketMessageTypeEntry,
@@ -52,7 +161,7 @@ func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntryDetails) ([]byte, err
 	return json.Marshal(message)
 }
 
-func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte, error) {
+func CreateWebsocketTappedEntryMessage(base *tap.OutputChannelItem) ([]byte, error) {
 	message := &WebSocketTappedEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
 			MessageType: shared.WebSocketMessageTypeTappedEntry,
@@ -92,8 +201,26 @@ type ExtendedCreator struct {
 	Source *string `json:"_source"`
 }
 
-func RunValidationRulesState(harEntry har.Entry, service string) (tapApi.ApplicableRules, []rules.RulesMatched, bool) {
-	resultPolicyToSend, isEnabled := rules.MatchRequestPolicy(harEntry, service)
-	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend)
-	return tapApi.ApplicableRules{Status: statusPolicyToSend, Latency: latency, NumberOfRules: numberOfRules}, resultPolicyToSend, isEnabled
+type FullEntryWithPolicy struct {
+	RulesMatched []rules.RulesMatched `json:"rulesMatched,omitempty"`
+	Entry        har.Entry            `json:"entry"`
+	Service      string               `json:"service"`
+}
+
+func (fewp *FullEntryWithPolicy) UnmarshalData(entry *MizuEntry) error {
+	if err := json.Unmarshal([]byte(entry.Entry), &fewp.Entry); err != nil {
+		return err
+	}
+
+	_, resultPolicyToSend := rules.MatchRequestPolicy(fewp.Entry, entry.Service)
+	fewp.RulesMatched = resultPolicyToSend
+	fewp.Service = entry.Service
+	return nil
+}
+
+func RunValidationRulesState(harEntry har.Entry, service string) ApplicableRules {
+	numberOfRules, resultPolicyToSend := rules.MatchRequestPolicy(harEntry, service)
+	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend, numberOfRules)
+	ar := NewApplicableRules(statusPolicyToSend, latency, numberOfRules)
+	return ar
 }

agent/pkg/providers/status_provider.go

@@ -1,61 +1,19 @@
 package providers
 
 import (
-	"encoding/json"
-	"fmt"
 	"github.com/patrickmn/go-cache"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
-	"mizuserver/pkg/models"
-	"os"
-	"sync"
 	"time"
 )
 
 const tlsLinkRetainmentTime = time.Minute * 15
 
 var (
-	TappersCount   int
-	TapStatus      shared.TapStatus
-	authStatus     *models.AuthStatus
+	TapStatus shared.TapStatus
 	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
-
-	tappersCountLock = sync.Mutex{}
 )
 
-func GetAuthStatus() (*models.AuthStatus, error) {
-	if authStatus == nil {
-		syncEntriesConfigJson := os.Getenv(shared.SyncEntriesConfigEnvVar)
-		if syncEntriesConfigJson == "" {
-			authStatus = &models.AuthStatus{}
-			return authStatus, nil
-		}
-
-		syncEntriesConfig := &shared.SyncEntriesConfig{}
-		err := json.Unmarshal([]byte(syncEntriesConfigJson), syncEntriesConfig)
-		if err != nil {
-			return nil, fmt.Errorf("failed to marshal sync entries config, err: %v", err)
-		}
-
-		if syncEntriesConfig.Token == "" {
-			authStatus = &models.AuthStatus{}
-			return authStatus, nil
-		}
-
-		tokenEmail, err := shared.GetTokenEmail(syncEntriesConfig.Token)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get token email, err: %v", err)
-		}
-
-		authStatus = &models.AuthStatus{
-			Email: tokenEmail,
-			Model: syncEntriesConfig.Workspace,
-		}
-	}
-
-	return authStatus, nil
-}
-
 func GetAllRecentTLSAddresses() []string {
 	recentTLSLinks := make([]string, 0)
 
@@ -68,15 +26,3 @@ func GetAllRecentTLSAddresses() []string {
 
 	return recentTLSLinks
 }
-
-func TapperAdded() {
-	tappersCountLock.Lock()
-	TappersCount++
-	tappersCountLock.Unlock()
-}
-
-func TapperRemoved() {
-	tappersCountLock.Lock()
-	TappersCount--
-	tappersCountLock.Unlock()
-}

agent/pkg/resolver/README.md

@@ -32,7 +32,7 @@ Now you will be able to import `github.com/up9inc/mizu/resolver` in any `.go` fi
 errOut := make(chan error, 100)
 k8sResolver, err := resolver.NewFromOutOfCluster("", errOut)
 if err != nil {
-    logger.Log.Errorf("error creating k8s resolver %s", err)
+    rlog.Errorf("error creating k8s resolver %s", err)
 }
 
 ctx, cancel := context.WithCancel(context.Background())
@@ -40,15 +40,15 @@ k8sResolver.Start(ctx)
 
 resolvedName := k8sResolver.Resolve("10.107.251.91") // will always return `nil` in real scenarios as the internal map takes a moment to populate after `Start` is called
 if resolvedName != nil {
-    logger.Log.Errorf("resolved 10.107.251.91=%s", *resolvedName)
+    rlog.Errorf("resolved 10.107.251.91=%s", *resolvedName)
 } else {
-    logger.Log.Error("Could not find a resolved name for 10.107.251.91")
+    rlog.Error("Could not find a resolved name for 10.107.251.91")
 }
 
 for {
     select {
         case err := <- errOut:
-            logger.Log.Errorf("name resolving error %s", err)
+            rlog.Errorf("name resolving error %s", err)
     }
 }
 ```

agent/pkg/resolver/resolver.go

@@ -4,11 +4,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
-
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/romana/rlog"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 
-	cmap "github.com/orcaman/concurrent-map"
+	"github.com/orcaman/concurrent-map"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/watch"
@@ -140,13 +139,6 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
 			serviceHostname := fmt.Sprintf("%s.%s", service.Name, service.Namespace)
 			if service.Spec.ClusterIP != "" && service.Spec.ClusterIP != kubClientNullString {
 				resolver.saveResolvedName(service.Spec.ClusterIP, serviceHostname, event.Type)
-				if service.Spec.Ports != nil {
-					for _, port := range service.Spec.Ports {
-						if port.Port > 0 {
-							resolver.saveResolvedName(fmt.Sprintf("%s:%d", service.Spec.ClusterIP, port.Port), serviceHostname, event.Type)
-						}
-					}
-				}
 				resolver.saveServiceIP(service.Spec.ClusterIP, serviceHostname, event.Type)
 			}
 			if service.Status.LoadBalancer.Ingress != nil {
@@ -164,10 +156,10 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
 func (resolver *Resolver) saveResolvedName(key string, resolved string, eventType watch.EventType) {
 	if eventType == watch.Deleted {
 		resolver.nameMap.Remove(key)
-		logger.Log.Infof("setting %s=nil\n", key)
+		rlog.Infof("setting %s=nil\n", key)
 	} else {
 		resolver.nameMap.Set(key, resolved)
-		logger.Log.Infof("setting %s=%s\n", key, resolved)
+		rlog.Infof("setting %s=%s\n", key, resolved)
 	}
 }
 
@@ -188,7 +180,7 @@ func (resolver *Resolver) infiniteErrorHandleRetryFunc(ctx context.Context, fun
 			var statusError *k8serrors.StatusError
 			if errors.As(err, &statusError) {
 				if statusError.ErrStatus.Reason == metav1.StatusReasonForbidden {
-					logger.Log.Infof("Resolver loop encountered permission error, aborting event listening - %v\n", err)
+					rlog.Infof("Resolver loop encountered permission error, aborting event listening - %v\n", err)
 					return
 				}
 			}

agent/pkg/routes/entries_routes.go

@@ -7,8 +7,20 @@ import (
 
 // EntriesRoutes defines the group of har entries routes.
 func EntriesRoutes(ginApp *gin.Engine) {
-	routeGroup := ginApp.Group("/entries")
+	routeGroup := ginApp.Group("/api")
 
-	routeGroup.GET("/", controllers.GetEntries)        // get entries (base/thin entries)
-	routeGroup.GET("/:entryId", controllers.GetEntry) // get single (full) entry
+	routeGroup.GET("/entries", controllers.GetEntries)        // get entries (base/thin entries)
+	routeGroup.GET("/entries/:entryId", controllers.GetEntry) // get single (full) entry
+	routeGroup.GET("/exportEntries", controllers.GetFullEntries)
+	routeGroup.GET("/uploadEntries", controllers.UploadEntries)
+	routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
+
+	routeGroup.GET("/har", controllers.GetHARs)
+
+	routeGroup.GET("/resetDB", controllers.DeleteAllEntries)     // get single (full) entry
+	routeGroup.GET("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB
+
+	routeGroup.GET("/tapStatus", controllers.GetTappingStatus) // get tapping status
+	routeGroup.GET("/analyzeStatus", controllers.AnalyzeInformation)
+	routeGroup.GET("/recentTLSLinks", controllers.GetRecentTLSLinks)
 }

agent/pkg/routes/status_routes.go

@@ -9,16 +9,4 @@ func StatusRoutes(ginApp *gin.Engine) {
 	routeGroup := ginApp.Group("/status")
 
 	routeGroup.POST("/tappedPods", controllers.PostTappedPods)
-	routeGroup.GET("/tappersCount", controllers.GetTappersCount)
-	routeGroup.GET("/tap", controllers.GetTappingStatus)
-
-	routeGroup.GET("/auth", controllers.GetAuthStatus)
-
-	routeGroup.GET("/analyze", controllers.AnalyzeInformation)
-
-	routeGroup.GET("/general", controllers.GetGeneralStats) // get general stats about entries in DB
-
-	routeGroup.GET("/recentTLSLinks", controllers.GetRecentTLSLinks)
-
-	routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
 }

agent/pkg/rules/models.go

@@ -1,15 +1,12 @@
 package rules
 
 import (
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"reflect"
 	"regexp"
 	"strings"
 
-	"github.com/up9inc/mizu/shared/logger"
-
 	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
 	jsonpath "github.com/yalp/jsonpath"
@@ -44,19 +41,16 @@ func ValidateService(serviceFromRule string, service string) bool {
 	return true
 }
 
-func MatchRequestPolicy(harEntry har.Entry, service string) (resultPolicyToSend []RulesMatched, isEnabled bool) {
-	enforcePolicy, err := shared.DecodeEnforcePolicy(fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.RulePolicyFileName))
-	if err == nil && len(enforcePolicy.Rules) > 0 {
-		isEnabled = true
-	}
+func MatchRequestPolicy(harEntry har.Entry, service string) (int, []RulesMatched) {
+	enforcePolicy, _ := shared.DecodeEnforcePolicy(fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.RulePolicyFileName))
+	var resultPolicyToSend []RulesMatched
 	for _, rule := range enforcePolicy.Rules {
 		if !ValidatePath(rule.Path, harEntry.Request.URL) || !ValidateService(rule.Service, service) {
 			continue
 		}
 		if rule.Type == "json" {
 			var bodyJsonMap interface{}
-			contentTextDecoded, _ := base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text))
-			if err := json.Unmarshal(contentTextDecoded, &bodyJsonMap); err != nil {
+			if err := json.Unmarshal(harEntry.Response.Content.Text, &bodyJsonMap); err != nil {
 				continue
 			}
 			out, err := jsonpath.Read(bodyJsonMap, rule.Key)
@@ -69,7 +63,6 @@ func MatchRequestPolicy(harEntry har.Entry, service string) (resultPolicyToSend
 				if err != nil {
 					continue
 				}
-				logger.Log.Info(matchValue, rule.Value)
 			} else {
 				val := fmt.Sprint(out)
 				matchValue, err = regexp.MatchString(rule.Value, val)
@@ -96,28 +89,22 @@ func MatchRequestPolicy(harEntry har.Entry, service string) (resultPolicyToSend
 			resultPolicyToSend = appendRulesMatched(resultPolicyToSend, true, rule)
 		}
 	}
-	return
+	return len(enforcePolicy.Rules), resultPolicyToSend
 }
 
-func PassedValidationRules(rulesMatched []RulesMatched) (bool, int64, int) {
-	var numberOfRulesMatched = len(rulesMatched)
-	var responseTime int64 = -1
-
-	if numberOfRulesMatched == 0 {
-		return false, 0, numberOfRulesMatched
+func PassedValidationRules(rulesMatched []RulesMatched, numberOfRules int) (bool, int64, int) {
+	if len(rulesMatched) == 0 {
+		return false, 0, 0
 	}
-
 	for _, rule := range rulesMatched {
 		if rule.Matched == false {
-			return false, responseTime, numberOfRulesMatched
-		} else {
-			if strings.ToLower(rule.Rule.Type) == "slo" {
-				if rule.Rule.ResponseTime < responseTime || responseTime == -1 {
-					responseTime = rule.Rule.ResponseTime
-				}
-			}
+			return false, -1, len(rulesMatched)
 		}
 	}
-
-	return true, responseTime, numberOfRulesMatched
+	for _, rule := range rulesMatched {
+		if strings.ToLower(rule.Rule.Type) == "latency" {
+			return true, rule.Rule.Latency, len(rulesMatched)
+		}
+	}
+	return true, -1, len(rulesMatched)
 }

agent/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go

@@ -0,0 +1,200 @@
+package sensitiveDataFiltering
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"github.com/up9inc/mizu/tap"
+	"net/url"
+	"strings"
+
+	"github.com/beevik/etree"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared"
+)
+
+func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, options *shared.TrafficFilteringOptions) {
+	harOutputItem.HarEntry.Request.Headers = filterHarHeaders(harOutputItem.HarEntry.Request.Headers)
+	harOutputItem.HarEntry.Response.Headers = filterHarHeaders(harOutputItem.HarEntry.Response.Headers)
+
+	harOutputItem.HarEntry.Request.Cookies = make([]har.Cookie, 0, 0)
+	harOutputItem.HarEntry.Response.Cookies = make([]har.Cookie, 0, 0)
+
+	harOutputItem.HarEntry.Request.URL = filterUrl(harOutputItem.HarEntry.Request.URL)
+	for i, queryString := range harOutputItem.HarEntry.Request.QueryString {
+		if isFieldNameSensitive(queryString.Name) {
+			harOutputItem.HarEntry.Request.QueryString[i].Value = maskedFieldPlaceholderValue
+		}
+	}
+
+	if harOutputItem.HarEntry.Request.PostData != nil {
+		requestContentType := getContentTypeHeaderValue(harOutputItem.HarEntry.Request.Headers)
+		filteredRequestBody, err := filterHttpBody([]byte(harOutputItem.HarEntry.Request.PostData.Text), requestContentType, options)
+		if err == nil {
+			harOutputItem.HarEntry.Request.PostData.Text = string(filteredRequestBody)
+		}
+	}
+	if harOutputItem.HarEntry.Response.Content != nil {
+		responseContentType := getContentTypeHeaderValue(harOutputItem.HarEntry.Response.Headers)
+		filteredResponseBody, err := filterHttpBody(harOutputItem.HarEntry.Response.Content.Text, responseContentType, options)
+		if err == nil {
+			harOutputItem.HarEntry.Response.Content.Text = filteredResponseBody
+		}
+	}
+}
+
+func filterHarHeaders(headers []har.Header) []har.Header {
+	newHeaders := make([]har.Header, 0)
+	for i, header := range headers {
+		if strings.ToLower(header.Name) == "cookie" {
+			continue
+		} else if isFieldNameSensitive(header.Name) {
+			newHeaders = append(newHeaders, har.Header{Name: header.Name, Value: maskedFieldPlaceholderValue})
+			headers[i].Value = maskedFieldPlaceholderValue
+		} else {
+			newHeaders = append(newHeaders, header)
+		}
+	}
+	return newHeaders
+}
+
+func getContentTypeHeaderValue(headers []har.Header) string {
+	for _, header := range headers {
+		if strings.ToLower(header.Name) == "content-type" {
+			return header.Value
+		}
+	}
+	return ""
+}
+
+func isFieldNameSensitive(fieldName string) bool {
+	name := strings.ToLower(fieldName)
+	name = strings.ReplaceAll(name, "_", "")
+	name = strings.ReplaceAll(name, "-", "")
+	name = strings.ReplaceAll(name, " ", "")
+
+	for _, sensitiveField := range personallyIdentifiableDataFields {
+		if strings.Contains(name, sensitiveField) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func filterHttpBody(bytes []byte, contentType string, options *shared.TrafficFilteringOptions) ([]byte, error) {
+	mimeType := strings.Split(contentType, ";")[0]
+	switch strings.ToLower(mimeType) {
+	case "application/json":
+		return filterJsonBody(bytes)
+	case "text/html":
+		fallthrough
+	case "application/xhtml+xml":
+		fallthrough
+	case "text/xml":
+		fallthrough
+	case "application/xml":
+		return filterXmlEtree(bytes)
+	case "text/plain":
+		if options != nil && options.PlainTextMaskingRegexes != nil {
+			return filterPlainText(bytes, options), nil
+		}
+	}
+	return bytes, nil
+}
+
+func filterPlainText(bytes []byte, options *shared.TrafficFilteringOptions) []byte {
+	for _, regex := range options.PlainTextMaskingRegexes {
+		bytes = regex.ReplaceAll(bytes, []byte(maskedFieldPlaceholderValue))
+	}
+	return bytes
+}
+
+func filterXmlEtree(bytes []byte) ([]byte, error) {
+	if !IsValidXML(bytes) {
+		return nil, errors.New("Invalid XML")
+	}
+	xmlDoc := etree.NewDocument()
+	err := xmlDoc.ReadFromBytes(bytes)
+	if err != nil {
+		return nil, err
+	} else {
+		filterXmlElement(xmlDoc.Root())
+	}
+	return xmlDoc.WriteToBytes()
+}
+
+func IsValidXML(data []byte) bool {
+	return xml.Unmarshal(data, new(interface{})) == nil
+}
+
+func filterXmlElement(element *etree.Element) {
+	for i, attribute := range element.Attr {
+		if isFieldNameSensitive(attribute.Key) {
+			element.Attr[i].Value = maskedFieldPlaceholderValue
+		}
+	}
+	if element.ChildElements() == nil || len(element.ChildElements()) == 0 {
+		if isFieldNameSensitive(element.Tag) {
+			element.SetText(maskedFieldPlaceholderValue)
+		}
+	} else {
+		for _, element := range element.ChildElements() {
+			filterXmlElement(element)
+		}
+	}
+}
+
+func filterJsonBody(bytes []byte) ([]byte, error) {
+	var bodyJsonMap map[string] interface{}
+	err := json.Unmarshal(bytes ,&bodyJsonMap)
+	if err != nil {
+		return nil, err
+	}
+	filterJsonMap(bodyJsonMap)
+	return json.Marshal(bodyJsonMap)
+}
+
+func filterJsonMap(jsonMap map[string] interface{}) {
+	for key, value := range jsonMap {
+		// Do not replace nil values with maskedFieldPlaceholderValue
+		if value == nil {
+			continue
+		}
+
+		nestedMap, isNested := value.(map[string] interface{})
+		if isNested {
+			filterJsonMap(nestedMap)
+		} else {
+			if isFieldNameSensitive(key) {
+				jsonMap[key] = maskedFieldPlaceholderValue
+			}
+		}
+	}
+}
+
+// receives string representing url, returns string url without sensitive query param values (http://service/api?userId=bob&password=123&type=login -> http://service/api?userId=[REDACTED]&password=[REDACTED]&type=login)
+func filterUrl(originalUrl string) string {
+	parsedUrl, err := url.Parse(originalUrl)
+	if err != nil {
+		return fmt.Sprintf("http://%s", maskedFieldPlaceholderValue)
+	} else {
+		if len(parsedUrl.RawQuery) > 0 {
+			newQueryArgs := make([]string, 0)
+			for urlQueryParamName, urlQueryParamValues := range parsedUrl.Query() {
+				newValues := urlQueryParamValues
+				if isFieldNameSensitive(urlQueryParamName) {
+					newValues = []string {maskedFieldPlaceholderValue}
+				}
+				for _, paramValue := range newValues {
+					newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, paramValue))
+				}
+			}
+
+			parsedUrl.RawQuery = strings.Join(newQueryArgs, "&")
+		}
+
+		return parsedUrl.String()
+	}
+}

agent/pkg/up9/main.go

@@ -3,22 +3,18 @@ package up9
 import (
 	"bytes"
 	"compress/zlib"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared"
 	"io/ioutil"
+	"log"
 	"mizuserver/pkg/database"
-	"mizuserver/pkg/utils"
+	"mizuserver/pkg/models"
 	"net/http"
 	"net/url"
-	"regexp"
 	"strings"
 	"time"
-
-	"github.com/google/martian/har"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 const (
@@ -34,24 +30,41 @@ type ModelStatus struct {
 	LastMajorGeneration float64 `json:"lastMajorGeneration"`
 }
 
-func GetRemoteUrl(analyzeDestination string, analyzeModel string, analyzeToken string, guestMode bool) string {
-	if guestMode {
-		return fmt.Sprintf("https://%s/share/%s", analyzeDestination, analyzeToken)
+func getGuestToken(url string, target *GuestToken) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
 	}
-
-	return fmt.Sprintf("https://%s/app/workspaces/%s", analyzeDestination, analyzeModel)
+	defer resp.Body.Close()
+	rlog.Infof("Got token from the server, starting to json decode... status code: %v", resp.StatusCode)
+	return json.NewDecoder(resp.Body).Decode(target)
 }
 
-func CheckIfModelReady(analyzeDestination string, analyzeModel string, analyzeToken string, guestMode bool) bool {
-	statusUrl, _ := url.Parse(fmt.Sprintf("https://trcc.%s/models/%s/status", analyzeDestination, analyzeModel))
+func CreateAnonymousToken(envPrefix string) (*GuestToken, error) {
+	tokenUrl := fmt.Sprintf("https://trcc.%s/anonymous/token", envPrefix)
+	if strings.HasPrefix(envPrefix, "http") {
+		tokenUrl = fmt.Sprintf("%s/api/token", envPrefix)
+	}
+	token := &GuestToken{}
+	if err := getGuestToken(tokenUrl, token); err != nil {
+		rlog.Infof("Failed to get token, %s", err)
+		return nil, err
+	}
+	return token, nil
+}
 
-	authHeader := getAuthHeader(guestMode)
+func GetRemoteUrl(analyzeDestination string, analyzeToken string) string {
+	return fmt.Sprintf("https://%s/share/%s", analyzeDestination, analyzeToken)
+}
+
+func CheckIfModelReady(analyzeDestination string, analyzeModel string, analyzeToken string) bool {
+	statusUrl, _ := url.Parse(fmt.Sprintf("https://trcc.%s/models/%s/status", analyzeDestination, analyzeModel))
 	req := &http.Request{
 		Method: http.MethodGet,
 		URL:    statusUrl,
 		Header: map[string][]string{
 			"Content-Type": {"application/json"},
-			authHeader:     {analyzeToken},
+			"Guest-Auth":   {analyzeToken},
 		},
 	}
 	statusResp, err := http.DefaultClient.Do(req)
@@ -66,23 +79,17 @@ func CheckIfModelReady(analyzeDestination string, analyzeModel string, analyzeTo
 	return target.LastMajorGeneration > 0
 }
 
-func getAuthHeader(guestMode bool) string {
-	if guestMode {
-		return "Guest-Auth"
-	}
-
-	return "Authorization"
-}
-
 func GetTrafficDumpUrl(analyzeDestination string, analyzeModel string) *url.URL {
 	strUrl := fmt.Sprintf("https://traffic.%s/dumpTrafficBulk/%s", analyzeDestination, analyzeModel)
+	if strings.HasPrefix(analyzeDestination, "http") {
+		strUrl = fmt.Sprintf("%s/api/workspace/dumpTrafficBulk", analyzeDestination)
+	}
 	postUrl, _ := url.Parse(strUrl)
 	return postUrl
 }
 
 type AnalyzeInformation struct {
 	IsAnalyzing        bool
-	GuestMode          bool
 	SentCount          int
 	AnalyzedModel      string
 	AnalyzeToken       string
@@ -91,7 +98,6 @@ type AnalyzeInformation struct {
 
 func (info *AnalyzeInformation) Reset() {
 	info.IsAnalyzing = false
-	info.GuestMode = true
 	info.AnalyzedModel = ""
 	info.AnalyzeToken = ""
 	info.AnalyzeDestination = ""
@@ -103,151 +109,45 @@ var analyzeInformation = &AnalyzeInformation{}
 func GetAnalyzeInfo() *shared.AnalyzeStatus {
 	return &shared.AnalyzeStatus{
 		IsAnalyzing:   analyzeInformation.IsAnalyzing,
-		RemoteUrl:     GetRemoteUrl(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzedModel, analyzeInformation.AnalyzeToken, analyzeInformation.GuestMode),
-		IsRemoteReady: CheckIfModelReady(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzedModel, analyzeInformation.AnalyzeToken, analyzeInformation.GuestMode),
+		RemoteUrl:     GetRemoteUrl(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzeToken),
+		IsRemoteReady: CheckIfModelReady(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzedModel, analyzeInformation.AnalyzeToken),
 		SentCount:     analyzeInformation.SentCount,
 	}
 }
 
-func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error {
-	logger.Log.Infof("Sync entries - started\n")
-
-	var (
-		token, model string
-		guestMode    bool
-	)
-	if syncEntriesConfig.Token == "" {
-		logger.Log.Infof("Sync entries - creating anonymous token. env %s\n", syncEntriesConfig.Env)
-		guestToken, err := createAnonymousToken(syncEntriesConfig.Env)
-		if err != nil {
-			return fmt.Errorf("failed creating anonymous token, err: %v", err)
-		}
-
-		token = guestToken.Token
-		model = guestToken.Model
-		guestMode = true
-	} else {
-		token = fmt.Sprintf("bearer %s", syncEntriesConfig.Token)
-		model = syncEntriesConfig.Workspace
-		guestMode = false
-
-		logger.Log.Infof("Sync entries - upserting model. env %s, model %s\n", syncEntriesConfig.Env, model)
-		if err := upsertModel(token, model, syncEntriesConfig.Env); err != nil {
-			return fmt.Errorf("failed upserting model, err: %v", err)
-		}
-	}
-
-	modelRegex, _ := regexp.Compile("[A-Za-z0-9][-A-Za-z0-9_.]*[A-Za-z0-9]+$")
-	if len(model) > 63 || !modelRegex.MatchString(model) {
-		return fmt.Errorf("invalid model name, model name: %s", model)
-	}
-
-	logger.Log.Infof("Sync entries - syncing. token: %s, model: %s, guest mode: %v\n", token, model, guestMode)
-	go syncEntriesImpl(token, model, syncEntriesConfig.Env, syncEntriesConfig.UploadIntervalSec, guestMode)
-
-	return nil
-}
-
-func upsertModel(token string, model string, envPrefix string) error {
-	upsertModelUrl, _ := url.Parse(fmt.Sprintf("https://trcc.%s/models/%s", envPrefix, model))
-
-	authHeader := getAuthHeader(false)
-	req := &http.Request{
-		Method: http.MethodPost,
-		URL:    upsertModelUrl,
-		Header: map[string][]string{
-			authHeader: {token},
-		},
-	}
-
-	response, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return fmt.Errorf("failed request to upsert model, err: %v", err)
-	}
-
-	// In case the model is not created (not 201) and doesn't exists (not 409)
-	if response.StatusCode != 201 && response.StatusCode != 409 {
-		return fmt.Errorf("failed request to upsert model, status code: %v", response.StatusCode)
-	}
-
-	return nil
-}
-
-func createAnonymousToken(envPrefix string) (*GuestToken, error) {
-	tokenUrl := fmt.Sprintf("https://trcc.%s/anonymous/token", envPrefix)
-	if strings.HasPrefix(envPrefix, "http") {
-		tokenUrl = fmt.Sprintf("%s/api/token", envPrefix)
-	}
-	token := &GuestToken{}
-	if err := getGuestToken(tokenUrl, token); err != nil {
-		logger.Log.Infof("Failed to get token, %s", err)
-		return nil, err
-	}
-	return token, nil
-}
-
-func getGuestToken(url string, target *GuestToken) error {
-	resp, err := http.Get(url)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	logger.Log.Infof("Got token from the server, starting to json decode... status code: %v", resp.StatusCode)
-	return json.NewDecoder(resp.Body).Decode(target)
-}
-
-func syncEntriesImpl(token string, model string, envPrefix string, uploadIntervalSec int, guestMode bool) {
+func UploadEntriesImpl(token string, model string, envPrefix string, sleepIntervalSec int) {
 	analyzeInformation.IsAnalyzing = true
-	analyzeInformation.GuestMode = guestMode
 	analyzeInformation.AnalyzedModel = model
 	analyzeInformation.AnalyzeToken = token
 	analyzeInformation.AnalyzeDestination = envPrefix
 	analyzeInformation.SentCount = 0
 
-	sleepTime := time.Second * time.Duration(uploadIntervalSec)
+	sleepTime := time.Second * time.Duration(sleepIntervalSec)
 
 	var timestampFrom int64 = 0
 
 	for {
 		timestampTo := time.Now().UnixNano() / int64(time.Millisecond)
-		logger.Log.Infof("Getting entries from %v, to %v\n", timestampFrom, timestampTo)
-		protocolFilter := "http"
-		entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo, &protocolFilter)
+		rlog.Infof("Getting entries from %v, to %v\n", timestampFrom, timestampTo)
+		entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
 
 		if len(entriesArray) > 0 {
-			result := make([]har.Entry, 0)
+
+			fullEntriesExtra := make([]models.FullEntryDetailsExtra, 0)
 			for _, data := range entriesArray {
-				var pair tapApi.RequestResponsePair
-				if err := json.Unmarshal([]byte(data.Entry), &pair); err != nil {
+				harEntry := models.FullEntryDetailsExtra{}
+				if err := models.GetEntry(&data, &harEntry); err != nil {
 					continue
 				}
-				harEntry, err := utils.NewEntry(&pair)
-				if err != nil {
-					continue
-				}
-				if data.ResolvedSource != "" {
-					harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: data.ResolvedSource})
-				}
-				if data.ResolvedDestination != "" {
-					harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: data.ResolvedDestination})
-					harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, data.ResolvedDestination)
-				}
-
-				// go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64
-				if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil {
-					continue
-				}
-
-				result = append(result, *harEntry)
+				fullEntriesExtra = append(fullEntriesExtra, harEntry)
 			}
+			rlog.Infof("About to upload %v entries\n", len(fullEntriesExtra))
 
-			logger.Log.Infof("About to upload %v entries\n", len(result))
-
-			body, jMarshalErr := json.Marshal(result)
+			body, jMarshalErr := json.Marshal(fullEntriesExtra)
 			if jMarshalErr != nil {
 				analyzeInformation.Reset()
-				logger.Log.Infof("Stopping sync entries")
-				logger.Log.Fatal(jMarshalErr)
+				rlog.Infof("Stopping analyzing")
+				log.Fatal(jMarshalErr)
 			}
 
 			var in bytes.Buffer
@@ -256,31 +156,30 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 			_ = w.Close()
 			reqBody := ioutil.NopCloser(bytes.NewReader(in.Bytes()))
 
-			authHeader := getAuthHeader(guestMode)
 			req := &http.Request{
 				Method: http.MethodPost,
 				URL:    GetTrafficDumpUrl(envPrefix, model),
 				Header: map[string][]string{
 					"Content-Encoding": {"deflate"},
 					"Content-Type":     {"application/octet-stream"},
-					authHeader:         {token},
+					"Guest-Auth":       {token},
 				},
 				Body: reqBody,
 			}
 
 			if _, postErr := http.DefaultClient.Do(req); postErr != nil {
 				analyzeInformation.Reset()
-				logger.Log.Info("Stopping sync entries")
-				logger.Log.Fatal(postErr)
+				rlog.Info("Stopping analyzing")
+				log.Fatal(postErr)
 			}
 			analyzeInformation.SentCount += len(entriesArray)
-			logger.Log.Infof("Finish uploading %v entries to %s\n", len(entriesArray), GetTrafficDumpUrl(envPrefix, model))
+			rlog.Infof("Finish uploading %v entries to %s\n", len(entriesArray), GetTrafficDumpUrl(envPrefix, model))
 
 		} else {
-			logger.Log.Infof("Nothing to upload")
+			rlog.Infof("Nothing to upload")
 		}
 
-		logger.Log.Infof("Sleeping for %v...\n", sleepTime)
+		rlog.Infof("Sleeping for %v...\n", sleepTime)
 		time.Sleep(sleepTime)
 		timestampFrom = timestampTo
 	}

agent/pkg/utils/har.go

@@ -1,257 +0,0 @@
-package utils
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/google/martian/har"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/tap/api"
-)
-
-// Keep it because we might want cookies in the future
-//func BuildCookies(rawCookies []interface{}) []har.Cookie {
-//	cookies := make([]har.Cookie, 0, len(rawCookies))
-//
-//	for _, cookie := range rawCookies {
-//		c := cookie.(map[string]interface{})
-//		expiresStr := ""
-//		if c["expires"] != nil {
-//			expiresStr = c["expires"].(string)
-//		}
-//		expires, _ := time.Parse(time.RFC3339, expiresStr)
-//		httpOnly := false
-//		if c["httponly"] != nil {
-//			httpOnly, _ = strconv.ParseBool(c["httponly"].(string))
-//		}
-//		secure := false
-//		if c["secure"] != nil {
-//			secure, _ = strconv.ParseBool(c["secure"].(string))
-//		}
-//		path := ""
-//		if c["path"] != nil {
-//			path = c["path"].(string)
-//		}
-//		domain := ""
-//		if c["domain"] != nil {
-//			domain = c["domain"].(string)
-//		}
-//
-//		cookies = append(cookies, har.Cookie{
-//			Name:        c["name"].(string),
-//			Value:       c["value"].(string),
-//			Path:        path,
-//			Domain:      domain,
-//			HTTPOnly:    httpOnly,
-//			Secure:      secure,
-//			Expires:     expires,
-//			Expires8601: expiresStr,
-//		})
-//	}
-//
-//	return cookies
-//}
-
-func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, string, string, string) {
-	var host, scheme, authority, path, status string
-	headers := make([]har.Header, 0, len(rawHeaders))
-
-	for _, header := range rawHeaders {
-		h := header.(map[string]interface{})
-
-		headers = append(headers, har.Header{
-			Name:  h["name"].(string),
-			Value: h["value"].(string),
-		})
-
-		if h["name"] == "Host" {
-			host = h["value"].(string)
-		}
-		if h["name"] == ":authority" {
-			authority = h["value"].(string)
-		}
-		if h["name"] == ":scheme" {
-			scheme = h["value"].(string)
-		}
-		if h["name"] == ":path" {
-			path = h["value"].(string)
-		}
-		if h["name"] == ":status" {
-			status = h["value"].(string)
-		}
-	}
-
-	return headers, host, scheme, authority, path, status
-}
-
-func BuildPostParams(rawParams []interface{}) []har.Param {
-	params := make([]har.Param, 0, len(rawParams))
-	for _, param := range rawParams {
-		p := param.(map[string]interface{})
-		name := ""
-		if p["name"] != nil {
-			name = p["name"].(string)
-		}
-		value := ""
-		if p["value"] != nil {
-			value = p["value"].(string)
-		}
-		fileName := ""
-		if p["fileName"] != nil {
-			fileName = p["fileName"].(string)
-		}
-		contentType := ""
-		if p["contentType"] != nil {
-			contentType = p["contentType"].(string)
-		}
-
-		params = append(params, har.Param{
-			Name:        name,
-			Value:       value,
-			Filename:    fileName,
-			ContentType: contentType,
-		})
-	}
-
-	return params
-}
-
-func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error) {
-	reqDetails := request.Payload.(map[string]interface{})["details"].(map[string]interface{})
-
-	headers, host, scheme, authority, path, _ := BuildHeaders(reqDetails["headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(reqDetails["cookies"].([]interface{}))
-
-	postData, _ := reqDetails["postData"].(map[string]interface{})
-	mimeType, _ := postData["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
-	}
-	text, _ := postData["text"]
-	postDataText := ""
-	if text != nil {
-		postDataText = text.(string)
-	}
-
-	queryString := make([]har.QueryString, 0)
-	for _, _qs := range reqDetails["queryString"].([]interface{}) {
-		qs := _qs.(map[string]interface{})
-		queryString = append(queryString, har.QueryString{
-			Name:  qs["name"].(string),
-			Value: qs["value"].(string),
-		})
-	}
-
-	url := fmt.Sprintf("http://%s%s", host, reqDetails["url"].(string))
-	if strings.HasPrefix(mimeType.(string), "application/grpc") {
-		url = fmt.Sprintf("%s://%s%s", scheme, authority, path)
-	}
-
-	harParams := make([]har.Param, 0)
-	if postData["params"] != nil {
-		harParams = BuildPostParams(postData["params"].([]interface{}))
-	}
-
-	harRequest = &har.Request{
-		Method:      reqDetails["method"].(string),
-		URL:         url,
-		HTTPVersion: reqDetails["httpVersion"].(string),
-		HeadersSize: -1,
-		BodySize:    int64(bytes.NewBufferString(postDataText).Len()),
-		QueryString: queryString,
-		Headers:     headers,
-		Cookies:     cookies,
-		PostData: &har.PostData{
-			MimeType: mimeType.(string),
-			Params:   harParams,
-			Text:     postDataText,
-		},
-	}
-
-	return
-}
-
-func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err error) {
-	resDetails := response.Payload.(map[string]interface{})["details"].(map[string]interface{})
-
-	headers, _, _, _, _, _status := BuildHeaders(resDetails["headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(resDetails["cookies"].([]interface{}))
-
-	content, _ := resDetails["content"].(map[string]interface{})
-	mimeType, _ := content["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
-	}
-	encoding, _ := content["encoding"]
-	text, _ := content["text"]
-	bodyText := ""
-	if text != nil {
-		bodyText = text.(string)
-	}
-
-	harContent := &har.Content{
-		Encoding: encoding.(string),
-		MimeType: mimeType.(string),
-		Text:     []byte(bodyText),
-		Size:     int64(len(bodyText)),
-	}
-
-	status := int(resDetails["status"].(float64))
-	if strings.HasPrefix(mimeType.(string), "application/grpc") {
-		status, err = strconv.Atoi(_status)
-		if err != nil {
-			logger.Log.Errorf("Failed converting status to int %s (%v,%+v)", err, err, err)
-			return nil, errors.New("failed converting response status to int for HAR")
-		}
-	}
-
-	harResponse = &har.Response{
-		HTTPVersion: resDetails["httpVersion"].(string),
-		Status:      status,
-		StatusText:  resDetails["statusText"].(string),
-		HeadersSize: -1,
-		BodySize:    int64(bytes.NewBufferString(bodyText).Len()),
-		Headers:     headers,
-		Cookies:     cookies,
-		Content:     harContent,
-	}
-	return
-}
-
-func NewEntry(pair *api.RequestResponsePair) (*har.Entry, error) {
-	harRequest, err := NewRequest(&pair.Request)
-	if err != nil {
-		logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
-		return nil, errors.New("failed converting request to HAR")
-	}
-
-	harResponse, err := NewResponse(&pair.Response)
-	if err != nil {
-		logger.Log.Errorf("Failed converting response to HAR %s (%v,%+v)", err, err, err)
-		return nil, errors.New("failed converting response to HAR")
-	}
-
-	totalTime := pair.Response.CaptureTime.Sub(pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
-	if totalTime < 1 {
-		totalTime = 1
-	}
-
-	harEntry := har.Entry{
-		StartedDateTime: pair.Request.CaptureTime,
-		Time:            totalTime,
-		Request:         harRequest,
-		Response:        harResponse,
-		Cache:           &har.Cache{},
-		Timings: &har.Timings{
-			Send:    -1,
-			Wait:    -1,
-			Receive: totalTime,
-		},
-	}
-
-	return &harEntry, nil
-}

agent/pkg/utils/truncating_logger.go

@@ -3,11 +3,10 @@ package utils
 import (
 	"context"
 	"fmt"
-	"time"
-
-	loggerShared "github.com/up9inc/mizu/shared/logger"
+	"github.com/romana/rlog"
 	"gorm.io/gorm/logger"
 	"gorm.io/gorm/utils"
+	"time"
 )
 
 // TruncatingLogger implements the gorm logger.Interface interface. Its purpose is to act as gorm's logger while truncating logs to a max of 50 characters to minimise the performance impact
@@ -25,21 +24,21 @@ func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string
 	if truncatingLogger.LogLevel < logger.Info {
 		return
 	}
-	loggerShared.Log.Errorf("gorm info: %.150s", message)
+	rlog.Errorf("gorm info: %.150s", message)
 }
 
 func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) {
 	if truncatingLogger.LogLevel < logger.Warn {
 		return
 	}
-	loggerShared.Log.Errorf("gorm warning: %.150s", message)
+	rlog.Errorf("gorm warning: %.150s", message)
 }
 
 func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) {
 	if truncatingLogger.LogLevel < logger.Error {
 		return
 	}
-	loggerShared.Log.Errorf("gorm error: %.150s", message)
+	rlog.Errorf("gorm error: %.150s", message)
 }
 
 func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) {

agent/pkg/utils/utils.go

@@ -2,7 +2,9 @@ package utils
 
 import (
 	"context"
-	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/romana/rlog"
+	"log"
 	"net/http"
 	"net/url"
 	"os"
@@ -10,18 +12,14 @@ import (
 	"reflect"
 	"syscall"
 	"time"
-
-	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 // StartServer starts the server with a graceful shutdown
 func StartServer(app *gin.Engine) {
 	signals := make(chan os.Signal, 2)
 	signal.Notify(signals,
-		os.Interrupt,    // this catch ctrl + c
-		syscall.SIGTSTP, // this catch ctrl + z
+		os.Interrupt,  	  // this catch ctrl + c
+		syscall.SIGTSTP,  // this catch ctrl + z
 	)
 
 	srv := &http.Server{
@@ -31,16 +29,15 @@ func StartServer(app *gin.Engine) {
 
 	go func() {
 		_ = <-signals
-		logger.Log.Infof("Shutting down...")
+		rlog.Infof("Shutting down...")
 		ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
 		_ = srv.Shutdown(ctx)
 		os.Exit(0)
 	}()
 
 	// Run server.
-	logger.Log.Infof("Starting the server...")
-	if err := app.Run(fmt.Sprintf(":%d", shared.DefaultApiServerPort)); err != nil {
-		logger.Log.Errorf("Server is not running! Reason: %v", err)
+	if err := app.Run(":8899"); err != nil {
+		log.Printf("Oops... Server is not running! Reason: %v", err)
 	}
 }
 
@@ -57,14 +54,15 @@ func ReverseSlice(data interface{}) {
 
 func CheckErr(e error) {
 	if e != nil {
-		logger.Log.Errorf("%v", e)
+		log.Printf("%v", e)
+		//panic(e)
 	}
 }
 
 func SetHostname(address, newHostname string) string {
 	replacedUrl, err := url.Parse(address)
-	if err != nil {
-		logger.Log.Errorf("error replacing hostname to %s in address %s, returning original %v", newHostname, address, err)
+	if err != nil{
+		log.Printf("error replacing hostname to %s in address %s, returning original %v",newHostname, address, err)
 		return address
 	}
 	replacedUrl.Host = newHostname

cli/Makefile

@@ -27,9 +27,8 @@ build-all: ## Build for all supported platforms.
 	@mkdir -p bin && sed s/_SEM_VER_/$(SEM_VER)/g README.md.TEMPLATE >  bin/README.md
 	@$(MAKE) build GOOS=darwin GOARCH=amd64
 	@$(MAKE) build GOOS=linux GOARCH=amd64
-	@$(MAKE) build GOOS=darwin GOARCH=arm64
-	@$(MAKE) build GOOS=windows GOARCH=amd64
-	@mv ./bin/mizu_windows_amd64 ./bin/mizu.exe
+	@# $(MAKE) build GOOS=darwin GOARCH=arm64
+	@# $(MAKE) GOOS=windows GOARCH=amd64
 	@# $(MAKE) GOOS=linux GOARCH=386
 	@# $(MAKE) GOOS=windows GOARCH=386
 	@# $(MAKE) GOOS=linux GOARCH=arm64

cli/README.md.TEMPLATE

@@ -2,25 +2,16 @@
 
 Download Mizu for your platform
 
-**Mac** (Intel)
+**Mac** (on Intel chip)
 ```
 curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_darwin_amd64 && chmod 755 mizu
 ```
 
-**Mac** (Apple M1 silicon)
-```
-curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_darwin_arm64 && chmod 755 mizu
-```
-
 **Linux** 
 ```
 curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_linux_amd64 && chmod 755 mizu
 ```
 
-**Windows** (Intel 64bit)
-```
-curl -LO https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu.exe
-```
 
 ### Checksums
 SHA256 checksums available for compiled binaries.

cli/apiserver/provider.go

@@ -1,133 +0,0 @@
-package apiserver
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"time"
-
-	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	core "k8s.io/api/core/v1"
-)
-
-type apiServerProvider struct {
-	url     string
-	isReady bool
-	retries int
-}
-
-var Provider = apiServerProvider{retries: config.GetIntEnvConfig(config.ApiServerRetries, 20)}
-
-func (provider *apiServerProvider) InitAndTestConnection(url string) error {
-	healthUrl := fmt.Sprintf("%s/", url)
-	retriesLeft := provider.retries
-	for retriesLeft > 0 {
-		if response, err := http.Get(healthUrl); err != nil {
-			logger.Log.Debugf("[ERROR] failed connecting to api server %v", err)
-		} else if response.StatusCode != 200 {
-			responseBody := ""
-			data, readErr := ioutil.ReadAll(response.Body)
-			if readErr == nil {
-				responseBody = string(data)
-			}
-
-			logger.Log.Debugf("can't connect to api server yet, response status code: %v, body: %v", response.StatusCode, responseBody)
-
-			response.Body.Close()
-		} else {
-			logger.Log.Debugf("connection test to api server passed successfully")
-			break
-		}
-		retriesLeft -= 1
-		time.Sleep(time.Second)
-	}
-
-	if retriesLeft == 0 {
-		provider.isReady = false
-		return fmt.Errorf("couldn't reach the api server after %v retries", provider.retries)
-	}
-	provider.url = url
-	provider.isReady = true
-	return nil
-}
-
-func (provider *apiServerProvider) ReportTappedPods(pods []core.Pod) error {
-	if !provider.isReady {
-		return fmt.Errorf("trying to reach api server when not initialized yet")
-	}
-	tappedPodsUrl := fmt.Sprintf("%s/status/tappedPods", provider.url)
-
-	podInfos := make([]shared.PodInfo, 0)
-	for _, pod := range pods {
-		podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
-	}
-	tapStatus := shared.TapStatus{Pods: podInfos}
-
-	if jsonValue, err := json.Marshal(tapStatus); err != nil {
-		return fmt.Errorf("failed Marshal the tapped pods %w", err)
-	} else {
-		if response, err := http.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
-			return fmt.Errorf("failed sending to API server the tapped pods %w", err)
-		} else if response.StatusCode != 200 {
-			return fmt.Errorf("failed sending to API server the tapped pods, response status code %v", response.StatusCode)
-		} else {
-			logger.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos))
-			return nil
-		}
-	}
-}
-
-func (provider *apiServerProvider) GetGeneralStats() (map[string]interface{}, error) {
-	if !provider.isReady {
-		return nil, fmt.Errorf("trying to reach api server when not initialized yet")
-	}
-	generalStatsUrl := fmt.Sprintf("%s/status/general", provider.url)
-
-	response, requestErr := http.Get(generalStatsUrl)
-	if requestErr != nil {
-		return nil, fmt.Errorf("failed to get general stats for telemetry, err: %w", requestErr)
-	} else if response.StatusCode != 200 {
-		return nil, fmt.Errorf("failed to get general stats for telemetry, status code: %v", response.StatusCode)
-	}
-
-	defer func() { _ = response.Body.Close() }()
-
-	data, readErr := ioutil.ReadAll(response.Body)
-	if readErr != nil {
-		return nil, fmt.Errorf("failed to read general stats for telemetry, err: %v", readErr)
-	}
-
-	var generalStats map[string]interface{}
-	if parseErr := json.Unmarshal(data, &generalStats); parseErr != nil {
-		return nil, fmt.Errorf("failed to parse general stats for telemetry, err: %v", parseErr)
-	}
-	return generalStats, nil
-}
-
-func (provider *apiServerProvider) GetVersion() (string, error) {
-	if !provider.isReady {
-		return "", fmt.Errorf("trying to reach api server when not initialized yet")
-	}
-	versionUrl, _ := url.Parse(fmt.Sprintf("%s/metadata/version", provider.url))
-	req := &http.Request{
-		Method: http.MethodGet,
-		URL:    versionUrl,
-	}
-	statusResp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer statusResp.Body.Close()
-
-	versionResponse := &shared.VersionResponse{}
-	if err := json.NewDecoder(statusResp.Body).Decode(&versionResponse); err != nil {
-		return "", err
-	}
-
-	return versionResponse.SemVer, nil
-}

cli/auth/authProvider.go

@@ -1,159 +0,0 @@
-package auth
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"net"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/logger"
-	"golang.org/x/oauth2"
-)
-
-const loginTimeoutInMin = 2
-
-// Ports are configured in keycloak "cli" client as valid redirect URIs. A change here must be reflected there as well.
-var listenPorts = []int{3141, 4001, 5002, 6003, 7004, 8005, 9006, 10007}
-
-func Login() error {
-	token, loginErr := loginInteractively()
-	if loginErr != nil {
-		return fmt.Errorf("failed login interactively, err: %v", loginErr)
-	}
-
-	authConfig := configStructs.AuthConfig{
-		EnvName: config.Config.Auth.EnvName,
-		Token:   token.AccessToken,
-	}
-
-	configFile, defaultConfigErr := config.GetConfigWithDefaults()
-	if defaultConfigErr != nil {
-		return fmt.Errorf("failed getting config with defaults, err: %v", defaultConfigErr)
-	}
-
-	if err := config.LoadConfigFile(config.Config.ConfigFilePath, configFile); err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("failed getting config file, err: %v", err)
-	}
-
-	configFile.Auth = authConfig
-
-	if err := config.WriteConfig(configFile); err != nil {
-		return fmt.Errorf("failed writing config with auth, err: %v", err)
-	}
-
-	config.Config.Auth = authConfig
-
-	logger.Log.Infof("Login successfully, token stored in config path: %s", fmt.Sprintf(uiUtils.Purple, config.Config.ConfigFilePath))
-	return nil
-}
-
-func loginInteractively() (*oauth2.Token, error) {
-	tokenChannel := make(chan *oauth2.Token)
-	errorChannel := make(chan error)
-
-	server := http.Server{}
-	go startLoginServer(tokenChannel, errorChannel, &server)
-
-	defer func() {
-		if err := server.Shutdown(context.Background()); err != nil {
-			logger.Log.Debugf("Error shutting down server, err: %v", err)
-		}
-	}()
-
-	select {
-	case <-time.After(loginTimeoutInMin * time.Minute):
-		return nil, errors.New("auth timed out")
-	case err := <-errorChannel:
-		return nil, err
-	case token := <-tokenChannel:
-		return token, nil
-	}
-}
-
-func startLoginServer(tokenChannel chan *oauth2.Token, errorChannel chan error, server *http.Server) {
-	for _, port := range listenPorts {
-		var authConfig = &oauth2.Config{
-			ClientID:    "cli",
-			RedirectURL: fmt.Sprintf("http://localhost:%v/callback", port),
-			Endpoint: oauth2.Endpoint{
-				AuthURL:  fmt.Sprintf("https://auth.%s/auth/realms/testr/protocol/openid-connect/auth", config.Config.Auth.EnvName),
-				TokenURL: fmt.Sprintf("https://auth.%s/auth/realms/testr/protocol/openid-connect/token", config.Config.Auth.EnvName),
-			},
-		}
-
-		state := uuid.New()
-
-		mux := http.NewServeMux()
-		server.Handler = mux
-		mux.Handle("/callback", loginCallbackHandler(tokenChannel, errorChannel, authConfig, state))
-
-		listener, listenErr := net.Listen("tcp", fmt.Sprintf("%s:%d", "127.0.0.1", port))
-		if listenErr != nil {
-			logger.Log.Debugf("failed to start listening on port %v, err: %v", port, listenErr)
-			continue
-		}
-
-		authorizationUrl := authConfig.AuthCodeURL(state.String())
-		uiUtils.OpenBrowser(authorizationUrl)
-
-		serveErr := server.Serve(listener)
-		if serveErr == http.ErrServerClosed {
-			logger.Log.Debugf("received server shutdown, server on port %v is closed", port)
-			return
-		} else if serveErr != nil {
-			logger.Log.Debugf("failed to start serving on port %v, err: %v", port, serveErr)
-			continue
-		}
-
-		logger.Log.Debugf("didn't receive server closed on port %v", port)
-		return
-	}
-
-	errorChannel <- fmt.Errorf("failed to start serving on all listen ports, ports: %v", listenPorts)
-}
-
-func loginCallbackHandler(tokenChannel chan *oauth2.Token, errorChannel chan error, authConfig *oauth2.Config, state uuid.UUID) http.Handler {
-	return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
-		if err := request.ParseForm(); err != nil {
-			errorMsg := fmt.Sprintf("failed to parse form, err: %v", err)
-			http.Error(writer, errorMsg, http.StatusBadRequest)
-			errorChannel <- fmt.Errorf(errorMsg)
-			return
-		}
-
-		requestState := request.Form.Get("state")
-		if requestState != state.String() {
-			errorMsg := fmt.Sprintf("state invalid, requestState: %v, authState:%v", requestState, state.String())
-			http.Error(writer, errorMsg, http.StatusBadRequest)
-			errorChannel <- fmt.Errorf(errorMsg)
-			return
-		}
-
-		code := request.Form.Get("code")
-		if code == "" {
-			errorMsg := "code not found"
-			http.Error(writer, errorMsg, http.StatusBadRequest)
-			errorChannel <- fmt.Errorf(errorMsg)
-			return
-		}
-
-		token, err := authConfig.Exchange(context.Background(), code)
-		if err != nil {
-			errorMsg := fmt.Sprintf("failed to create token, err: %v", err)
-			http.Error(writer, errorMsg, http.StatusInternalServerError)
-			errorChannel <- fmt.Errorf(errorMsg)
-			return
-		}
-
-		tokenChannel <- token
-
-		http.Redirect(writer, request, fmt.Sprintf("https://%s/CliLogin", config.Config.Auth.EnvName), http.StatusFound)
-	})
-}

cli/cmd/clean.go

@@ -1,20 +0,0 @@
-package cmd
-
-import (
-	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/telemetry"
-)
-
-var cleanCmd = &cobra.Command{
-	Use:   "clean",
-	Short: "Removes all mizu resources",
-	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("clean", nil)
-		performCleanCommand()
-		return nil
-	},
-}
-
-func init() {
-	rootCmd.AddCommand(cleanCmd)
-}

cli/cmd/cleanRunner.go

@@ -1,17 +0,0 @@
-package cmd
-
-import (
-	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/cli/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-func performCleanCommand() {
-	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
-	if err != nil {
-		logger.Log.Error(err)
-		return
-	}
-
-	finishMizuExecution(kubernetesProvider)
-}

cli/cmd/common.go

@@ -1,48 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/signal"
-	"syscall"
-
-	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/errormessage"
-	"github.com/up9inc/mizu/cli/kubernetes"
-	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-func GetApiServerUrl() string {
-	return fmt.Sprintf("http://%s", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort))
-}
-
-func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
-	err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
-	if err != nil {
-		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+
-			"Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName))
-		cancel()
-	}
-
-	logger.Log.Debugf("proxy ended")
-}
-
-func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
-	logger.Log.Debugf("waiting for finish...")
-	sigChan := make(chan os.Signal, 1)
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
-
-	// block until ctx cancel is called or termination signal is received
-	select {
-	case <-ctx.Done():
-		logger.Log.Debugf("ctx done")
-		break
-	case <-sigChan:
-		logger.Log.Debugf("Got termination signal, canceling execution...")
-		cancel()
-	}
-}

cli/cmd/config.go

@@ -2,14 +2,14 @@ package cmd
 
 import (
 	"fmt"
-
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/logger"
+	"io/ioutil"
 )
 
 var configCmd = &cobra.Command{
@@ -18,30 +18,22 @@ var configCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		go telemetry.ReportRun("config", config.Config.Config)
 
-		configWithDefaults, err := config.GetConfigWithDefaults()
+		template, err := config.GetConfigWithDefaults()
 		if err != nil {
-			logger.Log.Errorf("Failed generating config with defaults, err: %v", err)
+			logger.Log.Errorf("Failed generating config with defaults %v", err)
 			return nil
 		}
-
 		if config.Config.Config.Regenerate {
-			if err := config.WriteConfig(configWithDefaults); err != nil {
-				logger.Log.Errorf("Failed writing config with defaults, err: %v", err)
+			data := []byte(template)
+			if err := ioutil.WriteFile(config.GetConfigFilePath(), data, 0644); err != nil {
+				logger.Log.Errorf("Failed writing config %v", err)
 				return nil
 			}
-
-			logger.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, config.Config.ConfigFilePath)))
+			logger.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, config.GetConfigFilePath())))
 		} else {
-			template, err := uiUtils.PrettyYaml(configWithDefaults)
-			if err != nil {
-				logger.Log.Errorf("Failed converting config with defaults to yaml, err: %v", err)
-				return nil
-			}
-
 			logger.Log.Debugf("Writing template config.\n%v", template)
 			fmt.Printf("%v", template)
 		}
-
 		return nil
 	},
 }
@@ -49,8 +41,8 @@ var configCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(configCmd)
 
-	defaultConfig := config.ConfigStruct{}
-	defaults.Set(&defaultConfig)
+	defaultConfigConfig := configStructs.ConfigConfig{}
+	defaults.Set(&defaultConfigConfig)
 
-	configCmd.Flags().BoolP(configStructs.RegenerateConfigName, "r", defaultConfig.Config.Regenerate, fmt.Sprintf("Regenerate the config file with default values to path %s or to chosen path using --%s", defaultConfig.ConfigFilePath, config.ConfigFilePathCommandName))
+	configCmd.Flags().BoolP(configStructs.RegenerateConfigName, "r", defaultConfigConfig.Regenerate, fmt.Sprintf("Regenerate the config file with default values %s", config.GetConfigFilePath()))
 }

cli/cmd/fetch.go

@@ -0,0 +1,38 @@
+package cmd
+
+import (
+	"github.com/creasty/defaults"
+	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/mizu/version"
+	"github.com/up9inc/mizu/cli/telemetry"
+)
+
+var fetchCmd = &cobra.Command{
+	Use:   "fetch",
+	Short: "Download recorded traffic to files",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("fetch", config.Config.Fetch)
+
+		if isCompatible, err := version.CheckVersionCompatibility(config.Config.Fetch.GuiPort); err != nil {
+			return err
+		} else if !isCompatible {
+			return nil
+		}
+		RunMizuFetch()
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(fetchCmd)
+
+	defaultFetchConfig := configStructs.FetchConfig{}
+	defaults.Set(&defaultFetchConfig)
+
+	fetchCmd.Flags().StringP(configStructs.DirectoryFetchName, "d", defaultFetchConfig.Directory, "Provide a custom directory for fetched entries")
+	fetchCmd.Flags().Int(configStructs.FromTimestampFetchName, defaultFetchConfig.FromTimestamp, "Custom start timestamp for fetched entries")
+	fetchCmd.Flags().Int(configStructs.ToTimestampFetchName, defaultFetchConfig.ToTimestamp, "Custom end timestamp fetched entries")
+	fetchCmd.Flags().Uint16P(configStructs.GuiPortFetchName, "p", defaultFetchConfig.GuiPort, "Provide a custom port for the web interface webserver")
+}

cli/cmd/fetchRunner.go

@@ -0,0 +1,96 @@
+package cmd
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func RunMizuFetch() {
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Fetch.GuiPort)
+	resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, config.Config.Fetch.FromTimestamp, config.Config.Fetch.ToTimestamp))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	defer func() { _ = resp.Body.Close() }()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	_ = Unzip(zipReader, config.Config.Fetch.Directory)
+}
+
+func Unzip(reader *zip.Reader, dest string) error {
+	dest, _ = filepath.Abs(dest)
+	_ = os.MkdirAll(dest, os.ModePerm)
+
+	// Closure to address file descriptors issue with all the deferred .Close() methods
+	extractAndWriteFile := func(f *zip.File) error {
+		rc, err := f.Open()
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if err := rc.Close(); err != nil {
+				panic(err)
+			}
+		}()
+
+		path := filepath.Join(dest, f.Name)
+
+		// Check for ZipSlip (Directory traversal)
+		if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) {
+			return fmt.Errorf("illegal file path: %s", path)
+		}
+
+		if f.FileInfo().IsDir() {
+			_ = os.MkdirAll(path, f.Mode())
+		} else {
+			_ = os.MkdirAll(filepath.Dir(path), f.Mode())
+			logger.Log.Infof("writing HAR file [ %v ]", path)
+			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				if err := f.Close(); err != nil {
+					panic(err)
+				}
+				logger.Log.Info(" done")
+			}()
+
+			_, err = io.Copy(f, rc)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	for _, f := range reader.File {
+		err := extractAndWriteFile(f)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cli/cmd/logs.go

@@ -2,16 +2,15 @@ package cmd
 
 import (
 	"context"
-
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/telemetry"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 var logsCmd = &cobra.Command{
@@ -33,7 +32,7 @@ var logsCmd = &cobra.Command{
 
 		logger.Log.Debugf("Using file path %s", config.Config.Logs.FilePath())
 
-		if dumpLogsErr := fsUtils.DumpLogs(ctx, kubernetesProvider, config.Config.Logs.FilePath()); dumpLogsErr != nil {
+		if dumpLogsErr := fsUtils.DumpLogs(kubernetesProvider, ctx, config.Config.Logs.FilePath()); dumpLogsErr != nil {
 			logger.Log.Errorf("Failed dump logs %v", dumpLogsErr)
 		}
 

cli/cmd/root.go

@@ -2,16 +2,13 @@ package cmd
 
 import (
 	"fmt"
-	"time"
-
-	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/mizu/version"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 var rootCmd = &cobra.Command{
@@ -28,20 +25,13 @@ Further info is available at https://github.com/up9inc/mizu`,
 }
 
 func init() {
-	defaultConfig := config.ConfigStruct{}
-	defaults.Set(&defaultConfig)
-
 	rootCmd.PersistentFlags().StringSlice(config.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", config.SetCommandName))
-	rootCmd.PersistentFlags().String(config.ConfigFilePathCommandName, defaultConfig.ConfigFilePath, fmt.Sprintf("Override config file path using --%s", config.ConfigFilePathCommandName))
 }
 
 func printNewVersionIfNeeded(versionChan chan string) {
-	select {
-	case versionMsg := <-versionChan:
-		if versionMsg != "" {
-			logger.Log.Infof(uiUtils.Yellow, versionMsg)
-		}
-	case <-time.After(2 * time.Second):
+	versionMsg := <-versionChan
+	if versionMsg != "" {
+		logger.Log.Infof(uiUtils.Yellow, versionMsg)
 	}
 }
 
@@ -51,7 +41,7 @@ func Execute() {
 	if err := fsUtils.EnsureDir(mizu.GetMizuFolderPath()); err != nil {
 		logger.Log.Errorf("Failed to use mizu folder, %v", err)
 	}
-	logger.InitLogger(fsUtils.GetLogFilePath())
+	logger.InitLogger()
 
 	versionChan := make(chan string)
 	defer printNewVersionIfNeeded(versionChan)

cli/cmd/tap.go

@@ -2,22 +2,19 @@ package cmd
 
 import (
 	"errors"
-	"fmt"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"os"
 
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/auth"
-	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
-const uploadTrafficMessageToConfirm = `NOTE: running mizu with --%s flag will upload recorded traffic for further analysis and enriched presentation options.`
+const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic for further analysis and enriched presentation options.`
 
 var tapCmd = &cobra.Command{
 	Use:   "tap [POD REGEX]",
@@ -40,52 +37,20 @@ Supported protocols are HTTP and gRPC.`,
 			return errormessage.FormatError(err)
 		}
 
-		if config.Config.Tap.Workspace != "" {
-			askConfirmation(configStructs.WorkspaceTapName)
+		logger.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", config.Config.Tap.HumanMaxEntriesDBSize)
 
-			if config.Config.Auth.Token == "" {
-				logger.Log.Infof("This action requires authentication, please log in to continue")
-				if err := auth.Login(); err != nil {
-					logger.Log.Errorf("failed to log in, err: %v", err)
-					return nil
-				}
-			} else {
-				tokenExpired, err := shared.IsTokenExpired(config.Config.Auth.Token)
-				if err != nil {
-					logger.Log.Errorf("failed to check if token is expired, err: %v", err)
-					return nil
-				}
-
-				if tokenExpired {
-					logger.Log.Infof("Token expired, please log in again to continue")
-					if err := auth.Login(); err != nil {
-						logger.Log.Errorf("failed to log in, err: %v", err)
-						return nil
-					}
-				}
+		if config.Config.Tap.Analysis {
+			logger.Log.Infof(analysisMessageToConfirm)
+			if !uiUtils.AskForConfirmation("Would you like to proceed [Y/n]: ") {
+				logger.Log.Infof("You can always run mizu without analysis, aborting")
+				os.Exit(0)
 			}
 		}
 
-		if config.Config.Tap.Analysis {
-			askConfirmation(configStructs.AnalysisTapName)
-
-			config.Config.Auth.Token = ""
-		}
-
-		logger.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", config.Config.Tap.HumanMaxEntriesDBSize)
-
 		return nil
 	},
 }
 
-func askConfirmation(flagName string) {
-	logger.Log.Infof(fmt.Sprintf(uploadTrafficMessageToConfirm, flagName))
-	if !uiUtils.AskForConfirmation("Would you like to proceed [Y/n]: ") {
-		logger.Log.Infof("You can always run mizu without %s, aborting", flagName)
-		os.Exit(0)
-	}
-}
-
 func init() {
 	rootCmd.AddCommand(tapCmd)
 
@@ -99,8 +64,7 @@ func init() {
 	tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
 	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
+	tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any")
 	tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
-	tapCmd.Flags().StringP(configStructs.WorkspaceTapName, "w", defaultTapConfig.Workspace, "Uploads traffic to your UP9 workspace for further analysis (requires auth)")
-	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules")
-	tapCmd.Flags().String(configStructs.ContractFile, defaultTapConfig.ContractFile, "OAS/Swagger file to validate to monitor the contracts")
+	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file with policy rules")
 }

cli/cmd/tapRunner.go

@@ -1,37 +1,35 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
-	"errors"
+	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"path"
-	"regexp"
-	"strings"
-	"time"
-
-	"gopkg.in/yaml.v3"
-	core "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
-
-	"github.com/getkin/kin-openapi/openapi3"
-	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/errormessage"
-
-	"github.com/up9inc/mizu/cli/kubernetes"
-	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/mizu/goUtils"
 	"github.com/up9inc/mizu/cli/telemetry"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"path"
+	"regexp"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/up9inc/mizu/cli/errormessage"
+	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/tap/api"
+	yaml "gopkg.in/yaml.v3"
+	core "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 const (
@@ -43,6 +41,7 @@ type tapState struct {
 	apiServerService         *core.Service
 	currentlyTappedPods      []core.Pod
 	mizuServiceAccountExists bool
+	doNotRemoveConfigMap     bool
 }
 
 var state tapState
@@ -53,7 +52,6 @@ func RunMizuTap() {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error parsing regex-masking: %v", errormessage.FormatError(err)))
 		return
 	}
-
 	var mizuValidationRules string
 	if config.Config.Tap.EnforcePolicyFile != "" {
 		mizuValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile)
@@ -63,30 +61,6 @@ func RunMizuTap() {
 		}
 	}
 
-	// Read and validate the OAS file
-	var contract string
-	if config.Config.Tap.ContractFile != "" {
-		bytes, err := ioutil.ReadFile(config.Config.Tap.ContractFile)
-		if err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading contract file: %v", errormessage.FormatError(err)))
-			return
-		}
-		contract = string(bytes)
-
-		ctx := context.Background()
-		loader := &openapi3.Loader{Context: ctx}
-		doc, err := loader.LoadFromData(bytes)
-		if err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error loading contract file: %v", errormessage.FormatError(err)))
-			return
-		}
-		err = doc.Validate(ctx)
-		if err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error validating contract file: %v", errormessage.FormatError(err)))
-			return
-		}
-	}
-
 	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
 	if err != nil {
 		logger.Log.Error(err)
@@ -99,7 +73,7 @@ func RunMizuTap() {
 	targetNamespaces := getNamespaces(kubernetesProvider)
 
 	if config.Config.IsNsRestrictedMode() {
-		if len(targetNamespaces) != 1 || !shared.Contains(targetNamespaces, config.Config.MizuResourcesNamespace) {
+		if len(targetNamespaces) != 1 || !mizu.Contains(targetNamespaces, config.Config.MizuResourcesNamespace) {
 			logger.Log.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+
 				"You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, config.MizuResourcesNamespaceConfigName)
 			return
@@ -107,7 +81,7 @@ func RunMizuTap() {
 	}
 
 	var namespacesStr string
-	if !shared.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
+	if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
 		namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(targetNamespaces, "\", \""))
 	} else {
 		namespacesStr = "all namespaces"
@@ -122,7 +96,7 @@ func RunMizuTap() {
 
 	if len(state.currentlyTappedPods) == 0 {
 		var suggestionStr string
-		if !shared.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
+		if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
 			suggestionStr = ". Select a different namespace with -n or tap all namespaces with -A"
 		}
 		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr))
@@ -132,24 +106,18 @@ func RunMizuTap() {
 		return
 	}
 
-	if err := createMizuResources(ctx, kubernetesProvider, mizuValidationRules, contract); err != nil {
-		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
+	nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
 
-		var statusError *k8serrors.StatusError
-		if errors.As(err, &statusError) {
-			if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists {
-				logger.Log.Info("Mizu is already running in this namespace, change the `mizu-resources-namespace` configuration or run `mizu clean` to remove the currently running Mizu instance")
-			}
-		}
+	defer cleanUpMizu(kubernetesProvider)
+	if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
 		return
 	}
-	defer finishMizuExecution(kubernetesProvider)
 
-	go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel, mizuApiFilteringOptions)
-	go goUtils.HandleExcWrapper(watchTapperPod, ctx, kubernetesProvider, cancel)
-	go goUtils.HandleExcWrapper(watchPodsForTapping, ctx, kubernetesProvider, targetNamespaces, cancel, mizuApiFilteringOptions)
+	go goUtils.HandleExcWrapper(createProxyToApiServerPod, ctx, kubernetesProvider, cancel)
+	go goUtils.HandleExcWrapper(watchPodsForTapping, ctx, kubernetesProvider, targetNamespaces, cancel)
 
-	// block until exit signal or error
+	//block until exit signal or error
 	waitForFinish(ctx, cancel)
 }
 
@@ -162,26 +130,33 @@ func readValidationRules(file string) (string, error) {
 	return string(newContent), nil
 }
 
-func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuValidationRules string, contract string) error {
+func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, mizuApiFilteringOptions *shared.TrafficFilteringOptions, mizuValidationRules string) error {
 	if !config.Config.IsNsRestrictedMode() {
 		if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
 			return err
 		}
 	}
 
-	if err := createMizuApiServer(ctx, kubernetesProvider); err != nil {
+	if err := createMizuApiServer(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
 		return err
 	}
 
-	if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules, contract); err != nil {
+	if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
+		return err
+	}
+
+	if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules); err != nil {
 		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err)))
+		state.doNotRemoveConfigMap = true
+	} else if mizuValidationRules == "" {
+		state.doNotRemoveConfigMap = true
 	}
 
 	return nil
 }
 
-func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, data string, contract string) error {
-	err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName, data, contract)
+func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, data string) error {
+	err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName, data)
 	return err
 }
 
@@ -190,7 +165,7 @@ func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	return err
 }
 
-func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider) error {
+func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
 	var err error
 
 	state.mizuServiceAccountExists, err = createRBACIfNecessary(ctx, kubernetesProvider)
@@ -206,15 +181,15 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	}
 
 	opts := &kubernetes.ApiServerOptions{
-		Namespace:             config.Config.MizuResourcesNamespace,
-		PodName:               mizu.ApiServerPodName,
-		PodImage:              config.Config.AgentImage,
-		ServiceAccountName:    serviceAccountName,
-		IsNamespaceRestricted: config.Config.IsNsRestrictedMode(),
-		SyncEntriesConfig:     getSyncEntriesConfig(),
-		MaxEntriesDBSizeBytes: config.Config.Tap.MaxEntriesDBSizeBytes(),
-		Resources:             config.Config.Tap.ApiServerResources,
-		ImagePullPolicy:       config.Config.ImagePullPolicy(),
+		Namespace:               config.Config.MizuResourcesNamespace,
+		PodName:                 mizu.ApiServerPodName,
+		PodImage:                config.Config.AgentImage,
+		ServiceAccountName:      serviceAccountName,
+		IsNamespaceRestricted:   config.Config.IsNsRestrictedMode(),
+		MizuApiFilteringOptions: mizuApiFilteringOptions,
+		MaxEntriesDBSizeBytes:   config.Config.Tap.MaxEntriesDBSizeBytes(),
+		Resources:               config.Config.Tap.ApiServerResources,
+		ImagePullPolicy:         config.Config.ImagePullPolicy(),
 	}
 	_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, opts)
 	if err != nil {
@@ -231,13 +206,13 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	return nil
 }
 
-func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
-	var compiledRegexSlice []*api.SerializableRegexp
+func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
+	var compiledRegexSlice []*shared.SerializableRegexp
 
 	if config.Config.Tap.PlainTextFilterRegexes != nil && len(config.Config.Tap.PlainTextFilterRegexes) > 0 {
-		compiledRegexSlice = make([]*api.SerializableRegexp, 0)
+		compiledRegexSlice = make([]*shared.SerializableRegexp, 0)
 		for _, regexStr := range config.Config.Tap.PlainTextFilterRegexes {
-			compiledRegex, err := api.CompileRegexToSerializableRegexp(regexStr)
+			compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr)
 			if err != nil {
 				return nil, err
 			}
@@ -245,29 +220,14 @@ func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
 		}
 	}
 
-	return &api.TrafficFilteringOptions{
-		PlainTextMaskingRegexes: compiledRegexSlice,
-		IgnoredUserAgents:       config.Config.Tap.IgnoredUserAgents,
-		DisableRedaction:        config.Config.Tap.DisableRedaction,
+	return &shared.TrafficFilteringOptions{
+		PlainTextMaskingRegexes:      compiledRegexSlice,
+		HealthChecksUserAgentHeaders: config.Config.Tap.HealthChecksUserAgentHeaders,
+		DisableRedaction:             config.Config.Tap.DisableRedaction,
 	}, nil
 }
 
-func getSyncEntriesConfig() *shared.SyncEntriesConfig {
-	if !config.Config.Tap.Analysis && config.Config.Tap.Workspace == "" {
-		return nil
-	}
-
-	return &shared.SyncEntriesConfig{
-		Token:             config.Config.Auth.Token,
-		Env:               config.Config.Auth.EnvName,
-		Workspace:         config.Config.Tap.Workspace,
-		UploadIntervalSec: config.Config.Tap.UploadIntervalSec,
-	}
-}
-
-func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *api.TrafficFilteringOptions) error {
-	nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
-
+func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error {
 	if len(nodeToTappedPodIPMap) > 0 {
 		var serviceAccountName string
 		if state.mizuServiceAccountExists {
@@ -285,9 +245,9 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
 			fmt.Sprintf("%s.%s.svc.cluster.local", state.apiServerService.Name, state.apiServerService.Namespace),
 			nodeToTappedPodIPMap,
 			serviceAccountName,
+			config.Config.Tap.TapOutgoing(),
 			config.Config.Tap.TapperResources,
 			config.Config.ImagePullPolicy(),
-			mizuApiFilteringOptions,
 		); err != nil {
 			return err
 		}
@@ -301,112 +261,76 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
 	return nil
 }
 
-func finishMizuExecution(kubernetesProvider *kubernetes.Provider) {
-	telemetry.ReportAPICalls()
+func cleanUpMizu(kubernetesProvider *kubernetes.Provider) {
+	telemetry.ReportAPICalls(config.Config.Tap.GuiPort)
+	cleanUpMizuResources(kubernetesProvider)
+}
+
+func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
 	removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
 	defer cancel()
-	dumpLogsIfNeeded(removalCtx, kubernetesProvider)
-	cleanUpMizuResources(removalCtx, cancel, kubernetesProvider)
-}
 
-func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provider) {
-	if !config.Config.DumpLogs {
-		return
+	if config.Config.DumpLogs {
+		mizuDir := mizu.GetMizuFolderPath()
+		filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
+		if err := fsUtils.DumpLogs(kubernetesProvider, removalCtx, filePath); err != nil {
+			logger.Log.Errorf("Failed dump logs %v", err)
+		}
 	}
-	mizuDir := mizu.GetMizuFolderPath()
-	filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
-	if err := fsUtils.DumpLogs(ctx, kubernetesProvider, filePath); err != nil {
-		logger.Log.Errorf("Failed dump logs %v", err)
-	}
-}
 
-func cleanUpMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
 	logger.Log.Infof("\nRemoving mizu resources\n")
 
-	var leftoverResources []string
-
-	if config.Config.IsNsRestrictedMode() {
-		leftoverResources = cleanUpRestrictedMode(ctx, kubernetesProvider)
-	} else {
-		leftoverResources = cleanUpNonRestrictedMode(ctx, cancel, kubernetesProvider)
-	}
-
-	if len(leftoverResources) > 0 {
-		errMsg := fmt.Sprintf("Failed to remove the following resources, for more info check logs at %s:", fsUtils.GetLogFilePath())
-		for _, resource := range leftoverResources {
-			errMsg += "\n- " + resource
+	if !config.Config.IsNsRestrictedMode() {
+		if err := kubernetesProvider.RemoveNamespace(removalCtx, config.Config.MizuResourcesNamespace); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Namespace %s: %v", config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			return
 		}
-		logger.Log.Errorf(uiUtils.Error, errMsg)
-	}
-}
-
-func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.Provider) []string {
-	leftoverResources := make([]string, 0)
-
-	if err := kubernetesProvider.RemovePod(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
-		resourceDesc := fmt.Sprintf("Pod %s in namespace %s", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveService(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
-		resourceDesc := fmt.Sprintf("Service %s in namespace %s", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveDaemonSet(ctx, config.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
-		resourceDesc := fmt.Sprintf("DaemonSet %s in namespace %s", mizu.TapperDaemonSetName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName); err != nil {
-		resourceDesc := fmt.Sprintf("ConfigMap %s in namespace %s", mizu.ConfigMapName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveServicAccount(ctx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName); err != nil {
-		resourceDesc := fmt.Sprintf("Service Account %s in namespace %s", mizu.ServiceAccountName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveRole(ctx, config.Config.MizuResourcesNamespace, mizu.RoleName); err != nil {
-		resourceDesc := fmt.Sprintf("Role %s in namespace %s", mizu.RoleName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveRoleBinding(ctx, config.Config.MizuResourcesNamespace, mizu.RoleBindingName); err != nil {
-		resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", mizu.RoleBindingName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	return leftoverResources
-}
-
-func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) []string {
-	leftoverResources := make([]string, 0)
-
-	if err := kubernetesProvider.RemoveNamespace(ctx, config.Config.MizuResourcesNamespace); err != nil {
-		resourceDesc := fmt.Sprintf("Namespace %s", config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
 	} else {
-		defer waitUntilNamespaceDeleted(ctx, cancel, kubernetesProvider)
+		if err := kubernetesProvider.RemovePod(removalCtx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Pod %s in namespace %s: %v", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+		}
+
+		if err := kubernetesProvider.RemoveService(removalCtx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service %s in namespace %s: %v", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+		}
+
+		if err := kubernetesProvider.RemoveDaemonSet(removalCtx, config.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing DaemonSet %s in namespace %s: %v", mizu.TapperDaemonSetName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+		}
+
+		if !state.doNotRemoveConfigMap {
+			if err := kubernetesProvider.RemoveConfigMap(removalCtx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing ConfigMap %s in namespace %s: %v", mizu.ConfigMapName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			}
+		}
+
 	}
 
-	if err := kubernetesProvider.RemoveClusterRole(ctx, mizu.ClusterRoleName); err != nil {
-		resourceDesc := fmt.Sprintf("ClusterRole %s", mizu.ClusterRoleName)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
+	if state.mizuServiceAccountExists {
+		if !config.Config.IsNsRestrictedMode() {
+			if err := kubernetesProvider.RemoveNonNamespacedResources(removalCtx, mizu.ClusterRoleName, mizu.ClusterRoleBindingName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing non-namespaced resources: %v", errormessage.FormatError(err)))
+				return
+			}
+		} else {
+			if err := kubernetesProvider.RemoveServicAccount(removalCtx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service Account %s in namespace %s: %v", mizu.ServiceAccountName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+				return
+			}
+
+			if err := kubernetesProvider.RemoveRole(removalCtx, config.Config.MizuResourcesNamespace, mizu.RoleName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Role %s in namespace %s: %v", mizu.RoleName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			}
+
+			if err := kubernetesProvider.RemoveRoleBinding(removalCtx, config.Config.MizuResourcesNamespace, mizu.RoleBindingName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing RoleBinding %s in namespace %s: %v", mizu.RoleBindingName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			}
+		}
 	}
 
-	if err := kubernetesProvider.RemoveClusterRoleBinding(ctx, mizu.ClusterRoleBindingName); err != nil {
-		resourceDesc := fmt.Sprintf("ClusterRoleBinding %s", mizu.ClusterRoleBindingName)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
+	if !config.Config.IsNsRestrictedMode() {
+		waitUntilNamespaceDeleted(removalCtx, cancel, kubernetesProvider)
 	}
-
-	return leftoverResources
-}
-
-func handleDeletionError(err error, resourceDesc string, leftoverResources *[]string) {
-	logger.Log.Debugf("Error removing %s: %v", resourceDesc, errormessage.FormatError(err))
-	*leftoverResources = append(*leftoverResources, resourceDesc)
 }
 
 func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
@@ -418,7 +342,7 @@ func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, k
 	if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, config.Config.MizuResourcesNamespace); err != nil {
 		switch {
 		case ctx.Err() == context.Canceled:
-			logger.Log.Debugf("Do nothing. User interrupted the wait")
+			// Do nothing. User interrupted the wait.
 		case err == wait.ErrWaitTimeout:
 			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", config.Config.MizuResourcesNamespace))
 		default:
@@ -427,7 +351,30 @@ func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, k
 	}
 }
 
-func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, targetNamespaces []string, cancel context.CancelFunc, mizuApiFilteringOptions *api.TrafficFilteringOptions) {
+func reportTappedPods() {
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort)
+	tappedPodsUrl := fmt.Sprintf("http://%s/status/tappedPods", mizuProxiedUrl)
+
+	podInfos := make([]shared.PodInfo, 0)
+	for _, pod := range state.currentlyTappedPods {
+		podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
+	}
+	tapStatus := shared.TapStatus{Pods: podInfos}
+
+	if jsonValue, err := json.Marshal(tapStatus); err != nil {
+		logger.Log.Debugf("[ERROR] failed Marshal the tapped pods %v", err)
+	} else {
+		if response, err := http.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+			logger.Log.Debugf("[ERROR] failed sending to API server the tapped pods %v", err)
+		} else if response.StatusCode != 200 {
+			logger.Log.Debugf("[ERROR] failed sending to API server the tapped pods, response status code %v", response.StatusCode)
+		} else {
+			logger.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos))
+		}
+	}
+}
+
+func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, targetNamespaces []string, cancel context.CancelFunc) {
 	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, targetNamespaces, config.Config.Tap.PodRegex())
 
 	restartTappers := func() {
@@ -442,12 +389,15 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 			return
 		}
 
-		if err := apiserver.Provider.ReportTappedPods(state.currentlyTappedPods); err != nil {
-			logger.Log.Debugf("[Error] failed update tapped pods %v", err)
-		}
+		reportTappedPods()
 
-		if err := updateMizuTappers(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating tappers: %v", errormessage.FormatError(err)))
+		nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
+		if err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error building node to ips map: %v", errormessage.FormatError(err)))
+			cancel()
+		}
+		if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating daemonset: %v", errormessage.FormatError(err)))
 			cancel()
 		}
 	}
@@ -455,28 +405,13 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 
 	for {
 		select {
-		case pod, ok := <-added:
-			if !ok {
-				added = nil
-				continue
-			}
-
+		case pod := <-added:
 			logger.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
 			restartTappersDebouncer.SetOn()
-		case pod, ok := <-removed:
-			if !ok {
-				removed = nil
-				continue
-			}
-
+		case pod := <-removed:
 			logger.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
 			restartTappersDebouncer.SetOn()
-		case pod, ok := <-modified:
-			if !ok {
-				modified = nil
-				continue
-			}
-
+		case pod := <-modified:
 			logger.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
 			// Act only if the modified pod has already obtained an IP address.
 			// After filtering for IPs, on a normal pod restart this includes the following events:
@@ -487,12 +422,8 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 			if pod.Status.PodIP != "" {
 				restartTappersDebouncer.SetOn()
 			}
-		case err, ok := <-errorChan:
-			if !ok {
-				errorChan = nil
-				continue
-			}
 
+		case err := <-errorChan:
 			logger.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err)
 			restartTappersDebouncer.Cancel()
 			// TODO: Does this also perform cleanup?
@@ -565,158 +496,77 @@ func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod {
 	return missingPods
 }
 
-func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, mizuApiFilteringOptions *api.TrafficFilteringOptions) {
+func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName))
 	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
 	isPodReady := false
 	timeAfter := time.After(25 * time.Second)
 	for {
 		select {
-		case _, ok := <-added:
-			if !ok {
-				added = nil
-				continue
-			}
-
+		case <-ctx.Done():
+			logger.Log.Debugf("Watching API Server pod loop, ctx done")
+			return
+		case <-added:
 			logger.Log.Debugf("Watching API Server pod loop, added")
-		case _, ok := <-removed:
-			if !ok {
-				removed = nil
-				continue
-			}
-
+			continue
+		case <-removed:
 			logger.Log.Infof("%s removed", mizu.ApiServerPodName)
 			cancel()
 			return
-		case modifiedPod, ok := <-modified:
-			if !ok {
-				modified = nil
+		case modifiedPod := <-modified:
+			if modifiedPod == nil {
+				logger.Log.Debugf("Watching API Server pod loop, modifiedPod with nil")
 				continue
 			}
-
 			logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
-
-			if modifiedPod.Status.Phase == core.PodPending {
-				if modifiedPod.Status.Conditions[0].Type == core.PodScheduled && modifiedPod.Status.Conditions[0].Status != core.ConditionTrue {
-					logger.Log.Debugf("Wasn't able to deploy the API server. Reason: \"%s\"", modifiedPod.Status.Conditions[0].Message)
-					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Wasn't able to deploy the API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
-					cancel()
-					break
-				}
-
-				if len(modifiedPod.Status.ContainerStatuses) > 0 && modifiedPod.Status.ContainerStatuses[0].State.Waiting != nil && modifiedPod.Status.ContainerStatuses[0].State.Waiting.Reason == "ErrImagePull" {
-					logger.Log.Debugf("Wasn't able to deploy the API server. (ErrImagePull) Reason: \"%s\"", modifiedPod.Status.ContainerStatuses[0].State.Waiting.Message)
-					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Wasn't able to deploy the API server: failed to pull the image, for more info check logs at %v", fsUtils.GetLogFilePath()))
-					cancel()
-					break
-				}
-			}
-
 			if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
 				isPodReady = true
 				go startProxyReportErrorIfAny(kubernetesProvider, cancel)
-
-				url := GetApiServerUrl()
-				if err := apiserver.Provider.InitAndTestConnection(url); err != nil {
-					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
-					cancel()
-					break
-				}
-				if err := updateMizuTappers(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
-					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating tappers: %v", errormessage.FormatError(err)))
-					cancel()
-				}
-
-				logger.Log.Infof("Mizu is available at %s\n", url)
-				uiUtils.OpenBrowser(url)
-				if err := apiserver.Provider.ReportTappedPods(state.currentlyTappedPods); err != nil {
-					logger.Log.Debugf("[Error] failed update tapped pods %v", err)
-				}
+				logger.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort))
+				time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
+				requestForAnalysis()
+				reportTappedPods()
 			}
-		case err, ok := <-errorChan:
-			if !ok {
-				errorChan = nil
-				continue
-			}
-
-			logger.Log.Debugf("[ERROR] Agent creation, watching %v namespace, error: %v", config.Config.MizuResourcesNamespace, err)
-			cancel()
-
 		case <-timeAfter:
 			if !isPodReady {
 				logger.Log.Errorf(uiUtils.Error, "Mizu API server was not ready in time")
 				cancel()
 			}
-		case <-ctx.Done():
-			logger.Log.Debugf("Watching API Server pod loop, ctx done")
-			return
+		case <-errorChan:
+			logger.Log.Debugf("[ERROR] Agent creation, watching %v namespace", config.Config.MizuResourcesNamespace)
+			cancel()
 		}
 	}
 }
 
-func watchTapperPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
-	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", mizu.TapperDaemonSetName))
-	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
-	var prevPodPhase core.PodPhase
-	for {
-		select {
-		case addedPod, ok := <-added:
-			if !ok {
-				added = nil
-				continue
-			}
+func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+	err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
+	if err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+
+			"Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName))
+		cancel()
+	}
+}
 
-			logger.Log.Debugf("Tapper is created [%s]", addedPod.Name)
-		case removedPod, ok := <-removed:
-			if !ok {
-				removed = nil
-				continue
-			}
+func requestForAnalysis() {
+	if !config.Config.Tap.Analysis {
+		return
+	}
 
-			logger.Log.Debugf("Tapper is removed [%s]", removedPod.Name)
-		case modifiedPod, ok := <-modified:
-			if !ok {
-				modified = nil
-				continue
-			}
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort)
+	urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(config.Config.Tap.AnalysisDestination), config.Config.Tap.SleepIntervalSec)
+	u, parseErr := url.ParseRequestURI(urlPath)
+	if parseErr != nil {
+		logger.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr)
+	}
 
-			if modifiedPod.Status.Phase == core.PodPending && modifiedPod.Status.Conditions[0].Type == core.PodScheduled && modifiedPod.Status.Conditions[0].Status != core.ConditionTrue {
-				logger.Log.Infof(uiUtils.Red, fmt.Sprintf("Wasn't able to deploy the tapper %s. Reason: \"%s\"", modifiedPod.Name, modifiedPod.Status.Conditions[0].Message))
-				cancel()
-				break
-			}
-
-			podStatus := modifiedPod.Status
-			if podStatus.Phase == core.PodPending && prevPodPhase == podStatus.Phase {
-				logger.Log.Debugf("Tapper %s is %s", modifiedPod.Name, strings.ToLower(string(podStatus.Phase)))
-				continue
-			}
-			prevPodPhase = podStatus.Phase
-
-			if podStatus.Phase == core.PodRunning {
-				state := podStatus.ContainerStatuses[0].State
-				if state.Terminated != nil {
-					switch state.Terminated.Reason {
-					case "OOMKilled":
-						logger.Log.Infof(uiUtils.Red, fmt.Sprintf("Tapper %s was terminated (reason: OOMKilled). You should consider increasing machine resources.", modifiedPod.Name))
-					}
-				}
-			}
-
-			logger.Log.Debugf("Tapper %s is %s", modifiedPod.Name, strings.ToLower(string(podStatus.Phase)))
-		case err, ok := <-errorChan:
-			if !ok {
-				errorChan = nil
-				continue
-			}
-
-			logger.Log.Debugf("[Error] Error in mizu tapper watch, err: %v", err)
-			cancel()
-
-		case <-ctx.Done():
-			logger.Log.Debugf("Watching tapper pod loop, ctx done")
-			return
-		}
+	logger.Log.Debugf("Sending get request to %v", u.String())
+	if response, requestErr := http.Get(u.String()); requestErr != nil {
+		logger.Log.Errorf("Failed to notify agent for analysis, err: %v", requestErr)
+	} else if response.StatusCode != 200 {
+		logger.Log.Errorf("Failed to notify agent for analysis, status code: %v", response.StatusCode)
+	} else {
+		logger.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis")
 	}
 }
 
@@ -748,11 +598,24 @@ func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) map[string][]string {
 	return nodeToTappedPodIPMap
 }
 
+func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+
+	// block until ctx cancel is called or termination signal is received
+	select {
+	case <-ctx.Done():
+		break
+	case <-sigChan:
+		cancel()
+	}
+}
+
 func getNamespaces(kubernetesProvider *kubernetes.Provider) []string {
 	if config.Config.Tap.AllNamespaces {
 		return []string{mizu.K8sAllNamespaces}
 	} else if len(config.Config.Tap.Namespaces) > 0 {
-		return shared.Unique(config.Config.Tap.Namespaces)
+		return config.Config.Tap.Namespaces
 	} else {
 		return []string{kubernetesProvider.CurrentNamespace()}
 	}

cli/cmd/version.go

@@ -1,13 +1,12 @@
 package cmd
 
 import (
-	"strconv"
-	"time"
-
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/telemetry"
-	"github.com/up9inc/mizu/shared/logger"
+	"strconv"
+	"time"
 
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"

cli/cmd/view.go

@@ -25,7 +25,4 @@ func init() {
 	defaults.Set(&defaultViewConfig)
 
 	viewCmd.Flags().Uint16P(configStructs.GuiPortViewName, "p", defaultViewConfig.GuiPort, "Provide a custom port for the web interface webserver")
-	viewCmd.Flags().StringP(configStructs.UrlViewName, "u", defaultViewConfig.Url, "Provide a custom host")
-
-	viewCmd.Flags().MarkHidden(configStructs.UrlViewName)
 }

cli/cmd/viewRunner.go

@@ -3,16 +3,13 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"net/http"
-
-	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/mizu/version"
-	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/logger"
+	"net/http"
+	"time"
 )
 
 func runMizuView() {
@@ -25,42 +22,31 @@ func runMizuView() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	url := config.Config.View.Url
-
-	if url == "" {
-		exists, err := kubernetesProvider.DoesServicesExist(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
-		if err != nil {
-			logger.Log.Errorf("Failed to found mizu service %v", err)
-			cancel()
-			return
-		}
-		if !exists {
-			logger.Log.Infof("%s service not found, you should run `mizu tap` command first", mizu.ApiServerPodName)
-			cancel()
-			return
-		}
-
-		url = GetApiServerUrl()
-
-		response, err := http.Get(fmt.Sprintf("%s/", url))
-		if err == nil && response.StatusCode == 200 {
-			logger.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, config.Config.View.GuiPort)
-			return
-		}
-		logger.Log.Infof("Establishing connection to k8s cluster...")
-		go startProxyReportErrorIfAny(kubernetesProvider, cancel)
-
-		if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl()); err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
-			return
-		}
+	exists, err := kubernetesProvider.DoesServicesExist(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
+	if err != nil {
+		logger.Log.Errorf("Failed to found mizu service %v", err)
+		cancel()
+		return
+	}
+	if !exists {
+		logger.Log.Infof("%s service not found, you should run `mizu tap` command first", mizu.ApiServerPodName)
+		cancel()
+		return
 	}
 
-	logger.Log.Infof("Mizu is available at %s\n", url)
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.View.GuiPort)
+	_, err = http.Get(fmt.Sprintf("http://%s/", mizuProxiedUrl))
+	if err == nil {
+		logger.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, config.Config.View.GuiPort)
+		return
+	}
+	logger.Log.Infof("Establishing connection to k8s cluster...")
+	go startProxyReportErrorIfAny(kubernetesProvider, cancel)
 
-	uiUtils.OpenBrowser(url)
+	time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
 
-	if isCompatible, err := version.CheckVersionCompatibility(); err != nil {
+	logger.Log.Infof("Mizu is available at  http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.View.GuiPort))
+	if isCompatible, err := version.CheckVersionCompatibility(config.Config.View.GuiPort); err != nil {
 		logger.Log.Errorf("Failed to check versions compatibility %v", err)
 		cancel()
 		return

cli/config/config.go

@@ -3,15 +3,15 @@ package config
 import (
 	"errors"
 	"fmt"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu"
 	"io/ioutil"
 	"os"
+	"path"
 	"reflect"
 	"strconv"
 	"strings"
 
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -27,7 +27,7 @@ const (
 )
 
 var (
-	Config  = ConfigStruct{}
+	Config = ConfigStruct{}
 	cmdName string
 )
 
@@ -38,13 +38,9 @@ func InitConfig(cmd *cobra.Command) error {
 		return err
 	}
 
-	configFilePathFlag := cmd.Flags().Lookup(ConfigFilePathCommandName)
-	configFilePath := configFilePathFlag.Value.String()
-	if err := LoadConfigFile(configFilePath, &Config); err != nil {
-		if configFilePathFlag.Changed || !os.IsNotExist(err) {
-			return fmt.Errorf("invalid config, %w\n"+
-				"you can regenerate the file by removing it (%v) and using `mizu config -r`", err, configFilePath)
-		}
+	if err := mergeConfigFile(); err != nil {
+		return fmt.Errorf("invalid config, %w\n" +
+			"you can regenerate the file by removing it (%v) and using `mizu config -r`", err, GetConfigFilePath())
 	}
 
 	cmd.Flags().Visit(initFlag)
@@ -55,36 +51,26 @@ func InitConfig(cmd *cobra.Command) error {
 	return nil
 }
 
-func GetConfigWithDefaults() (*ConfigStruct, error) {
+func GetConfigWithDefaults() (string, error) {
 	defaultConf := ConfigStruct{}
 	if err := defaults.Set(&defaultConf); err != nil {
-		return nil, err
+		return "", err
 	}
 
 	configElem := reflect.ValueOf(&defaultConf).Elem()
 	setZeroForReadonlyFields(configElem)
 
-	return &defaultConf, nil
+	return uiUtils.PrettyYaml(defaultConf)
 }
 
-func WriteConfig(config *ConfigStruct) error {
-	template, err := uiUtils.PrettyYaml(config)
-	if err != nil {
-		return fmt.Errorf("failed converting config to yaml, err: %v", err)
-	}
-
-	data := []byte(template)
-	if err := ioutil.WriteFile(Config.ConfigFilePath, data, 0644); err != nil {
-		return fmt.Errorf("failed writing config, err: %v", err)
-	}
-
-	return nil
+func GetConfigFilePath() string {
+	return path.Join(mizu.GetMizuFolderPath(), "config.yaml")
 }
 
-func LoadConfigFile(configFilePath string, config *ConfigStruct) error {
-	reader, openErr := os.Open(configFilePath)
+func mergeConfigFile() error {
+	reader, openErr := os.Open(GetConfigFilePath())
 	if openErr != nil {
-		return openErr
+		return nil
 	}
 
 	buf, readErr := ioutil.ReadAll(reader)
@@ -92,11 +78,10 @@ func LoadConfigFile(configFilePath string, config *ConfigStruct) error {
 		return readErr
 	}
 
-	if err := yaml.Unmarshal(buf, config); err != nil {
+	if err := yaml.Unmarshal(buf, &Config); err != nil {
 		return err
 	}
-
-	logger.Log.Debugf("Found config file, config path: %s", configFilePath)
+	logger.Log.Debugf("Found config file, merged to default options")
 
 	return nil
 }
@@ -104,12 +89,7 @@ func LoadConfigFile(configFilePath string, config *ConfigStruct) error {
 func initFlag(f *pflag.Flag) {
 	configElemValue := reflect.ValueOf(&Config).Elem()
 
-	var flagPath []string
-	if shared.Contains([]string{ConfigFilePathCommandName}, f.Name) {
-		flagPath = []string{f.Name}
-	} else {
-		flagPath = []string{cmdName, f.Name}
-	}
+	flagPath := []string {cmdName, f.Name}
 
 	sliceValue, isSliceValue := f.Value.(pflag.SliceValue)
 	if !isSliceValue {

cli/config/configStruct.go

@@ -7,22 +7,19 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/util/homedir"
 	"os"
-	"path"
 	"path/filepath"
 )
 
 const (
 	MizuResourcesNamespaceConfigName = "mizu-resources-namespace"
-	ConfigFilePathCommandName        = "config-path"
-	KubeConfigPathConfigName         = "kube-config-path"
 )
 
 type ConfigStruct struct {
 	Tap                    configStructs.TapConfig     `yaml:"tap"`
+	Fetch                  configStructs.FetchConfig   `yaml:"fetch"`
 	Version                configStructs.VersionConfig `yaml:"version"`
 	View                   configStructs.ViewConfig    `yaml:"view"`
 	Logs                   configStructs.LogsConfig    `yaml:"logs"`
-	Auth                   configStructs.AuthConfig    `yaml:"auth"`
 	Config                 configStructs.ConfigConfig  `yaml:"config,omitempty"`
 	AgentImage             string                      `yaml:"agent-image,omitempty" readonly:""`
 	ImagePullPolicyStr     string                      `yaml:"image-pull-policy" default:"Always"`
@@ -30,12 +27,10 @@ type ConfigStruct struct {
 	Telemetry              bool                        `yaml:"telemetry" default:"true"`
 	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`
 	KubeConfigPathStr      string                      `yaml:"kube-config-path"`
-	ConfigFilePath         string                      `yaml:"config-path,omitempty" readonly:""`
 }
 
 func (config *ConfigStruct) SetDefaults() {
 	config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer)
-	config.ConfigFilePath = path.Join(mizu.GetMizuFolderPath(), "config.yaml")
 }
 
 func (config *ConfigStruct) ImagePullPolicy() v1.PullPolicy {

cli/config/configStructs/authConfig.go

@@ -1,6 +0,0 @@
-package configStructs
-
-type AuthConfig struct {
-	EnvName      string    `yaml:"env-name" default:"up9.app"`
-	Token        string    `yaml:"token"`
-}

cli/config/configStructs/fetchConfig.go

@@ -0,0 +1,15 @@
+package configStructs
+
+const (
+	DirectoryFetchName     = "directory"
+	FromTimestampFetchName = "from"
+	ToTimestampFetchName   = "to"
+	GuiPortFetchName       = "gui-port"
+)
+
+type FetchConfig struct {
+	Directory     string `yaml:"directory" default:"."`
+	FromTimestamp int    `yaml:"from" default:"0"`
+	ToTimestamp   int    `yaml:"to" default:"0"`
+	GuiPort       uint16 `yaml:"gui-port" default:"8899"`
+}

cli/config/configStructs/tapConfig.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"regexp"
+	"strings"
 
 	"github.com/up9inc/mizu/shared/units"
 )
@@ -16,29 +17,28 @@ const (
 	PlainTextFilterRegexesTapName = "regex-masking"
 	DisableRedactionTapName       = "no-redact"
 	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
+	DirectionTapName              = "direction"
 	DryRunTapName                 = "dry-run"
-	WorkspaceTapName              = "workspace"
-	EnforcePolicyFile             = "traffic-validation-file"
-	ContractFile                  = "contract"
+	EnforcePolicyFile             = "test-rules"
 )
 
 type TapConfig struct {
-	UploadIntervalSec      int       `yaml:"upload-interval" default:"10"`
-	PodRegexStr            string    `yaml:"regex" default:".*"`
-	GuiPort                uint16    `yaml:"gui-port" default:"8899"`
-	Namespaces             []string  `yaml:"namespaces"`
-	Analysis               bool      `yaml:"analysis" default:"false"`
-	AllNamespaces          bool      `yaml:"all-namespaces" default:"false"`
-	PlainTextFilterRegexes []string  `yaml:"regex-masking"`
-	IgnoredUserAgents      []string  `yaml:"ignored-user-agents"`
-	DisableRedaction       bool      `yaml:"no-redact" default:"false"`
-	HumanMaxEntriesDBSize  string    `yaml:"max-entries-db-size" default:"200MB"`
-	DryRun                 bool      `yaml:"dry-run" default:"false"`
-	Workspace              string    `yaml:"workspace"`
-	EnforcePolicyFile      string    `yaml:"traffic-validation-file"`
-	ContractFile           string    `yaml:"contract"`
-	ApiServerResources     Resources `yaml:"api-server-resources"`
-	TapperResources        Resources `yaml:"tapper-resources"`
+	AnalysisDestination          string    `yaml:"dest" default:"up9.app"`
+	SleepIntervalSec             int       `yaml:"upload-interval" default:"10"`
+	PodRegexStr                  string    `yaml:"regex" default:".*"`
+	GuiPort                      uint16    `yaml:"gui-port" default:"8899"`
+	Namespaces                   []string  `yaml:"namespaces"`
+	Analysis                     bool      `yaml:"analysis" default:"false"`
+	AllNamespaces                bool      `yaml:"all-namespaces" default:"false"`
+	PlainTextFilterRegexes       []string  `yaml:"regex-masking"`
+	HealthChecksUserAgentHeaders []string  `yaml:"ignored-user-agents"`
+	DisableRedaction             bool      `yaml:"no-redact" default:"false"`
+	HumanMaxEntriesDBSize        string    `yaml:"max-entries-db-size" default:"200MB"`
+	Direction                    string    `yaml:"direction" default:"in"`
+	DryRun                       bool      `yaml:"dry-run" default:"false"`
+	EnforcePolicyFile            string    `yaml:"test-rules"`
+	ApiServerResources           Resources `yaml:"api-server-resources"`
+	TapperResources              Resources `yaml:"tapper-resources"`
 }
 
 type Resources struct {
@@ -53,6 +53,15 @@ func (config *TapConfig) PodRegex() *regexp.Regexp {
 	return podRegex
 }
 
+func (config *TapConfig) TapOutgoing() bool {
+	directionLowerCase := strings.ToLower(config.Direction)
+	if directionLowerCase == "any" {
+		return true
+	}
+
+	return false
+}
+
 func (config *TapConfig) MaxEntriesDBSizeBytes() int64 {
 	maxEntriesDBSizeBytes, _ := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
 	return maxEntriesDBSizeBytes
@@ -69,15 +78,9 @@ func (config *TapConfig) Validate() error {
 		return errors.New(fmt.Sprintf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize))
 	}
 
-	if config.Workspace != "" {
-		workspaceRegex, _ := regexp.Compile("[A-Za-z0-9][-A-Za-z0-9_.]*[A-Za-z0-9]+$")
-		if len(config.Workspace) > 63 || !workspaceRegex.MatchString(config.Workspace) {
-			return errors.New("invalid workspace name")
-		}
-	}
-
-	if config.Analysis && config.Workspace != "" {
-		return errors.New(fmt.Sprintf("Can't run with both --%s and --%s flags", AnalysisTapName, WorkspaceTapName))
+	directionLowerCase := strings.ToLower(config.Direction)
+	if directionLowerCase != "any" && directionLowerCase != "in" {
+		return errors.New(fmt.Sprintf("%s is not a valid value for flag --%s. Acceptable values are in/any.", config.Direction, DirectionTapName))
 	}
 
 	return nil

cli/config/configStructs/viewConfig.go

@@ -2,10 +2,8 @@ package configStructs
 
 const (
 	GuiPortViewName = "gui-port"
-	UrlViewName     = "url"
 )
 
 type ViewConfig struct {
 	GuiPort uint16 `yaml:"gui-port" default:"8899"`
-	Url     string `yaml:"url,omitempty" readonly:""`
 }

cli/config/config_test.go

@@ -1,9 +1,7 @@
 package config_test
 
 import (
-	"fmt"
 	"github.com/up9inc/mizu/cli/config"
-	"gopkg.in/yaml.v3"
 	"reflect"
 	"strings"
 	"testing"
@@ -16,11 +14,9 @@ func TestConfigWriteIgnoresReadonlyFields(t *testing.T) {
 	getFieldsWithReadonlyTag(configElem, &readonlyFields)
 
 	configWithDefaults, _ := config.GetConfigWithDefaults()
-	configWithDefaultsBytes, _ := yaml.Marshal(configWithDefaults)
 	for _, readonlyField := range readonlyFields {
 		t.Run(readonlyField, func(t *testing.T) {
-			readonlyFieldToCheck := fmt.Sprintf(" %s:", readonlyField)
-			if strings.Contains(string(configWithDefaultsBytes), readonlyFieldToCheck) {
+			if strings.Contains(configWithDefaults, readonlyField) {
 				t.Errorf("unexpected result - readonly field: %v, config: %v", readonlyField, configWithDefaults)
 			}
 		})

cli/config/envConfig.go

@@ -1,24 +0,0 @@
-package config
-
-import (
-	"os"
-	"strconv"
-)
-
-const (
-	ApiServerRetries = "API_SERVER_RETRIES"
-)
-
-func GetIntEnvConfig(key string, defaultValue int) int {
-	value := os.Getenv(key)
-	if value == "" {
-		return defaultValue
-	}
-
-	intValue, err := strconv.Atoi(value)
-	if err != nil {
-		return defaultValue
-	}
-
-	return intValue
-}

cli/go.mod

@@ -5,14 +5,12 @@ go 1.16
 require (
 	github.com/creasty/defaults v1.5.1
 	github.com/denisbrodbeck/machineid v1.0.1
-	github.com/getkin/kin-openapi v0.79.0
 	github.com/google/go-github/v37 v37.0.0
-	github.com/google/uuid v1.1.2
+	github.com/gorilla/websocket v1.4.2
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	github.com/up9inc/mizu/shared v0.0.0
-	github.com/up9inc/mizu/tap/api v0.0.0
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.21.2
 	k8s.io/apimachinery v0.21.2
@@ -21,5 +19,3 @@ require (
 )
 
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
-
-replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api

cli/go.sum

@@ -113,9 +113,6 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
-github.com/getkin/kin-openapi v0.79.0 h1:YLZIgIhZLq9z5WFHHIK+oWORRfn6jjwr7qN0xak0xbE=
-github.com/getkin/kin-openapi v0.79.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
-github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
@@ -143,8 +140,6 @@ github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwds
 github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
-github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
@@ -170,7 +165,6 @@ github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk
 github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
 github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
@@ -181,8 +175,6 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0=
-github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -227,7 +219,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -245,8 +236,8 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
 github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -307,7 +298,6 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@@ -414,7 +404,6 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
@@ -702,7 +691,6 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

cli/kubernetes/provider.go

@@ -7,22 +7,15 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/shared/semver"
-	"k8s.io/apimachinery/pkg/version"
-	"net/url"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
 	"path/filepath"
 	"regexp"
 	"strconv"
 
-	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/shared/logger"
-
-	"io"
-
-	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap/api"
+	"io"
 	core "k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -62,32 +55,24 @@ func NewProvider(kubeConfigPath string) (*Provider, error) {
 	restClientConfig, err := kubernetesConfig.ClientConfig()
 	if err != nil {
 		if clientcmd.IsEmptyConfig(err) {
-			return nil, fmt.Errorf("couldn't find the kube config file, or file is empty (%s)\n"+
+			return nil, fmt.Errorf("couldn't find the kube config file, or file is empty (%s)\n" +
 				"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 		}
 		if clientcmd.IsConfigurationInvalid(err) {
-			return nil, fmt.Errorf("invalid kube config file (%s)\n"+
+			return nil, fmt.Errorf("invalid kube config file (%s)\n" +
 				"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 		}
 
-		return nil, fmt.Errorf("error while using kube config (%s)\n"+
+		return nil, fmt.Errorf("error while using kube config (%s)\n" +
 			"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 	}
 
 	clientSet, err := getClientSet(restClientConfig)
 	if err != nil {
-		return nil, fmt.Errorf("error while using kube config (%s)\n"+
+		return nil, fmt.Errorf("error while using kube config (%s)\n" +
 			"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 	}
 
-	if err := validateNotProxy(kubernetesConfig, restClientConfig); err != nil {
-		return nil, err
-	}
-
-	if err := validateKubernetesVersion(clientSet); err != nil {
-		return nil, err
-	}
-
 	return &Provider{
 		clientSet:        clientSet,
 		kubernetesConfig: kubernetesConfig,
@@ -158,26 +143,22 @@ func (provider *Provider) CreateNamespace(ctx context.Context, name string) (*co
 }
 
 type ApiServerOptions struct {
-	Namespace             string
-	PodName               string
-	PodImage              string
-	ServiceAccountName    string
-	IsNamespaceRestricted bool
-	SyncEntriesConfig     *shared.SyncEntriesConfig
-	MaxEntriesDBSizeBytes int64
-	Resources             configStructs.Resources
-	ImagePullPolicy       core.PullPolicy
+	Namespace               string
+	PodName                 string
+	PodImage                string
+	ServiceAccountName      string
+	IsNamespaceRestricted   bool
+	MizuApiFilteringOptions *shared.TrafficFilteringOptions
+	MaxEntriesDBSizeBytes   int64
+	Resources               configStructs.Resources
+	ImagePullPolicy         core.PullPolicy
 }
 
 func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiServerOptions) (*core.Pod, error) {
-	var marshaledSyncEntriesConfig []byte
-	if opts.SyncEntriesConfig != nil {
-		var err error
-		if marshaledSyncEntriesConfig, err = json.Marshal(opts.SyncEntriesConfig); err != nil {
-			return nil, err
-		}
+	marshaledFilteringOptions, err := json.Marshal(opts.MizuApiFilteringOptions)
+	if err != nil {
+		return nil, err
 	}
-
 	configMapVolumeName := &core.ConfigMapVolumeSource{}
 	configMapVolumeName.Name = mizu.ConfigMapName
 	configMapOptional := true
@@ -205,13 +186,6 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS
 		command = append(command, "--namespace", opts.Namespace)
 	}
 
-	port := intstr.FromInt(shared.DefaultApiServerPort)
-
-	debugMode := ""
-	if config.Config.DumpLogs {
-		debugMode = "1"
-	}
-
 	pod := &core.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      opts.PodName,
@@ -233,17 +207,17 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS
 					Command: command,
 					Env: []core.EnvVar{
 						{
-							Name:  shared.SyncEntriesConfigEnvVar,
-							Value: string(marshaledSyncEntriesConfig),
+							Name:  shared.HostModeEnvVar,
+							Value: "1",
+						},
+						{
+							Name:  shared.MizuFilteringOptionsEnvVar,
+							Value: string(marshaledFilteringOptions),
 						},
 						{
 							Name:  shared.MaxEntriesDBSizeBytesEnvVar,
 							Value: strconv.FormatInt(opts.MaxEntriesDBSizeBytes, 10),
 						},
-						{
-							Name:  shared.DebugModeEnvVar,
-							Value: debugMode,
-						},
 					},
 					Resources: core.ResourceRequirements{
 						Limits: core.ResourceList{
@@ -255,25 +229,6 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS
 							"memory": memRequests,
 						},
 					},
-					ReadinessProbe: &core.Probe{
-						Handler: core.Handler{
-							TCPSocket: &core.TCPSocketAction{
-								Port: port,
-							},
-						},
-						InitialDelaySeconds: 5,
-						PeriodSeconds:       10,
-					},
-					LivenessProbe: &core.Probe{
-						Handler: core.Handler{
-							HTTPGet: &core.HTTPGetAction{
-								Path: "/echo",
-								Port: port,
-							},
-						},
-						InitialDelaySeconds: 5,
-						PeriodSeconds:       10,
-					},
 				},
 			},
 			Volumes: []core.Volume{
@@ -302,7 +257,7 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s
 			Namespace: namespace,
 		},
 		Spec: core.ServiceSpec{
-			Ports:    []core.ServicePort{{TargetPort: intstr.FromInt(shared.DefaultApiServerPort), Port: 80}},
+			Ports:    []core.ServicePort{{TargetPort: intstr.FromInt(8899), Port: 80}},
 			Type:     core.ServiceTypeClusterIP,
 			Selector: map[string]string{"app": appLabelValue},
 		},
@@ -310,21 +265,67 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s
 	return provider.clientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{})
 }
 
+func (provider *Provider) DoesServiceAccountExist(ctx context.Context, namespace string, serviceAccountName string) (bool, error) {
+	serviceAccount, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{})
+	return provider.doesResourceExist(serviceAccount, err)
+}
+
+func (provider *Provider) DoesConfigMapExist(ctx context.Context, namespace string, name string) (bool, error) {
+	resource, err := provider.clientSet.CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
 func (provider *Provider) DoesServicesExist(ctx context.Context, namespace string, name string) (bool, error) {
 	resource, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{})
 	return provider.doesResourceExist(resource, err)
 }
 
-func (provider *Provider) doesResourceExist(resource interface{}, err error) (bool, error) {
-	// Getting NotFound error is the expected behavior when a resource does not exist.
-	if k8serrors.IsNotFound(err) {
-		return false, nil
-	}
+func (provider *Provider) DoesNamespaceExist(ctx context.Context, name string) (bool, error) {
+	resource, err := provider.clientSet.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
 
+func (provider *Provider) DoesClusterRoleExist(ctx context.Context, name string) (bool, error) {
+	resource, err := provider.clientSet.RbacV1().ClusterRoles().Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) DoesClusterRoleBindingExist(ctx context.Context, name string) (bool, error) {
+	resource, err := provider.clientSet.RbacV1().ClusterRoleBindings().Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) DoesRoleExist(ctx context.Context, namespace string, name string) (bool, error) {
+	resource, err := provider.clientSet.RbacV1().Roles(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) DoesRoleBindingExist(ctx context.Context, namespace string, name string) (bool, error) {
+	resource, err := provider.clientSet.RbacV1().RoleBindings(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) DoesPodExist(ctx context.Context, namespace string, name string) (bool, error) {
+	resource, err := provider.clientSet.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) DoesDaemonSetExist(ctx context.Context, namespace string, name string) (bool, error) {
+	resource, err := provider.clientSet.AppsV1().DaemonSets(namespace).Get(ctx, name, metav1.GetOptions{})
+	return provider.doesResourceExist(resource, err)
+}
+
+func (provider *Provider) doesResourceExist(resource interface{}, err error) (bool, error) {
+	var statusError *k8serrors.StatusError
+	if errors.As(err, &statusError) {
+		// expected behavior when resource does not exist
+		if statusError.ErrStatus.Reason == metav1.StatusReasonNotFound {
+			return false, nil
+		}
+	}
 	if err != nil {
 		return false, err
 	}
-
 	return resource != nil, nil
 }
 
@@ -437,73 +438,124 @@ func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context,
 }
 
 func (provider *Provider) RemoveNamespace(ctx context.Context, name string) error {
-	err := provider.clientSet.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{})
-	return provider.handleRemovalError(err)
-}
-
-func (provider *Provider) RemoveClusterRole(ctx context.Context, name string) error {
-	err := provider.clientSet.RbacV1().ClusterRoles().Delete(ctx, name, metav1.DeleteOptions{})
-	return provider.handleRemovalError(err)
-}
-
-func (provider *Provider) RemoveClusterRoleBinding(ctx context.Context, name string) error {
-	err := provider.clientSet.RbacV1().ClusterRoleBindings().Delete(ctx, name, metav1.DeleteOptions{})
-	return provider.handleRemovalError(err)
-}
-
-func (provider *Provider) RemoveRoleBinding(ctx context.Context, namespace string, name string) error {
-	err := provider.clientSet.RbacV1().RoleBindings(namespace).Delete(ctx, name, metav1.DeleteOptions{})
-	return provider.handleRemovalError(err)
-}
-
-func (provider *Provider) RemoveRole(ctx context.Context, namespace string, name string) error {
-	err := provider.clientSet.RbacV1().Roles(namespace).Delete(ctx, name, metav1.DeleteOptions{})
-	return provider.handleRemovalError(err)
-}
-
-func (provider *Provider) RemoveServicAccount(ctx context.Context, namespace string, name string) error {
-	err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{})
-	return provider.handleRemovalError(err)
-}
-
-func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) error {
-	err := provider.clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{})
-	return provider.handleRemovalError(err)
-}
-
-func (provider *Provider) RemoveConfigMap(ctx context.Context, namespace string, configMapName string) error {
-	err := provider.clientSet.CoreV1().ConfigMaps(namespace).Delete(ctx, configMapName, metav1.DeleteOptions{})
-	return provider.handleRemovalError(err)
-}
-
-func (provider *Provider) RemoveService(ctx context.Context, namespace string, serviceName string) error {
-	err := provider.clientSet.CoreV1().Services(namespace).Delete(ctx, serviceName, metav1.DeleteOptions{})
-	return provider.handleRemovalError(err)
-}
-
-func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, daemonSetName string) error {
-	err := provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{})
-	return provider.handleRemovalError(err)
-}
-
-func (provider *Provider) handleRemovalError(err error) error {
-	// Ignore NotFound - There is nothing to delete.
-	// Ignore Forbidden - Assume that a user could not have created the resource in the first place.
-	if k8serrors.IsNotFound(err) || k8serrors.IsForbidden(err) {
+	if isFound, err := provider.DoesNamespaceExist(ctx, name); err != nil {
+		return err
+	} else if !isFound {
 		return nil
 	}
 
-	return err
+	return provider.clientSet.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{})
 }
 
-func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, data string, contract string) error {
-	if data == "" && contract == "" {
+func (provider *Provider) RemoveNonNamespacedResources(ctx context.Context, clusterRoleName string, clusterRoleBindingName string) error {
+	if err := provider.RemoveClusterRole(ctx, clusterRoleName); err != nil {
+		return err
+	}
+
+	if err := provider.RemoveClusterRoleBinding(ctx, clusterRoleBindingName); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (provider *Provider) RemoveClusterRole(ctx context.Context, name string) error {
+	if isFound, err := provider.DoesClusterRoleExist(ctx, name); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.RbacV1().ClusterRoles().Delete(ctx, name, metav1.DeleteOptions{})
+}
+
+func (provider *Provider) RemoveClusterRoleBinding(ctx context.Context, name string) error {
+	if isFound, err := provider.DoesClusterRoleBindingExist(ctx, name); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.RbacV1().ClusterRoleBindings().Delete(ctx, name, metav1.DeleteOptions{})
+}
+
+func (provider *Provider) RemoveRoleBinding(ctx context.Context, namespace string, name string) error {
+	if isFound, err := provider.DoesRoleBindingExist(ctx, namespace, name); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.RbacV1().RoleBindings(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+}
+
+func (provider *Provider) RemoveRole(ctx context.Context, namespace string, name string) error {
+	if isFound, err := provider.DoesRoleExist(ctx, namespace, name); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.RbacV1().Roles(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+}
+
+func (provider *Provider) RemoveServicAccount(ctx context.Context, namespace string, name string) error {
+	if isFound, err := provider.DoesServiceAccountExist(ctx, namespace, name); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+}
+
+func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) error {
+	if isFound, err := provider.DoesPodExist(ctx, namespace, podName); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{})
+}
+
+func (provider *Provider) RemoveConfigMap(ctx context.Context, namespace string, configMapName string) error {
+	if isFound, err := provider.DoesConfigMapExist(ctx, namespace, configMapName); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.CoreV1().ConfigMaps(namespace).Delete(ctx, configMapName, metav1.DeleteOptions{})
+}
+
+func (provider *Provider) RemoveService(ctx context.Context, namespace string, serviceName string) error {
+	if isFound, err := provider.DoesServicesExist(ctx, namespace, serviceName); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.CoreV1().Services(namespace).Delete(ctx, serviceName, metav1.DeleteOptions{})
+}
+
+func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, daemonSetName string) error {
+	if isFound, err := provider.DoesDaemonSetExist(ctx, namespace, daemonSetName); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{})
+}
+
+func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, data string) error {
+	if data == "" {
 		return nil
 	}
 
 	configMapData := make(map[string]string, 0)
 	configMapData[shared.RulePolicyFileName] = data
-	configMapData[shared.ContractFileName] = contract
 	configMap := &core.ConfigMap{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "ConfigMap",
@@ -521,11 +573,11 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
 	return nil
 }
 
-func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, resources configStructs.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions *api.TrafficFilteringOptions) error {
-	logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
+func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool, resources configStructs.Resources, imagePullPolicy core.PullPolicy) error {
+	logger.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
 
 	if len(nodeToTappedPodIPMap) == 0 {
-		return fmt.Errorf("daemon set %s must tap at least 1 pod", daemonSetName)
+		return fmt.Errorf("Daemon set %s must tap at least 1 pod", daemonSetName)
 	}
 
 	nodeToTappedPodIPMapJsonStr, err := json.Marshal(nodeToTappedPodIPMap)
@@ -533,22 +585,14 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 		return err
 	}
 
-	marshaledFilteringOptions, err := json.Marshal(mizuApiFilteringOptions)
-	if err != nil {
-		return err
-	}
-
 	mizuCmd := []string{
 		"./mizuagent",
 		"-i", "any",
 		"--tap",
 		"--api-server-address", fmt.Sprintf("ws://%s/wsTapper", apiServerPodIp),
-		"--nodefrag",
 	}
-
-	debugMode := ""
-	if config.Config.DumpLogs {
-		debugMode = "1"
+	if tapOutgoing {
+		mizuCmd = append(mizuCmd, "--anydirection")
 	}
 
 	agentContainer := applyconfcore.Container()
@@ -558,11 +602,8 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true))
 	agentContainer.WithCommand(mizuCmd...)
 	agentContainer.WithEnv(
-		applyconfcore.EnvVar().WithName(shared.DebugModeEnvVar).WithValue(debugMode),
 		applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"),
 		applyconfcore.EnvVar().WithName(shared.TappedAddressesPerNodeDictEnvVar).WithValue(string(nodeToTappedPodIPMapJsonStr)),
-		applyconfcore.EnvVar().WithName(shared.GoGCEnvVar).WithValue("12800"),
-		applyconfcore.EnvVar().WithName(shared.MizuFilteringOptionsEnvVar).WithValue(string(marshaledFilteringOptions)),
 	)
 	agentContainer.WithEnv(
 		applyconfcore.EnvVar().WithName(shared.NodeNameEnvVar).WithValueFrom(
@@ -682,7 +723,7 @@ func (provider *Provider) ListAllRunningPodsMatchingRegex(ctx context.Context, r
 	return matchingPods, nil
 }
 
-func (provider *Provider) GetPodLogs(ctx context.Context, namespace string, podName string) (string, error) {
+func (provider *Provider) GetPodLogs(namespace string, podName string, ctx context.Context) (string, error) {
 	podLogOpts := core.PodLogOptions{}
 	req := provider.clientSet.CoreV1().Pods(namespace).GetLogs(podName, &podLogOpts)
 	podLogs, err := req.Stream(ctx)
@@ -698,16 +739,6 @@ func (provider *Provider) GetPodLogs(ctx context.Context, namespace string, podN
 	return str, nil
 }
 
-func (provider *Provider) GetNamespaceEvents(ctx context.Context, namespace string) (string, error) {
-	eventsOpts := metav1.ListOptions{}
-	eventList, err := provider.clientSet.CoreV1().Events(namespace).List(ctx, eventsOpts)
-	if err != nil {
-		return "", fmt.Errorf("error getting events on ns: %s, %w", namespace, err)
-	}
-
-	return eventList.String(), nil
-}
-
 func getClientSet(config *restclient.Config) (*kubernetes.Clientset, error) {
 	clientSet, err := kubernetes.NewForConfig(config)
 	if err != nil {
@@ -738,47 +769,3 @@ func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
 func isPodRunning(pod *core.Pod) bool {
 	return pod.Status.Phase == core.PodRunning
 }
-
-// We added this after a customer tried to run mizu from lens, which used len's kube config, which have cluster server configuration, which points to len's local proxy.
-// The workaround was to use the user's local default kube config.
-// For now - we are blocking the option to run mizu through a proxy to k8s server
-func validateNotProxy(kubernetesConfig clientcmd.ClientConfig, restClientConfig *restclient.Config) error {
-	kubernetesUrl, err := url.Parse(restClientConfig.Host)
-	if err != nil {
-		logger.Log.Debugf("validateNotProxy - error while parsing kubernetes host, err: %v", err)
-		return nil
-	}
-
-	restProxyClientConfig, _ := kubernetesConfig.ClientConfig()
-	restProxyClientConfig.Host = kubernetesUrl.Host
-
-	clientProxySet, err := getClientSet(restProxyClientConfig)
-	if err == nil {
-		proxyServerVersion, err := clientProxySet.ServerVersion()
-		if err != nil {
-			return nil
-		}
-
-		if *proxyServerVersion == (version.Info{}) {
-			return fmt.Errorf("cannot establish http-proxy connection to the Kubernetes cluster. If you’re using Lens or similar tool, please run mizu with regular kubectl config using --%v %v=$HOME/.kube/config flag", config.SetCommandName, config.KubeConfigPathConfigName)
-		}
-	}
-
-	return nil
-}
-
-func validateKubernetesVersion(clientSet *kubernetes.Clientset) error {
-	serverVersion, err := clientSet.ServerVersion()
-	if err != nil {
-		logger.Log.Debugf("error while getting kubernetes server version, err: %v", err)
-		return nil
-	}
-
-	serverVersionSemVer := semver.SemVersion(serverVersion.GitVersion)
-	minKubernetesServerVersionSemVer := semver.SemVersion(mizu.MinKubernetesServerVersion)
-	if minKubernetesServerVersionSemVer.GreaterThan(serverVersionSemVer) {
-		return fmt.Errorf("kubernetes server version %v is not supported, supporting only kubernetes server version of %v or higher", serverVersion.GitVersion, mizu.MinKubernetesServerVersion)
-	}
-
-	return nil
-}

cli/kubernetes/proxy.go

@@ -2,13 +2,12 @@ package kubernetes
 
 import (
 	"fmt"
+	"github.com/up9inc/mizu/cli/logger"
+	"k8s.io/kubectl/pkg/proxy"
 	"net"
 	"net/http"
 	"strings"
 	"time"
-
-	"github.com/up9inc/mizu/shared/logger"
-	"k8s.io/kubectl/pkg/proxy"
 )
 
 const k8sProxyApiPrefix = "/"
@@ -40,7 +39,6 @@ func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace str
 	server := http.Server{
 		Handler: mux,
 	}
-
 	return server.Serve(l)
 }
 

cli/kubernetes/watch.go

@@ -3,15 +3,10 @@ package kubernetes
 import (
 	"context"
 	"errors"
-	"fmt"
-	"github.com/up9inc/mizu/shared/debounce"
-	"github.com/up9inc/mizu/shared/logger"
 	"regexp"
 	"sync"
-	"time"
 
 	corev1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/watch"
 )
 
@@ -21,7 +16,6 @@ func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetName
 	removedChan := make(chan *corev1.Pod)
 	errorChan := make(chan error)
 
-
 	var wg sync.WaitGroup
 
 	for _, targetNamespace := range targetNamespaces {
@@ -29,33 +23,33 @@ func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetName
 
 		go func(targetNamespace string) {
 			defer wg.Done()
-			watchRestartDebouncer := debounce.NewDebouncer(1 * time.Minute, func() {})
+			watcher := kubernetesProvider.GetPodWatcher(ctx, targetNamespace)
 
 			for {
-				watcher := kubernetesProvider.GetPodWatcher(ctx, targetNamespace)
-				err := startWatchLoop(ctx, watcher, podFilter, addedChan, modifiedChan, removedChan) // blocking
-				watcher.Stop()
-
 				select {
-				case <- ctx.Done():
-					return
-				default:
-					break
-				}
-
-				if err != nil {
-					errorChan <- fmt.Errorf("error in k8 watch: %v", err)
-					break
-				} else {
-					if !watchRestartDebouncer.IsOn() {
-						watchRestartDebouncer.SetOn()
-						logger.Log.Debug("k8s watch channel closed, restarting watcher")
-						time.Sleep(time.Second * 5)
-						continue
-					} else {
-						errorChan <- errors.New("k8s watch unstable, closes frequently")
-						break
+				case e := <-watcher.ResultChan():
+					if e.Object == nil {
+						errorChan <- errors.New("kubernetes pod watch failed")
+						return
 					}
+
+					pod := e.Object.(*corev1.Pod)
+
+					if !podFilter.MatchString(pod.Name) {
+						continue
+					}
+
+					switch e.Type {
+					case watch.Added:
+						addedChan <- pod
+					case watch.Modified:
+						modifiedChan <- pod
+					case watch.Deleted:
+						removedChan <- pod
+					}
+				case <-ctx.Done():
+					watcher.Stop()
+					return
 				}
 			}
 		}(targetNamespace)
@@ -72,39 +66,3 @@ func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetName
 
 	return addedChan, modifiedChan, removedChan, errorChan
 }
-
-func startWatchLoop(ctx context.Context, watcher watch.Interface, podFilter *regexp.Regexp, addedChan chan *corev1.Pod, modifiedChan chan *corev1.Pod, removedChan chan *corev1.Pod) error {
-	resultChan := watcher.ResultChan()
-	for {
-		select {
-		case e, isChannelOpen := <-resultChan:
-			if !isChannelOpen {
-				return nil
-			}
-
-			if e.Type == watch.Error {
-				return apierrors.FromObject(e.Object)
-			}
-
-			pod, ok := e.Object.(*corev1.Pod)
-			if !ok {
-				continue
-			}
-
-			if !podFilter.MatchString(pod.Name) {
-				continue
-			}
-
-			switch e.Type {
-			case watch.Added:
-				addedChan <- pod
-			case watch.Modified:
-				modifiedChan <- pod
-			case watch.Deleted:
-				removedChan <- pod
-			}
-		case <-ctx.Done():
-			return nil
-		}
-	}
-}

cli/logger/logger.go

@@ -1,18 +1,24 @@
 package logger
 
 import (
-	"os"
-
 	"github.com/op/go-logging"
+	"github.com/up9inc/mizu/cli/mizu"
+	"os"
+	"path"
 )
 
-var Log = logging.MustGetLogger("mizu")
+var Log = logging.MustGetLogger("mizu_cli")
 
 var format = logging.MustStringFormatter(
 	`%{time} %{level:.5s} ▶ %{pid} %{shortfile} %{shortfunc} ▶ %{message}`,
 )
 
-func InitLogger(logPath string) {
+func GetLogFilePath() string {
+	return path.Join(mizu.GetMizuFolderPath(), "mizu_cli.log")
+}
+
+func InitLogger() {
+	logPath := GetLogFilePath()
 	f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
 	if err != nil {
 		Log.Infof("Failed to open mizu log file: %v, err %v", logPath, err)
@@ -27,12 +33,7 @@ func InitLogger(logPath string) {
 	backend1Leveled.SetLevel(logging.INFO, "")
 
 	logging.SetBackend(backend1Leveled, backend2Formatter)
-}
 
-func InitLoggerStderrOnly(level logging.Level) {
-	backend := logging.NewLogBackend(os.Stderr, "", 0)
-	backendFormatter := logging.NewBackendFormatter(backend, format)
-
-	logging.SetBackend(backendFormatter)
-	logging.SetLevel(level, "")
+	Log.Debugf("\n\n\n")
+	Log.Debugf("Running mizu version %v", mizu.SemVer)
 }

cli/mizu/consts.go

@@ -14,18 +14,17 @@ var (
 )
 
 const (
-	MizuResourcesPrefix        = "mizu-"
-	ApiServerPodName           = MizuResourcesPrefix + "api-server"
-	ClusterRoleBindingName     = MizuResourcesPrefix + "cluster-role-binding"
-	ClusterRoleName            = MizuResourcesPrefix + "cluster-role"
-	K8sAllNamespaces           = ""
-	RoleBindingName            = MizuResourcesPrefix + "role-binding"
-	RoleName                   = MizuResourcesPrefix + "role"
-	ServiceAccountName         = MizuResourcesPrefix + "service-account"
-	TapperDaemonSetName        = MizuResourcesPrefix + "tapper-daemon-set"
-	TapperPodName              = MizuResourcesPrefix + "tapper"
-	ConfigMapName              = MizuResourcesPrefix + "policy"
-	MinKubernetesServerVersion = "1.16.0"
+	MizuResourcesPrefix    = "mizu-"
+	ApiServerPodName       = MizuResourcesPrefix + "api-server"
+	ClusterRoleBindingName = MizuResourcesPrefix + "cluster-role-binding"
+	ClusterRoleName        = MizuResourcesPrefix + "cluster-role"
+	K8sAllNamespaces       = ""
+	RoleBindingName        = MizuResourcesPrefix + "role-binding"
+	RoleName               = MizuResourcesPrefix + "role"
+	ServiceAccountName     = MizuResourcesPrefix + "service-account"
+	TapperDaemonSetName    = MizuResourcesPrefix + "tapper-daemon-set"
+	TapperPodName          = MizuResourcesPrefix + "tapper"
+	ConfigMapName          = MizuResourcesPrefix + "policy"
 )
 
 func GetMizuFolderPath() string {

cli/mizu/controlSocket.go

@@ -0,0 +1,42 @@
+package mizu
+
+import (
+	"encoding/json"
+	"github.com/gorilla/websocket"
+	"github.com/up9inc/mizu/shared"
+	core "k8s.io/api/core/v1"
+	"time"
+)
+
+type ControlSocket struct {
+	connection *websocket.Conn
+}
+
+func CreateControlSocket(socketServerAddress string) (*ControlSocket, error) {
+	connection, err := shared.ConnectToSocketServer(socketServerAddress, 30, 2 * time.Second, true)
+	if err != nil {
+		return nil, err
+	} else {
+		return &ControlSocket{connection: connection}, nil
+	}
+}
+
+func (controlSocket *ControlSocket) SendNewTappedPodsListMessage(pods []core.Pod) error {
+	podInfos := make([]shared.PodInfo, 0)
+	for _, pod := range pods {
+		podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
+	}
+	tapStatus := shared.TapStatus{Pods: podInfos}
+	socketMessage := shared.CreateWebSocketStatusMessage(tapStatus)
+
+	jsonMessage, err := json.Marshal(socketMessage)
+	if err != nil {
+		return err
+	}
+	err = controlSocket.connection.WriteMessage(websocket.TextMessage, jsonMessage)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

cli/mizu/fsUtils/mizuLogsUtils.go

@@ -4,21 +4,15 @@ import (
 	"archive/zip"
 	"context"
 	"fmt"
-	"os"
-	"path"
-	"regexp"
-
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/shared/logger"
+	"os"
+	"regexp"
 )
 
-func GetLogFilePath() string {
-	return path.Join(mizu.GetMizuFolderPath(), "mizu_cli.log")
-}
-
-func DumpLogs(ctx context.Context, provider *kubernetes.Provider, filePath string) error {
+func DumpLogs(provider *kubernetes.Provider, ctx context.Context, filePath string) error {
 	podExactRegex := regexp.MustCompile("^" + mizu.MizuResourcesPrefix)
 	pods, err := provider.ListAllPodsMatchingRegex(ctx, podExactRegex, []string{config.Config.MizuResourcesNamespace})
 	if err != nil {
@@ -38,46 +32,29 @@ func DumpLogs(ctx context.Context, provider *kubernetes.Provider, filePath strin
 	defer zipWriter.Close()
 
 	for _, pod := range pods {
-		logs, err := provider.GetPodLogs(ctx, pod.Namespace, pod.Name)
+		logs, err := provider.GetPodLogs(pod.Namespace, pod.Name, ctx)
 		if err != nil {
 			logger.Log.Errorf("Failed to get logs, %v", err)
 			continue
 		} else {
 			logger.Log.Debugf("Successfully read log length %d for pod: %s.%s", len(logs), pod.Namespace, pod.Name)
 		}
-
 		if err := AddStrToZip(zipWriter, logs, fmt.Sprintf("%s.%s.log", pod.Namespace, pod.Name)); err != nil {
 			logger.Log.Errorf("Failed write logs, %v", err)
 		} else {
 			logger.Log.Debugf("Successfully added log length %d from pod: %s.%s", len(logs), pod.Namespace, pod.Name)
 		}
 	}
-
-	events, err := provider.GetNamespaceEvents(ctx, config.Config.MizuResourcesNamespace)
-	if err != nil {
-		logger.Log.Debugf("Failed to get k8b events, %v", err)
-	} else {
-		logger.Log.Debugf("Successfully read events for k8b namespace: %s", config.Config.MizuResourcesNamespace)
-	}
-
-	if err := AddStrToZip(zipWriter, events, fmt.Sprintf("%s_events.log", config.Config.MizuResourcesNamespace)); err != nil {
-		logger.Log.Debugf("Failed write logs, %v", err)
-	} else {
-		logger.Log.Debugf("Successfully added events for k8b namespace: %s", config.Config.MizuResourcesNamespace)
-	}
-
-	if err := AddFileToZip(zipWriter, config.Config.ConfigFilePath); err != nil {
+	if err := AddFileToZip(zipWriter, config.GetConfigFilePath()); err != nil {
 		logger.Log.Debugf("Failed write file, %v", err)
 	} else {
-		logger.Log.Debugf("Successfully added file %s", config.Config.ConfigFilePath)
+		logger.Log.Debugf("Successfully added file %s", config.GetConfigFilePath())
 	}
-
-	if err := AddFileToZip(zipWriter, GetLogFilePath()); err != nil {
+	if err := AddFileToZip(zipWriter, logger.GetLogFilePath()); err != nil {
 		logger.Log.Debugf("Failed write file, %v", err)
 	} else {
-		logger.Log.Debugf("Successfully added file %s", GetLogFilePath())
+		logger.Log.Debugf("Successfully added file %s", logger.GetLogFilePath())
 	}
-
 	logger.Log.Infof("You can find the zip file with all logs in %s\n", filePath)
 	return nil
 }

cli/mizu/fsUtils/zipUtils.go

@@ -6,9 +6,6 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"strings"
-
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 func AddFileToZip(zipWriter *zip.Writer, filename string) error {
@@ -56,60 +53,3 @@ func AddStrToZip(writer *zip.Writer, logs string, fileName string) error {
 	}
 	return nil
 }
-
-func Unzip(reader *zip.Reader, dest string) error {
-	dest, _ = filepath.Abs(dest)
-	_ = os.MkdirAll(dest, os.ModePerm)
-
-	// Closure to address file descriptors issue with all the deferred .Close() methods
-	extractAndWriteFile := func(f *zip.File) error {
-		rc, err := f.Open()
-		if err != nil {
-			return err
-		}
-		defer func() {
-			if err := rc.Close(); err != nil {
-				panic(err)
-			}
-		}()
-
-		path := filepath.Join(dest, f.Name)
-
-		// Check for ZipSlip (Directory traversal)
-		if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) {
-			return fmt.Errorf("illegal file path: %s", path)
-		}
-
-		if f.FileInfo().IsDir() {
-			_ = os.MkdirAll(path, f.Mode())
-		} else {
-			_ = os.MkdirAll(filepath.Dir(path), f.Mode())
-			logger.Log.Infof("writing HAR file [ %v ]", path)
-			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
-			if err != nil {
-				return err
-			}
-			defer func() {
-				if err := f.Close(); err != nil {
-					panic(err)
-				}
-				logger.Log.Info(" done")
-			}()
-
-			_, err = io.Copy(f, rc)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	for _, f := range reader.File {
-		err := extractAndWriteFile(f)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

cli/mizu/goUtils/funcWrappers.go

@@ -1,10 +1,9 @@
 package goUtils
 
 import (
+	"github.com/up9inc/mizu/cli/logger"
 	"reflect"
 	"runtime/debug"
-
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 func HandleExcWrapper(fn interface{}, params ...interface{}) (result []reflect.Value) {

cli/mizu/sliceUtils.go

@@ -0,0 +1,11 @@
+package mizu
+
+func Contains(slice []string, containsValue string) bool {
+	for _, sliceValue := range slice {
+		if sliceValue == containsValue {
+			return true
+		}
+	}
+
+	return false
+}

cli/mizu/sliceUtils_test.go

@@ -1,9 +1,7 @@
-package shared_test
+package mizu_test
 
 import (
-	"fmt"
-	"github.com/up9inc/mizu/shared"
-	"reflect"
+	"github.com/up9inc/mizu/cli/mizu"
 	"testing"
 )
 
@@ -21,7 +19,7 @@ func TestContainsExists(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.ContainsValue, func(t *testing.T) {
-			actual := shared.Contains(test.Slice, test.ContainsValue)
+			actual := mizu.Contains(test.Slice, test.ContainsValue)
 			if actual != test.Expected {
 				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
 			}
@@ -43,7 +41,7 @@ func TestContainsNotExists(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.ContainsValue, func(t *testing.T) {
-			actual := shared.Contains(test.Slice, test.ContainsValue)
+			actual := mizu.Contains(test.Slice, test.ContainsValue)
 			if actual != test.Expected {
 				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
 			}
@@ -63,7 +61,7 @@ func TestContainsEmptySlice(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.ContainsValue, func(t *testing.T) {
-			actual := shared.Contains(test.Slice, test.ContainsValue)
+			actual := mizu.Contains(test.Slice, test.ContainsValue)
 			if actual != test.Expected {
 				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
 			}
@@ -83,48 +81,10 @@ func TestContainsNilSlice(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.ContainsValue, func(t *testing.T) {
-			actual := shared.Contains(test.Slice, test.ContainsValue)
+			actual := mizu.Contains(test.Slice, test.ContainsValue)
 			if actual != test.Expected {
 				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
 			}
 		})
 	}
 }
-
-func TestUniqueNoDuplicateValues(t *testing.T) {
-	tests := []struct {
-		Slice         []string
-		Expected      []string
-	}{
-		{Slice: []string{"apple", "orange", "banana", "grapes"}, Expected: []string{"apple", "orange", "banana", "grapes"}},
-		{Slice: []string{"dog", "cat", "mouse"}, Expected: []string{"dog", "cat", "mouse"}},
-	}
-
-	for index, test := range tests {
-		t.Run(fmt.Sprintf("%v", index), func(t *testing.T) {
-			actual := shared.Unique(test.Slice)
-			if !reflect.DeepEqual(test.Expected, actual) {
-				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
-			}
-		})
-	}
-}
-
-func TestUniqueDuplicateValues(t *testing.T) {
-	tests := []struct {
-		Slice         []string
-		Expected      []string
-	}{
-		{Slice: []string{"apple", "apple", "orange", "orange", "banana", "banana", "grapes", "grapes"}, Expected: []string{"apple", "orange", "banana", "grapes"}},
-		{Slice: []string{"dog", "cat", "cat", "mouse"}, Expected: []string{"dog", "cat", "mouse"}},
-	}
-
-	for index, test := range tests {
-		t.Run(fmt.Sprintf("%v", index), func(t *testing.T) {
-			actual := shared.Unique(test.Slice)
-			if !reflect.DeepEqual(test.Expected, actual) {
-				t.Errorf("unexpected result - Expected: %v, actual: %v", test.Expected, actual)
-			}
-		})
-	}
-}

cli/mizu/version/versionCheck.go

@@ -2,24 +2,43 @@ package version
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu"
 	"io/ioutil"
 	"net/http"
-	"runtime"
-	"strings"
+	"net/url"
 	"time"
 
-	"github.com/up9inc/mizu/cli/apiserver"
-	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/shared/logger"
-
 	"github.com/google/go-github/v37/github"
 	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/semver"
 )
 
-func CheckVersionCompatibility() (bool, error) {
-	apiSemVer, err := apiserver.Provider.GetVersion()
+func getApiVersion(port uint16) (string, error) {
+	versionUrl, _ := url.Parse(fmt.Sprintf("http://localhost:%d/mizu/metadata/version", port))
+	req := &http.Request{
+		Method: http.MethodGet,
+		URL:    versionUrl,
+	}
+	statusResp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer statusResp.Body.Close()
+
+	versionResponse := &shared.VersionResponse{}
+	if err := json.NewDecoder(statusResp.Body).Decode(&versionResponse); err != nil {
+		return "", err
+	}
+
+	return versionResponse.SemVer, nil
+}
+
+func CheckVersionCompatibility(port uint16) (bool, error) {
+	apiSemVer, err := getApiVersion(port)
 	if err != nil {
 		return false, err
 	}
@@ -40,7 +59,6 @@ func CheckNewerVersion(versionChan chan string) {
 	latestRelease, _, err := client.Repositories.GetLatestRelease(context.Background(), "up9inc", "mizu")
 	if err != nil {
 		logger.Log.Debugf("[ERROR] Failed to get latest release")
-		versionChan <- ""
 		return
 	}
 
@@ -53,14 +71,12 @@ func CheckNewerVersion(versionChan chan string) {
 	}
 	if versionFileUrl == "" {
 		logger.Log.Debugf("[ERROR] Version file not found in the latest release")
-		versionChan <- ""
 		return
 	}
 
 	res, err := http.Get(versionFileUrl)
 	if err != nil {
 		logger.Log.Debugf("[ERROR] Failed to get the version file %v", err)
-		versionChan <- ""
 		return
 	}
 
@@ -68,7 +84,6 @@ func CheckNewerVersion(versionChan chan string) {
 	res.Body.Close()
 	if err != nil {
 		logger.Log.Debugf("[ERROR] Failed to read the version file -> %v", err)
-		versionChan <- ""
 		return
 	}
 	gitHubVersion := string(data)
@@ -76,23 +91,10 @@ func CheckNewerVersion(versionChan chan string) {
 
 	gitHubVersionSemVer := semver.SemVersion(gitHubVersion)
 	currentSemVer := semver.SemVersion(mizu.SemVer)
-	if !gitHubVersionSemVer.IsValid() || !currentSemVer.IsValid() {
-		logger.Log.Debugf("[ERROR] Semver version is not valid, github version %v, current version %v", gitHubVersion, currentSemVer)
-		versionChan <- ""
-		return
-	}
-
 	logger.Log.Debugf("Finished version validation, github version %v, current version %v, took %v", gitHubVersion, currentSemVer, time.Since(start))
 
 	if gitHubVersionSemVer.GreaterThan(currentSemVer) {
-		var downloadMessage string
-		if runtime.GOOS == "windows" {
-			downloadMessage = fmt.Sprintf("curl -LO %v/mizu.exe", strings.Replace(*latestRelease.HTMLURL, "tag", "download", 1))
-		} else {
-			downloadMessage = fmt.Sprintf("curl -Lo mizu %v/mizu_%s_%s && chmod 755 mizu", strings.Replace(*latestRelease.HTMLURL, "tag", "download", 1), runtime.GOOS, runtime.GOARCH)
-		}
-		versionChan <- fmt.Sprintf("Update available! %v -> %v (%s)", mizu.SemVer, gitHubVersion, downloadMessage)
-	} else {
-		versionChan <- ""
+		versionChan <- fmt.Sprintf("Update available! %v -> %v (%v)", mizu.SemVer, gitHubVersion, *latestRelease.HTMLURL)
 	}
+	versionChan <- ""
 }

cli/telemetry/telemetry.go

@@ -4,13 +4,13 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"net/http"
-
 	"github.com/denisbrodbeck/machineid"
-	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/shared/logger"
+	"io/ioutil"
+	"net/http"
 )
 
 const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
@@ -35,15 +35,35 @@ func ReportRun(cmd string, args interface{}) {
 	logger.Log.Debugf("successfully reported telemetry for cmd %v", cmd)
 }
 
-func ReportAPICalls() {
+func ReportAPICalls(mizuPort uint16) {
 	if !shouldRunTelemetry() {
 		logger.Log.Debugf("not reporting telemetry")
 		return
 	}
 
-	generalStats, err := apiserver.Provider.GetGeneralStats()
-	if err != nil {
-		logger.Log.Debugf("[ERROR] failed get general stats from api server %v", err)
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizuPort)
+	generalStatsUrl := fmt.Sprintf("http://%s/api/generalStats", mizuProxiedUrl)
+
+	response, requestErr := http.Get(generalStatsUrl)
+	if requestErr != nil {
+		logger.Log.Debugf("ERROR: failed to get general stats for telemetry, err: %v", requestErr)
+		return
+	} else if response.StatusCode != 200 {
+		logger.Log.Debugf("ERROR: failed to get general stats for telemetry, status code: %v", response.StatusCode)
+		return
+	}
+
+	defer func() { _ = response.Body.Close() }()
+
+	data, readErr := ioutil.ReadAll(response.Body)
+	if readErr != nil {
+		logger.Log.Debugf("ERROR: failed to read general stats for telemetry, err: %v", readErr)
+		return
+	}
+
+	var generalStats map[string]interface{}
+	if parseErr := json.Unmarshal(data, &generalStats); parseErr != nil {
+		logger.Log.Debugf("ERROR: failed to parse general stats for telemetry, err: %v", parseErr)
 		return
 	}
 

cli/uiUtils/confirmation.go

@@ -3,10 +3,9 @@ package uiUtils
 import (
 	"bufio"
 	"fmt"
+	"log"
 	"os"
 	"strings"
-
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 func AskForConfirmation(s string) bool {
@@ -16,7 +15,7 @@ func AskForConfirmation(s string) bool {
 
 	response, err := reader.ReadString('\n')
 	if err != nil {
-		logger.Log.Fatalf("Error while reading confirmation string, err: %v", err)
+		log.Fatal(err)
 	}
 	response = strings.ToLower(strings.TrimSpace(response))
 	if response == "" || response == "y" || response == "yes" {

cli/uiUtils/openBrowser.go

@@ -1,28 +0,0 @@
-package uiUtils
-
-import (
-	"fmt"
-	"os/exec"
-	"runtime"
-
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-func OpenBrowser(url string) {
-	var err error
-
-	switch runtime.GOOS {
-	case "linux":
-		err = exec.Command("xdg-open", url).Start()
-	case "windows":
-		err = exec.Command("rundll32", "url.dll,FileProtocolHandler", url).Start()
-	case "darwin":
-		err = exec.Command("open", url).Start()
-	default:
-		err = fmt.Errorf("unsupported platform")
-	}
-
-	if err != nil {
-		logger.Log.Errorf("error while opening browser, %v", err)
-	}
-}

debug.Dockerfile

@@ -12,7 +12,7 @@ FROM golang:1.16-alpine AS builder
 # Set necessary environment variables needed for our image.
 ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 
-RUN apk add libpcap-dev gcc g++ make bash
+RUN apk add libpcap-dev gcc g++ make
 
 # Move to agent working directory (/agent-build).
 WORKDIR /app/agent-build
@@ -20,25 +20,17 @@ WORKDIR /app/agent-build
 COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
 COPY tap/go.mod tap/go.mod ../tap/
-COPY tap/api/go.* ../tap/api/
+
 RUN go mod download
 # cheap trick to make the build faster (As long as go.mod wasn't changes)
 RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
 
-ARG COMMIT_HASH
-ARG GIT_BRANCH
-ARG BUILD_TIMESTAMP
-ARG SEM_VER
-
 # Copy and build agent code
 COPY shared ../shared
 COPY tap ../tap
 COPY agent .
 RUN go build -gcflags="all=-N -l" -o mizuagent .
 
-COPY devops/build_extensions_debug.sh ..
-RUN cd .. && /bin/bash build_extensions_debug.sh
-
 
 FROM golang:1.16-alpine
 
@@ -47,7 +39,6 @@ WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
-COPY --from=builder ["/app/agent/build/extensions", "extensions"]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 
 # install remote debugging tool

devops/build-push-featurebranch-debug.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-set -e
-
-GCP_PROJECT=up9-docker-hub
-REPOSITORY=gcr.io/$GCP_PROJECT
-SERVER_NAME=mizu
-GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
-
-DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
-SEM_VER=${SEM_VER=0.0.0}
-
-DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
-
-if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
-then
-  echo "Pushing to $GIT_BRANCH is allowed only via CI"
-  exit 1
-fi
-
-echo "building ${DOCKER_TAGGED_BUILDS[@]}"
-DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
-docker build -f debug.Dockerfile $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
-
-for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
-do
-  echo pushing "$DOCKER_TAG"
-  docker push "$DOCKER_TAG"
-done

devops/build_extensions.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-for f in tap/extensions/*; do
-    if [ -d "$f" ]; then
-        extension=$(basename $f) && \
-        cd tap/extensions/${extension} && \
-        go build -buildmode=plugin -o ../${extension}.so .  && \
-        cd ../../..  && \
-        mkdir -p agent/build/extensions  && \
-        cp tap/extensions/${extension}.so agent/build/extensions
-    fi
-done

devops/build_extensions_debug.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-for f in tap/extensions/*; do
-    if [ -d "$f" ]; then
-        extension=$(basename $f) && \
-        cd tap/extensions/${extension} && \
-        go build -gcflags="all=-N -l" -buildmode=plugin -o ../${extension}.so .  && \
-        cd ../../..  && \
-        mkdir -p agent/build/extensions  && \
-        cp tap/extensions/${extension}.so agent/build/extensions
-    fi
-done

docs/CODE_OF_CONDUCT.md

@@ -1,76 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to making participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
-appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or
- advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
- address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
- professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at mizu@up9.com. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
-
-[homepage]: https://www.contributor-covenant.org
-
-For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq

docs/CONTRACT_MONITORING.md

@@ -1,172 +0,0 @@
-# OpenAPI Specification (OAS) Contract Monitoring
-
-An OAS/Swagger file can contain schemas under `parameters` and `responses` fields. With `--contract catalogue.yaml`
-CLI option, you can pass your API description to Mizu and the traffic will automatically be validated
-against the contracts.
-
-Below is an example of an OAS/Swagger file from [Sock Shop](https://microservices-demo.github.io/) microservice demo
-that contains a bunch contracts:
-
-```yaml
-openapi: 3.0.1
-info:
-  title: Catalogue resources
-  version: 1.0.0
-  description: ""
-  license:
-    name: MIT
-    url: http://github.com/gruntjs/grunt/blob/master/LICENSE-MIT
-paths:
-  /catalogue:
-    get:
-      description: Catalogue API
-      operationId: List catalogue
-      responses:
-        200:
-          description: ""
-          content:
-            application/json;charset=UTF-8:
-              schema:
-                type: array
-                items:
-                  $ref: '#/components/schemas/Listresponse'
-  /catalogue/{id}:
-    get:
-      operationId: Get an item
-      parameters:
-      - name: id
-        in: path
-        required: true
-        schema:
-          type: string
-        example: a0a4f044-b040-410d-8ead-4de0446aec7e
-      responses:
-        200:
-          description: ""
-          content:
-            application/json; charset=UTF-8:
-              schema:
-                $ref: '#/components/schemas/Getanitemresponse'
-  /catalogue/size:
-    get:
-      operationId: Get size
-      responses:
-        200:
-          description: ""
-          content:
-            application/json;charset=UTF-8:
-              schema:
-                $ref: '#/components/schemas/Getsizeresponse'
-  /tags:
-    get:
-      operationId: List_
-      responses:
-        200:
-          description: ""
-          content:
-            application/json;charset=UTF-8:
-              schema:
-                $ref: '#/components/schemas/Listresponse3'
-components:
-  schemas:
-    Listresponse:
-      title: List response
-      required:
-      - count
-      - description
-      - id
-      - imageUrl
-      - name
-      - price
-      - tag
-      type: object
-      properties:
-        id:
-          type: string
-        name:
-          type: string
-        description:
-          type: string
-        imageUrl:
-          type: array
-          items:
-            type: string
-        price:
-          type: number
-          format: double
-        count:
-          type: integer
-          format: int32
-        tag:
-          type: array
-          items:
-            type: string
-    Getanitemresponse:
-      title: Get an item response
-      required:
-      - count
-      - description
-      - id
-      - imageUrl
-      - name
-      - price
-      - tag
-      type: object
-      properties:
-        id:
-          type: string
-        name:
-          type: string
-        description:
-          type: string
-        imageUrl:
-          type: array
-          items:
-            type: string
-        price:
-          type: number
-          format: double
-        count:
-          type: integer
-          format: int32
-        tag:
-          type: array
-          items:
-            type: string
-    Getsizeresponse:
-      title: Get size response
-      required:
-      - size
-      type: object
-      properties:
-        size:
-          type: integer
-          format: int32
-    Listresponse3:
-      title: List response3
-      required:
-      - tags
-      type: object
-      properties:
-        tags:
-          type: array
-          items:
-            type: string
-```
-
-Pass it to Mizu through the CLI option: `mizu tap -n sock-shop --contract catalogue.yaml`
-
-Now Mizu will monitor the traffic against these contracts.
-
-If an entry fails to comply with the contract, it's marked with `Breach` notice in the UI.
-The reason of the failure can be seen under the `CONTRACT` tab in the details layout.
-
-### Notes
-
-Make sure that you;
-
-- specified the `openapi` version
-- specified the `info.version` version in the YAML
-- and removed `servers` field from the YAML
-
-Otherwise the OAS file cannot be recognized. (see [this issue](https://github.com/getkin/kin-openapi/issues/356))

docs/POLICY_RULES.md

@@ -1,42 +1,45 @@
 
-# Traffic validation rules
+# API rules validation
 
-This feature allows you to define set of simple rules, and test the traffic against them.
+This feature allows you to define set of simple rules, and test the API against them.
 Such validation may test response for specific JSON fields, headers, etc.
 
 ## Examples
 
-Example 1: HTTP request (REST API call) that didn't pass validation is highlighted in red
+
+Example 1: HTTP request (REST API call) that didn’t pass validation is highlighted in red
 
 ![Simple UI](../assets/validation-example1.png)
 
 - - -
 
+
 Example 2: Details pane shows the validation rule details and whether it passed or failed
 
 ![Simple UI](../assets/validation-example2.png)
 
 
 ## How to use
-
 To use this feature - create simple rules file (see details below) and pass this file as parameter to `mizu tap` command. For example, if rules are stored in file named `rules.yaml` — run the following command:
 
+
 ```shell
-mizu tap --traffic-validation-file rules.yaml
+mizu tap --test-rules rules.yaml PODNAME
 ```
 
 
+
 ## Rules file structure
 
-The structure of the traffic-validation-file is:
+The structure of the test-rules-file is:
 
 * `name`: string, name of the rule
-* `type`: string, type of the rule, must be `json` or `header` or `slo`
+* `type`: string, type of the rule, must be `json` or `header` or `latency`
 * `key`: string, [jsonpath](https://code.google.com/archive/p/jsonpath/wikis/Javascript.wiki) used only in `json` or `header` type
 * `value`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) used only in `json` or `header` type
 * `service`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) service name to filter
 * `path`: string, [regex](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions) URL path to filter
-* `response-time`: integer, time in ms of the expected latency.
+* `latency`: integer, time in ms of the expected latency.
 
 
 ### For example:
@@ -54,12 +57,11 @@ rules:
   key: "Content-Le.*"
   value: "(\\d+(?:\\.\\d+)?)"
 - name: latency-test
-  type: slo
-  response-time: 1
+  type: latency
+  latency: 1
   service: "carts.*"
 ```
 
-
 ### Explanation:
 
 * First rule `holy-in-name-property`:
@@ -72,4 +74,5 @@ rules:
 
 * Third rule `latency-test`:
 
-  > This rule will be applied to all request made to `carts.*` services. If the latency of the response is greater than `1ms` will be marked as failure, marked as success otherwise.
+  > This rule will be applied to all request made to `carts.*` services. If the latency of the response is greater than `1` will be marked as failure, marked as success otherwise.
+

docs/TESTING.md

@@ -1,33 +0,0 @@
-![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg)
-# Testing guidelines
-
-## Generic guidelines
-* Use "[testing](https://pkg.go.dev/testing)" package
-* Write [Table-driven tests using subtests](https://go.dev/blog/subtests)
-* Use cleanup in test/subtest in order to clean up resources
-* Name the test func "Test<tested_func_name><tested_case>"
-
-## Unit tests
-* Position the test file inside the folder of the tested package
-* In case of internal func testing
-  * Name the test file "<tested_file_name>_internal_test.go"
-  * Name the test package same as the package being tested
-  * Example - [Config](../cli/config/config_internal_test.go)
-* In case of exported func testing
-  * Name the test file "<tested_file_name>_test.go"
-  * Name the test package "<tested_package>_test"
-  * Example - [Slice Utils](../cli/mizu/sliceUtils_test.go)
-* Make sure to run test coverage to make sure you covered all the cases and lines in the func  
-  
-## Acceptance tests
-* Position the test file inside the [acceptance tests folder](../acceptanceTests)
-* Name the file "<tested_command>_test.go"
-* Name the package "acceptanceTests"
-* Do not run as part of the short tests
-* Use/Create generic test utils func in acceptanceTests/testsUtils
-* Don't use sleep inside the tests - active check 
-* Running acceptance tests locally
-  * Switch to the branch that is being tested
-  * Run acceptanceTests/setup.sh
-  * Run tests (make acceptance-test)
-* Example - [Tap](../acceptanceTests/tap_test.go)

shared/consts.go

@@ -2,15 +2,10 @@ package shared
 
 const (
 	MizuFilteringOptionsEnvVar       = "SENSITIVE_DATA_FILTERING_OPTIONS"
-	SyncEntriesConfigEnvVar          = "SYNC_ENTRIES_CONFIG"
 	HostModeEnvVar                   = "HOST_MODE"
 	NodeNameEnvVar                   = "NODE_NAME"
 	TappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST"
 	MaxEntriesDBSizeBytesEnvVar      = "MAX_ENTRIES_DB_BYTES"
 	RulePolicyPath                   = "/app/enforce-policy/"
 	RulePolicyFileName               = "enforce-policy.yaml"
-	ContractFileName                 = "contract-oas.yaml"
-	GoGCEnvVar                       = "GOGC"
-	DefaultApiServerPort             = 8899
-	DebugModeEnvVar                  = "MIZU_DEBUG"
 )

shared/debounce/debounce.go

@@ -52,7 +52,3 @@ func (d *Debouncer) SetOn() error {
 	d.timer = time.AfterFunc(d.timeout, d.callback)
 	return nil
 }
-
-func (d *Debouncer) IsOn() bool {
-	return d.running
-}