fixed version check, removed duplicate kube config, fix flags warning, fixed log of invalid config (#227)

* no message

* change local api path

* generic entry list item + rename files and vars

* entry detailed generic

* fix api file

* clean warnings

* switch

* empty lines

* fix scroll to end feature

Co-authored-by: Roee Gadot <roee.gadot@up9.com>

.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# This is a comment.
+# Each line is a file pattern followed by one or more owners.
+
+/ui/ @frontend

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Run mizu <command> '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Logs**
+Upload logs:
+1. Run the mizu command with `--set dump-logs=true` (e.g `mizu tap --set dump-logs=true`) 
+2. Try to reproduce the issue
+3. CNTRL+C on terminal tab which runs mizu
+4. Upload the logs zip file from ~/.mizu/mizu_logs_**.zip
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome]
+
+**Additional context**
+Add any other context about the problem here.

.github/workflows/acceptance_tests.yml

@@ -0,0 +1,26 @@
+name: acceptance tests
+on:
+  pull_request:
+    branches:
+      - 'main'
+  push:
+    branches:
+      - 'develop'
+jobs:
+  run-acceptance-tests:
+    name: Run acceptance tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Setup acceptance test
+        run: source ./acceptanceTests/setup.sh
+
+      - name: Test
+        run: make acceptance-test

.github/workflows/pr_validation.yml

@@ -0,0 +1,80 @@
+name: PR validation
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+jobs:
+  build-cli:
+    name: Build CLI
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Build CLI
+        run: make cli
+
+  build-agent:
+    name: Build Agent
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - shell: bash
+        run: |
+          sudo apt-get install libpcap-dev
+
+      - name: Build Agent
+        run: make agent
+
+  run-tests-cli:
+    name: Run CLI tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: make test-cli
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2
+
+  run-tests-agent:
+    name: Run Agent tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - shell: bash
+        run: |
+          sudo apt-get install libpcap-dev
+
+      - name: Test
+        run: make test-agent
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2

.github/workflows/publish-cli.yml

@@ -1,19 +0,0 @@
-name: public-cli
-on:
-  push:
-    branches:
-      - 'develop'
-      - 'main'
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@master
-        with:
-          service_account_key: ${{ secrets.GCR_JSON_KEY }}
-          export_default_credentials: true
-      - name: Build and Push CLI
-        run: make push-cli

.github/workflows/publish-docker.yml

@@ -1,39 +0,0 @@
-name: publish-docker
-on:
-  push:
-    branches:
-      - 'develop'
-      - 'main'
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get base image name
-        shell: bash
-        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
-        id: base_image_step
-      - name: Docker meta
-        id: meta
-        uses: crazy-max/ghaction-docker-meta@v2
-        with:
-          images: ${{ steps.base_image_step.outputs.image }}
-          tags: |
-            type=sha
-            type=raw,${{ github.sha }}
-            type=raw,latest
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          registry: gcr.io
-          username: _json_key
-          password: ${{ secrets.GCR_JSON_KEY }}
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-

.github/workflows/publish.yml

@@ -0,0 +1,81 @@
+name: publish
+on:
+  push:
+    branches:
+      - 'develop'
+      - 'main'
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
+        with:
+          service_account_key: ${{ secrets.GCR_JSON_KEY }}
+          export_default_credentials: true
+      - uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "minor"
+          if_false: "patch"
+      - name: Auto Increment Semver Action
+        uses: MCKanpolat/auto-semver-action@1.0.5
+        id: versioning
+        with:
+          releaseType: ${{ steps.condval.outputs.value }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+      - name: Get base image name
+        shell: bash
+        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
+        id: base_image_step
+      - name: Docker meta
+        id: meta
+        uses: crazy-max/ghaction-docker-meta@v2
+        with:
+          images: ${{ steps.base_image_step.outputs.image }}
+          tags: |
+            type=sha
+            type=raw,${{ github.sha }}
+            type=raw,${{ steps.versioning.outputs.version }}
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          registry: gcr.io
+          username: _json_key
+          password: ${{ secrets.GCR_JSON_KEY }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            SEM_VER=${{ steps.versioning.outputs.version }}
+            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
+            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
+            COMMIT_HASH=${{ github.sha }}
+      - name: Build and Push CLI
+        run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
+      - shell: bash
+        run: |
+          echo '${{ steps.versioning.outputs.version }}' >> cli/bin/version.txt
+      - name: publish
+        uses: ncipollo/release-action@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          artifacts: "cli/bin/*"
+          commit: ${{ steps.version_parameters.outputs.branch }}
+          tag: ${{ steps.versioning.outputs.version }}
+          prerelease: ${{ github.ref != 'refs/heads/main' }}
+          bodyFile: 'cli/bin/README.md'
+

CONTRIBUTE.md

@@ -0,0 +1,18 @@
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+# CONTRIBUTE
+We welcome code contributions from the community.
+Please read and follow the guidelines below.
+
+## Communication
+* Before starting work on a major feature, please reach out to us via [GitHub](https://github.com/up9inc/mizu), [Slack](https://join.slack.com/share/zt-u6bbs3pg-X1zhQOXOH0yEoqILgH~csw), [email](mailto:mizu@up9.com), etc. We will make sure no one else is already working on it. A _major feature_ is defined as any change that is > 100 LOC altered (not including tests), or changes any user-facing behavior
+* Small patches and bug fixes don't need prior communication.
+
+## Contribution requirements
+* Code style - most of the code is written in Go, please follow [these guidelines](https://golang.org/doc/effective_go)
+* Go-tools compatible (`go get`, `go test`, etc)
+* Unit-test coverage can’t go down ..
+* Code must be usefully commented. Not only for developers on the project, but also for external users of these packages
+* When reviewing PRs, you are encouraged to use Golang's [code review comments page](https://github.com/golang/go/wiki/CodeReviewComments)
+
+
+

Dockerfile

@@ -13,19 +13,30 @@ ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 
 RUN apk add libpcap-dev gcc g++ make
 
-# Move to api working directory (/api-build).
-WORKDIR /app/api-build
+# Move to agent working directory (/agent-build).
+WORKDIR /app/agent-build
 
-COPY api/go.mod api/go.sum ./
+COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
+COPY tap/go.mod tap/go.mod ../tap/
 RUN go mod download
 # cheap trick to make the build faster (As long as go.mod wasn't changes)
 RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
 
-# Copy and build api code
+ARG COMMIT_HASH
+ARG GIT_BRANCH
+ARG BUILD_TIMESTAMP
+ARG SEM_VER
+
+# Copy and build agent code
 COPY shared ../shared
-COPY api .
-RUN go build -ldflags="-s -w" -o mizuagent .
+COPY tap ../tap
+COPY agent .
+RUN go build -ldflags="-s -w \
+     -X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
+     -X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
+     -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
+     -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
 
 FROM alpine:3.13.5
@@ -34,10 +45,11 @@ RUN apk add bash libpcap-dev tcpdump
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
-COPY --from=builder ["/app/api-build/mizuagent", "."]
+COPY --from=builder ["/app/agent-build/mizuagent", "."]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 
-COPY api/start.sh .
+# gin-gonic runs in debug mode without this
+ENV GIN_MODE=release
 
 # this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
 ENTRYPOINT "/app/mizuagent"

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui api cli tap docker
+.PHONY: help ui agent cli tap docker
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -19,34 +19,37 @@ help: ## This help.
 TS_SUFFIX="$(shell date '+%s')"
 GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
 BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
+export SEM_VER?=0.0.0
 
-ui: ## build UI
+ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
 	@ls -l ui/build  
 
-cli: # build CLI
+cli: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build
 
-api: ## build API server
-	@(echo "building API server .." )
-	@(cd api; go build -o build/apiserver main.go)
-	@ls -l api/build
+build-cli-ci: ## Build CLI for CI.
+	@echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci
 
-#tap: ## build tap binary
-#	@(cd tap; go build -o build/tap ./src)
-#	@ls -l tap/build
+agent: ## Build agent.
+	@(echo "building mizu agent .." )
+	@(cd agent; go build -o build/mizuagent main.go)
+	@ls -l agent/build
 
-docker: ## build Docker image 
-	@(echo "building docker image" )
-	./build-push-featurebranch.sh
+docker: ## Build and publish agent docker image.
+	$(MAKE) push-docker
 
-push: push-docker push-cli ## build and publish Mizu docker image & CLI
+push: push-docker push-cli ## Build and publish agent docker image & CLI.
 
-push-docker: 
+push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
 	./build-push-featurebranch.sh
 
-push-cli:
+build-docker-ci: ## Build agent docker image for CI.
+	@echo "building docker image for ci"
+	./build-agent-ci.sh
+
+push-cli: ## Build and publish CLI.
 	@echo "publishing CLI .. "
 	@cd cli; $(MAKE) build-all
 	@echo "publishing file ${OUTPUT_FILE} .."
@@ -54,18 +57,25 @@ push-cli:
 	gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
+clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
 
-clean: clean-ui clean-api clean-cli clean-docker ## Clean all build artifacts
-
-clean-ui: 
+clean-ui: ## Clean UI.
 	@(rm -rf ui/build ; echo "UI cleanup done" )
 
-clean-api: 
-	@(rm -rf api/build ; echo "api cleanup done" )
+clean-agent: ## Clean agent.
+	@(rm -rf agent/build ; echo "agent cleanup done" )
 
-clean-cli: 
+clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
-clean-docker: 
+clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
+test-cli:
+	@echo "running cli tests"; cd cli && $(MAKE) test
+
+test-agent:
+	@echo "running agent tests"; cd agent && $(MAKE) test
+
+acceptance-test:
+	@echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test

PERMISSIONS.md

@@ -0,0 +1,328 @@
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+# Kubernetes permissions for MIZU  
+
+This document describes in details all permissions required for full and correct operation of Mizu
+
+We broke down this list into few categories:
+- Required - what is needed for `mizu` to run properly on your k8s cluster
+- Optional - permissions needed for proper name resolving for service & pod IPs 
+   - addition required for policy validation 
+ 
+
+
+# Required permissions
+
+Mizu needs following permissions on your Kubernetes cluster to run properly
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+```
+
+## Permissions required for service / pod name resolving (opt)
+
+Optionally, for proper resolving of IP addresses to Kubernetes service name, Mizu needs below permissions:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterroles
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterrolebindings
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  - apps
+  - extensions
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+```
+
+## Permissions for Policy rules validation feature (opt)
+
+Optionally, in order to use the policy rules validation feature, Mizu requires the following additional permissions:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - create
+  - delete
+```
+
+- - -
+
+## Namespace-Restricted mode
+
+Alternatively, in order to restrict Mizu to one namespace only (by setting `agent.namespace` in the config file), Mizu needs the following permissions in that namespace:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+```
+
+### Name resolving in Namespace-Restricted mode (opt)
+
+To restrict Mizu to one namespace while also resolving IPs, Mizu needs the following permissions in that namespace:
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+  - create
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  - apps
+  - extensions
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+```

README.md

@@ -1,18 +1,170 @@
-# 水 mizu
-standalone web app traffic viewer for Kubernetes
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+
+# The API Traffic Viewer for Kubernetes
+
+A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined
+
+![Simple UI](assets/mizu-ui.png)
+
+## Features
+
+- Simple and powerful CLI
+- Real-time view of all HTTP requests, REST and gRPC API calls
+- No installation or code instrumentation
+- Works completely on premises
 
 ## Download
 
-Download `mizu` for your platform as
+Download Mizu for your platform and operating system
 
-* for MacOS - `curl -o mizu https://static.up9.com/mizu/mizu-darwin-amd64 &&  chmod 755 mizu`
-* for Linux - `curl -o mizu https://static.up9.com/mizu/mizu-linux-amd64 && chmod 755 mizu`
+### Latest Stable Release
 
-## Run
+* for MacOS - Intel 
+```
+curl -Lo mizu \
+https://github.com/up9inc/mizu/releases/latest/download/mizu_darwin_amd64 \
+&& chmod 755 mizu
+```
+ 
+* for Linux - Intel 64bit
+```
+curl -Lo mizu \
+https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 \
+&& chmod 755 mizu
+``` 
+
+SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page
+
+### Development (unstable) Build
+Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page
+
+## Prerequisites
+1. Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
+2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination)
+
+For detailed list of k8s permissions see [PERMISSIONS](PERMISSIONS.md) document
 
 
+## How to Run
 
-1. Find pod you'd like to tap to in your Kubernetes cluster
-2. Run `mizu --pod podname`
-3. Open browser on `http://localhost:8899` as instructed .. 
-4. Watch the WebAPI traffic flowing ..
+1. Find pods you'd like to tap to in your Kubernetes cluster
+2. Run `mizu tap` or `mizu tap PODNAME`  
+3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI
+4. Watch the API traffic flowing
+5. Type ^C to stop
+
+## Examples
+
+Run `mizu help` for usage options
+
+To tap all pods in current namespace - 
+``` 
+ $ kubectl get pods 
+ NAME                            READY   STATUS    RESTARTS   AGE
+ carts-66c77f5fbb-fq65r          2/2     Running   0          20m
+ catalogue-5f4cb7cf5-7zrmn       2/2     Running   0          20m
+ front-end-649fc5fd6-kqbtn       2/2     Running   0          20m
+ ..
+
+ $ mizu tap
+ +carts-66c77f5fbb-fq65r
+ +catalogue-5f4cb7cf5-7zrmn
+ +front-end-649fc5fd6-kqbtn
+ Web interface is now available at http://localhost:8899
+ ^C
+```
+
+
+To tap specific pod - 
+```bash
+ $ kubectl get pods 
+ NAME                            READY   STATUS    RESTARTS   AGE
+ front-end-649fc5fd6-kqbtn       2/2     Running   0          7m
+ ..
+
+ $ mizu tap front-end-649fc5fd6-kqbtn
+ +front-end-649fc5fd6-kqbtn
+ Web interface is now available at http://localhost:8899
+ ^C
+```
+
+To tap multiple pods using regex - 
+```bash
+ $ kubectl get pods 
+ NAME                            READY   STATUS    RESTARTS   AGE
+ carts-66c77f5fbb-fq65r          2/2     Running   0          20m
+ catalogue-5f4cb7cf5-7zrmn       2/2     Running   0          20m
+ front-end-649fc5fd6-kqbtn       2/2     Running   0          20m
+ ..
+
+ $ mizu tap "^ca.*"
+ +carts-66c77f5fbb-fq65r
+ +catalogue-5f4cb7cf5-7zrmn
+ Web interface is now available at http://localhost:8899
+ ^C
+```
+
+## Configuration
+
+Mizu can work with config file which should be stored in ${HOME}/.mizu/config.yaml (macOS: ~/.mizu/config.yaml) <br />
+In case no config file found, defaults will be used <br />
+In case of partial configuration defined, all other fields will be used with defaults <br />
+You can always override the defaults or config file with CLI flags
+
+To get the default config params run `mizu config` <br />
+To generate a new config file with default values use `mizu config -r`
+
+### Telemetry
+
+By default, mizu reports usage telemetry. It can be disabled by adding a line of `telemetry: false` in the `${HOME}/.mizu/config.yaml` file
+
+
+## Advanced Usage
+
+### Namespace-Restricted Mode
+
+Some users have permission to only manage resources in one particular namespace assigned to them
+By default `mizu tap` creates a new namespace `mizu` for all of its Kubernetes resources. In order to instead install
+Mizu in an existing namespace, set the `mizu-resources-namespace` config option
+
+If `mizu-resources-namespace` is set to a value other than the default `mizu`, Mizu will operate in a
+Namespace-Restricted mode. It will only tap pods in `mizu-resources-namespace`. This way Mizu only requires permissions
+to the namespace set by `mizu-resources-namespace`. The user must set the tapped namespace to the same namespace by
+using the `--namespace` flag or by setting `tap.namespaces` in the config file
+
+Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
+
+### User agent filtering
+
+User-agent filtering (like health checks) - can be configured using command-line options:
+
+```shell
+$ mizu tap "^ca.*" --set ignored-user-agents=kube-probe --set ignored-user-agents=prometheus
++carts-66c77f5fbb-fq65r
++catalogue-5f4cb7cf5-7zrmn
+Web interface is now available at http://localhost:8899
+^C
+
+```
+
+Any request that contains `User-Agent` header with one of the specified values (`kube-probe` or `prometheus`) will not be captured
+
+### API Rules validation
+
+This feature allows you to define set of simple rules, and test the API against them.
+Such validation may test response for specific JSON fields, headers, etc.
+
+Please see [API RULES](docs/POLICY_RULES.md) page for more details and syntax.
+
+
+## How to Run local UI
+
+- run from mizu/agent `go run main.go --hars-read --hars-dir <folder>`
+
+- copy Har files into the folder from last command
+
+- change `MizuWebsocketURL` and `apiURL` in `api.js` file
+
+- run from mizu/ui - `npm run start`
+
+- open browser on `localhost:3000`

TESTING.md

@@ -0,0 +1,15 @@
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+# TESTING
+Testing guidelines for Mizu project 
+
+## Unit-tests
+* TBD 
+* TBD
+* TBD
+
+
+
+## System tests
+* TBD 
+* TBD
+* TBD

acceptanceTests/Makefile

@@ -0,0 +1,2 @@
+test: ## Run acceptance tests.
+	@go test ./...

acceptanceTests/go.mod

@@ -0,0 +1,3 @@
+module github.com/up9inc/mizu/tests
+
+go 1.16

acceptanceTests/setup.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+PREFIX=$HOME/local/bin
+VERSION=v1.22.0
+
+echo "Attempting to install minikube and assorted tools to $PREFIX"
+
+if ! [ -x "$(command -v kubectl)" ]; then
+  echo "Installing kubectl version $VERSION"
+  curl -LO "https://storage.googleapis.com/kubernetes-release/release/$VERSION/bin/linux/amd64/kubectl"
+  chmod +x kubectl
+  mv kubectl "$PREFIX"
+else
+  echo "kubetcl is already installed"
+fi
+
+if ! [ -x "$(command -v minikube)" ]; then
+  echo "Installing minikube version $VERSION"
+  curl -Lo minikube https://storage.googleapis.com/minikube/releases/$VERSION/minikube-linux-amd64
+  chmod +x minikube
+  mv minikube "$PREFIX"
+else
+  echo "minikube is already installed"
+fi
+
+echo "Starting minikube..."
+minikube start
+
+echo "Creating mizu tests namespace"
+kubectl create namespace mizu-tests
+
+echo "Creating httpbin deployment"
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests
+
+echo "Creating httpbin service"
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
+
+echo "Starting proxy"
+kubectl proxy --port=8080 &
+
+echo "Setting minikube docker env"
+eval $(minikube docker-env)
+
+echo "Build agent image"
+make build-docker-ci
+
+echo "Build cli"
+make build-cli-ci

acceptanceTests/tap_test.go

@@ -0,0 +1,125 @@
+package acceptanceTests
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+	"testing"
+	"time"
+)
+
+func TestTapAndFetch(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []int{1, 100}
+
+	for _, entriesCount := range tests {
+		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
+			cliPath, cliPathErr := GetCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			tapCmdArgs := GetDefaultTapCommandArgs()
+			tapCmd := exec.Command(cliPath, tapCmdArgs...)
+			t.Logf("running command: %v", tapCmd.String())
+
+			t.Cleanup(func() {
+				if err := CleanupCommand(tapCmd); err != nil {
+					t.Logf("failed to cleanup tap command, err: %v", err)
+				}
+			})
+
+			if err := tapCmd.Start(); err != nil {
+				t.Errorf("failed to start tap command, err: %v", err)
+				return
+			}
+
+			time.Sleep(30 * time.Second)
+
+			proxyUrl := "http://localhost:8080/api/v1/namespaces/mizu-tests/services/httpbin/proxy/get"
+			for i := 0; i < entriesCount; i++ {
+				if _, requestErr := ExecuteHttpRequest(proxyUrl); requestErr != nil {
+					t.Errorf("failed to send proxy request, err: %v", requestErr)
+					return
+				}
+			}
+
+			time.Sleep(5 * time.Second)
+			timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+			entriesUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries?limit=%v&operator=lt&timestamp=%v", entriesCount, timestamp)
+			requestResult, requestErr := ExecuteHttpRequest(entriesUrl)
+			if requestErr != nil {
+				t.Errorf("failed to get entries, err: %v", requestErr)
+				return
+			}
+
+			entries, ok := requestResult.([]interface{})
+			if !ok {
+				t.Errorf("invalid entries type")
+				return
+			}
+
+			if len(entries) != entriesCount {
+				t.Errorf("unexpected entries result - Expected: %v, actual: %v", entriesCount, len(entries))
+				return
+			}
+
+			entry, ok :=  entries[0].(map[string]interface{})
+			if !ok {
+				t.Errorf("invalid entry type")
+				return
+			}
+
+			entryUrl := fmt.Sprintf("http://localhost:8899/mizu/api/entries/%v", entry["id"])
+			requestResult, requestErr = ExecuteHttpRequest(entryUrl)
+			if requestErr != nil {
+				t.Errorf("failed to get entry, err: %v", requestErr)
+				return
+			}
+
+			if requestResult == nil {
+				t.Errorf("unexpected nil entry result")
+				return
+			}
+
+			fetchCmdArgs := GetDefaultFetchCommandArgs()
+			fetchCmd := exec.Command(cliPath, fetchCmdArgs...)
+			t.Logf("running command: %v", fetchCmd.String())
+
+			t.Cleanup(func() {
+				if err := CleanupCommand(fetchCmd); err != nil {
+					t.Logf("failed to cleanup fetch command, err: %v", err)
+				}
+			})
+
+			if err := fetchCmd.Start(); err != nil {
+				t.Errorf("failed to start fetch command, err: %v", err)
+				return
+			}
+
+			time.Sleep(5 * time.Second)
+
+			harBytes, readFileErr := ioutil.ReadFile("./unknown_source.har")
+			if readFileErr != nil {
+				t.Errorf("failed to read har file, err: %v", readFileErr)
+				return
+			}
+
+			harEntries, err := GetEntriesFromHarBytes(harBytes)
+			if err != nil {
+				t.Errorf("failed to get entries from har, err: %v", err)
+				return
+			}
+
+			if len(harEntries) != entriesCount {
+				t.Errorf("unexpected har entries result - Expected: %v, actual: %v", entriesCount, len(harEntries))
+				return
+			}
+		})
+	}
+}

acceptanceTests/testsUtils.go

@@ -0,0 +1,113 @@
+package acceptanceTests
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"syscall"
+)
+
+func GetCliPath() (string, error) {
+	dir, filePathErr := os.Getwd()
+	if filePathErr != nil {
+		return "", filePathErr
+	}
+
+	cliPath := path.Join(dir, "../cli/bin/mizu_ci")
+	return cliPath, nil
+}
+
+func GetDefaultCommandArgs() []string {
+	setFlag := "--set"
+	telemetry := "telemetry=false"
+
+	return []string{setFlag, telemetry}
+}
+
+func GetDefaultTapCommandArgs() []string {
+	tapCommand := "tap"
+	setFlag := "--set"
+	namespaces := "tap.namespaces=mizu-tests"
+	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
+	imagePullPolicy := "image-pull-policy=Never"
+
+	defaultCmdArgs := GetDefaultCommandArgs()
+
+	return append([]string{tapCommand, setFlag, namespaces, setFlag, agentImage, setFlag, imagePullPolicy}, defaultCmdArgs...)
+}
+
+func GetDefaultFetchCommandArgs() []string {
+	tapCommand := "fetch"
+
+	defaultCmdArgs := GetDefaultCommandArgs()
+
+	return append([]string{tapCommand}, defaultCmdArgs...)
+}
+
+func JsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
+	var result interface{}
+	if parseErr := json.Unmarshal(jsonBytes, &result); parseErr != nil {
+		return nil, parseErr
+	}
+
+	return result, nil
+}
+
+func ExecuteHttpRequest(url string) (interface{}, error) {
+	response, requestErr := http.Get(url)
+	if requestErr != nil {
+		return nil, requestErr
+	} else if response.StatusCode != 200 {
+		return nil, fmt.Errorf("invalid status code %v", response.StatusCode)
+	}
+
+	data, readErr := ioutil.ReadAll(response.Body)
+	if readErr != nil {
+		return nil, readErr
+	}
+
+	return JsonBytesToInterface(data)
+}
+
+func CleanupCommand(cmd *exec.Cmd) error {
+	if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
+		return err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func GetEntriesFromHarBytes(harBytes []byte) ([]interface{}, error){
+	harInterface, convertErr := JsonBytesToInterface(harBytes)
+	if convertErr != nil {
+		return nil, convertErr
+	}
+
+	har, ok := harInterface.(map[string]interface{})
+	if !ok {
+		return nil, errors.New("invalid har type")
+	}
+
+	harLogInterface := har["log"]
+	harLog, ok :=  harLogInterface.(map[string]interface{})
+	if !ok {
+		return nil, errors.New("invalid har log type")
+	}
+
+	harEntriesInterface := harLog["entries"]
+	harEntries, ok := harEntriesInterface.([]interface{})
+	if !ok {
+		return nil, errors.New("invalid har entries type")
+	}
+
+	return harEntries, nil
+}

agent/Makefile

@@ -0,0 +1,2 @@
+test: ## Run agent tests.
+	@go test ./... -coverpkg=./... -race -coverprofile=coverage.out -covermode=atomic

agent/README.md

@@ -1,5 +1,5 @@
-# mizu API server
-API server for MIZU
+# mizu agent
+Agent for MIZU (API server and tapper)
 Basic APIs:
 * /fetch - retrieve traffic data
 * /stats - retrieve statistics of collected data
@@ -14,7 +14,7 @@ Basic APIs:
 
 ### Connecting
 1. Start mizu using the cli with the debug image `mizu tap --mizu-image gcr.io/up9-docker-hub/mizu/debug:latest {tapped_pod_name}`
-2. Forward the debug port using `kubectl port-forward -n default mizu-collector 2345:2345`
+2. Forward the debug port using `kubectl port-forward -n default mizu-api-server 2345:2345`
 3. Run the run/debug configuration you've created earlier in Intellij.
 
 <small>Do note that dlv won't start the api until a debugger connects to it.</small>

agent/go.mod

@@ -3,23 +3,23 @@ module mizuserver
 go 1.16
 
 require (
-	github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a
 	github.com/beevik/etree v1.1.0
 	github.com/djherbis/atime v1.0.0
-	github.com/fasthttp/websocket v1.4.3-beta.1 // indirect
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/gin-contrib/static v0.0.1
+	github.com/gin-gonic/gin v1.7.2
 	github.com/go-playground/locales v0.13.0
 	github.com/go-playground/universal-translator v0.17.0
 	github.com/go-playground/validator/v10 v10.5.0
-	github.com/gofiber/fiber/v2 v2.8.0
-	github.com/google/gopacket v1.1.19
 	github.com/google/martian v2.1.0+incompatible
 	github.com/gorilla/websocket v1.4.2
-	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
 	github.com/up9inc/mizu/shared v0.0.0
+	github.com/up9inc/mizu/tap v0.0.0
+	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
 	go.mongodb.org/mongo-driver v1.5.1
-	golang.org/x/net v0.0.0-20210421230115-4e50805a0758
 	gorm.io/driver/sqlite v1.1.4
 	gorm.io/gorm v1.21.8
 	k8s.io/api v0.21.0
@@ -28,3 +28,5 @@ require (
 )
 
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
+
+replace github.com/up9inc/mizu/tap v0.0.0 => ../tap

agent/go.sum

@@ -41,15 +41,12 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
-github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc=
-github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
-github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a h1:76llBleIE3fkdqaJFDzdirtiYhQPdIQem8H8r2iwA1Q=
-github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a/go.mod h1:QvDfsDQDmGxUsvEeWabVZ5pp2FMXpOkwQV0L6SE6cp0=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
 github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
 github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
+github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M=
+github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -61,18 +58,26 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg=
 github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8=
+github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y=
-github.com/fasthttp/websocket v1.4.3-beta.1 h1:stc4P2aoxYKsdmbe1AJ5mAm73Fxc1NOgrZpPftvZIXQ=
-github.com/fasthttp/websocket v1.4.3-beta.1/go.mod h1:JGrgLaT02bL9NuJkZbHN8mVV2tkCJZQh7yJ5/XCXO2g=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-contrib/static v0.0.1 h1:JVxuvHPuUfkoul12N7dtQw7KRn/pSMq7Ue1Va9Swm1U=
+github.com/gin-contrib/static v0.0.1/go.mod h1:CSxeF+wep05e0kCOsqWdAWbSszmc31zTIbD8TvWl7Hs=
+github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/gin-gonic/gin v1.7.2 h1:Tg03T9yM2xa8j6I3Z3oqLaQRSmKvxPd6g/2HJ6zICFA=
+github.com/gin-gonic/gin v1.7.2/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -92,6 +97,8 @@ github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8c
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E=
 github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
@@ -120,12 +127,6 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe
 github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
 github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
-github.com/gofiber/fiber/v2 v2.1.3/go.mod h1:MMiSv1HrDkN8Pv7NeVDYK+T/lwXOEKAvPBbLvJPCEfA=
-github.com/gofiber/fiber/v2 v2.7.1/go.mod h1:f8BRRIMjMdRyt2qmJ/0Sea3j3rwwfufPrh9WNBRiVZ0=
-github.com/gofiber/fiber/v2 v2.8.0 h1:BdWvZmg/WY/Vjtjm38aXOp1Lks1BhuyS2b7lSWSPAzk=
-github.com/gofiber/fiber/v2 v2.8.0/go.mod h1:Ah3IJikrKNRepl/HuVawppS25X7FWohwfCSRn7kJG28=
-github.com/gofiber/websocket/v2 v2.0.3 h1:nqPGHB4LQhxKX5KJUjayOd2xiiENieS/dn6TPfCL8uk=
-github.com/gofiber/websocket/v2 v2.0.3/go.mod h1:/OTEImCxORKE5unw0dWqJYovid6vZF+wB1W0aaMKs2M=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -186,7 +187,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
@@ -198,6 +198,7 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -206,13 +207,7 @@ github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaR
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4=
-github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -222,13 +217,14 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
-github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
-github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ=
 github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -264,9 +260,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY=
-github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c h1:KKqhycXW1WVNkX7r4ekTV2gFkbhdyihlWD8c0/FiWmk=
-github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8=
+github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
+github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -278,28 +273,22 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
-github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
-github.com/valyala/fasthttp v1.15.1/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
-github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
-github.com/valyala/fasthttp v1.18.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A=
-github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5NcFs=
-github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU=
-github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc=
-github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
+github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
+github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -372,11 +361,8 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758 h1:aEpZnXcAmXkd6AvLb2OPt+EN1Zu/8Ne3pCqPjja5PXY=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -410,21 +396,20 @@ golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ=
 golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -555,8 +540,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=
 gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw=
 gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=

agent/main.go

@@ -0,0 +1,234 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"github.com/gin-contrib/static"
+	"github.com/gin-gonic/gin"
+	"github.com/gorilla/websocket"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap"
+	"mizuserver/pkg/api"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/routes"
+	"mizuserver/pkg/sensitiveDataFiltering"
+	"mizuserver/pkg/utils"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+)
+
+var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API")
+var apiServerMode = flag.Bool("api-server", false, "Run in API server mode with API")
+var standaloneMode = flag.Bool("standalone", false, "Run in standalone tapper and API mode")
+var apiServerAddress = flag.String("api-server-address", "", "Address of mizu API server")
+var namespace = flag.String("namespace", "", "Resolve IPs if they belong to resources in this namespace (default is all)")
+var harsReaderMode = flag.Bool("hars-read", false, "Run in hars-read mode")
+var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
+
+func main() {
+	flag.Parse()
+	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+	tapOpts := &tap.TapOpts{HostMode: hostMode}
+
+
+	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode{
+		panic("One of the flags --tap, --api or --standalone or --hars-read must be provided")
+	}
+
+	if *standaloneMode {
+		api.StartResolving(*namespace)
+
+		harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts)
+		filteredHarChannel := make(chan *tap.OutputChannelItem)
+
+		go filterHarItems(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions())
+		go api.StartReadingEntries(filteredHarChannel, nil)
+		go api.StartReadingOutbound(outboundLinkOutputChannel)
+
+		hostApi(nil)
+	} else if *tapperMode {
+		if *apiServerAddress == "" {
+			panic("API server address must be provided with --api-server-address when using --tap")
+		}
+
+		tapTargets := getTapTargets()
+		if tapTargets != nil {
+			tap.SetFilterAuthorities(tapTargets)
+			rlog.Infof("Filtering for the following authorities: %v", tap.GetFilterIPs())
+		}
+
+		harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts)
+
+		socketConnection, err := shared.ConnectToSocketServer(*apiServerAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false)
+		if err != nil {
+			panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
+		}
+
+		go pipeTapChannelToSocket(socketConnection, harOutputChannel)
+		go pipeOutboundLinksChannelToSocket(socketConnection, outboundLinkOutputChannel)
+	} else if *apiServerMode {
+		api.StartResolving(*namespace)
+
+		socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000)
+		filteredHarChannel := make(chan *tap.OutputChannelItem)
+
+		go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions())
+		go api.StartReadingEntries(filteredHarChannel, nil)
+
+		hostApi(socketHarOutChannel)
+	} else if *harsReaderMode {
+		socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000)
+		filteredHarChannel := make(chan *tap.OutputChannelItem)
+
+		go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions())
+		go api.StartReadingEntries(filteredHarChannel, harsDir)
+		hostApi(nil)
+	}
+
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, os.Interrupt)
+	<-signalChan
+
+	rlog.Info("Exiting")
+}
+
+func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
+	app := gin.Default()
+
+	app.GET("/echo", func(c *gin.Context) {
+		c.String(http.StatusOK, "Here is Mizu agent")
+	})
+
+	eventHandlers := api.RoutesEventHandlers{
+		SocketHarOutChannel: socketHarOutputChannel,
+	}
+
+	app.Use(static.ServeRoot("/", "./site"))
+	app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
+
+	api.WebSocketRoutes(app, &eventHandlers)
+	routes.EntriesRoutes(app)
+	routes.MetadataRoutes(app)
+	routes.StatusRoutes(app)
+	routes.NotFoundRoute(app)
+
+	utils.StartServer(app)
+}
+
+func CORSMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
+		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
+		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
+		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")
+
+		if c.Request.Method == "OPTIONS" {
+			c.AbortWithStatus(204)
+			return
+		}
+
+		c.Next()
+	}
+}
+
+func getTapTargets() []string {
+	nodeName := os.Getenv(shared.NodeNameEnvVar)
+	var tappedAddressesPerNodeDict map[string][]string
+	err := json.Unmarshal([]byte(os.Getenv(shared.TappedAddressesPerNodeDictEnvVar)), &tappedAddressesPerNodeDict)
+	if err != nil {
+		panic(fmt.Sprintf("env var %s's value of %s is invalid! must be map[string][]string %v", shared.TappedAddressesPerNodeDictEnvVar, tappedAddressesPerNodeDict, err))
+	}
+	return tappedAddressesPerNodeDict[nodeName]
+}
+
+func getTrafficFilteringOptions() *shared.TrafficFilteringOptions {
+	filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar)
+	if filteringOptionsJson == "" {
+		return nil
+	}
+	var filteringOptions shared.TrafficFilteringOptions
+	err := json.Unmarshal([]byte(filteringOptionsJson), &filteringOptions)
+	if err != nil {
+		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.TrafficFilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err))
+	}
+
+	return &filteringOptions
+}
+
+func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) {
+	for message := range inChannel {
+		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
+			continue
+		}
+		// TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441
+		if isHealthCheckByUserAgent(message, filterOptions.HealthChecksUserAgentHeaders) {
+			continue
+		}
+
+		if !filterOptions.DisableRedaction {
+			sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions)
+		}
+
+		outChannel <- message
+	}
+}
+
+func isHealthCheckByUserAgent(message *tap.OutputChannelItem, userAgentsToIgnore []string) bool {
+	for _, header := range message.HarEntry.Request.Headers {
+		if strings.ToLower(header.Name) == "user-agent" {
+			for _, userAgent := range userAgentsToIgnore {
+				if strings.Contains(strings.ToLower(header.Value), strings.ToLower(userAgent)) {
+					return true
+				}
+			}
+			return false
+		}
+	}
+	return false
+}
+
+func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) {
+	if connection == nil {
+		panic("Websocket connection is nil")
+	}
+
+	if messageDataChannel == nil {
+		panic("Channel of captured messages is nil")
+	}
+
+	for messageData := range messageDataChannel {
+		marshaledData, err := models.CreateWebsocketTappedEntryMessage(messageData)
+		if err != nil {
+			rlog.Infof("error converting message to json %s, (%v,%+v)\n", err, err, err)
+			continue
+		}
+
+		err = connection.WriteMessage(websocket.TextMessage, marshaledData)
+		if err != nil {
+			rlog.Infof("error sending message through socket server %s, (%v,%+v)\n", err, err, err)
+			continue
+		}
+	}
+}
+
+func pipeOutboundLinksChannelToSocket(connection *websocket.Conn, outboundLinkChannel <-chan *tap.OutboundLink) {
+	for outboundLink := range outboundLinkChannel {
+		if outboundLink.SuggestedProtocol == tap.TLSProtocol {
+			marshaledData, err := models.CreateWebsocketOutboundLinkMessage(outboundLink)
+			if err != nil {
+				rlog.Infof("Error converting outbound link to json %s, (%v,%+v)", err, err, err)
+				continue
+			}
+
+			err = connection.WriteMessage(websocket.TextMessage, marshaledData)
+			if err != nil {
+				rlog.Infof("error sending outbound link message through socket server %s, (%v,%+v)", err, err, err)
+				continue
+			}
+		}
+	}
+}

agent/pkg/api/main.go

@@ -0,0 +1,204 @@
+package api
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"mizuserver/pkg/holder"
+	"mizuserver/pkg/providers"
+	"net/url"
+	"os"
+	"path"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/google/martian/har"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/tap"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+
+	"mizuserver/pkg/database"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/resolver"
+	"mizuserver/pkg/utils"
+)
+
+var k8sResolver *resolver.Resolver
+
+func StartResolving(namespace string) {
+	errOut := make(chan error, 100)
+	res, err := resolver.NewFromInCluster(errOut, namespace)
+	if err != nil {
+		rlog.Infof("error creating k8s resolver %s", err)
+		return
+	}
+	ctx := context.Background()
+	res.Start(ctx)
+	go func() {
+		for {
+			select {
+			case err := <-errOut:
+				rlog.Infof("name resolving error %s", err)
+			}
+		}
+	}()
+
+	k8sResolver = res
+	holder.SetResolver(res)
+}
+
+func StartReadingEntries(harChannel <-chan *tap.OutputChannelItem, workingDir *string) {
+	if workingDir != nil && *workingDir != "" {
+		startReadingFiles(*workingDir)
+	} else {
+		startReadingChannel(harChannel)
+	}
+}
+
+func startReadingFiles(workingDir string) {
+	err := os.MkdirAll(workingDir, os.ModePerm)
+	utils.CheckErr(err)
+
+	for true {
+		dir, _ := os.Open(workingDir)
+		dirFiles, _ := dir.Readdir(-1)
+
+		var harFiles []os.FileInfo
+		for _, fileInfo := range dirFiles {
+			if strings.HasSuffix(fileInfo.Name(), ".har") {
+				harFiles = append(harFiles, fileInfo)
+			}
+		}
+		sort.Sort(utils.ByModTime(harFiles))
+
+		if len(harFiles) == 0 {
+			rlog.Infof("Waiting for new files\n")
+			time.Sleep(3 * time.Second)
+			continue
+		}
+		fileInfo := harFiles[0]
+		inputFilePath := path.Join(workingDir, fileInfo.Name())
+		file, err := os.Open(inputFilePath)
+		utils.CheckErr(err)
+
+		var inputHar har.HAR
+		decErr := json.NewDecoder(bufio.NewReader(file)).Decode(&inputHar)
+		utils.CheckErr(decErr)
+
+		for _, entry := range inputHar.Log.Entries {
+			time.Sleep(time.Millisecond * 250)
+			connectionInfo := &tap.ConnectionInfo{
+				ClientIP:   fileInfo.Name(),
+				ClientPort: "",
+				ServerIP:   "",
+				ServerPort: "",
+				IsOutgoing: false,
+			}
+			saveHarToDb(entry, connectionInfo)
+		}
+		rmErr := os.Remove(inputFilePath)
+		utils.CheckErr(rmErr)
+	}
+}
+
+func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) {
+	if outputItems == nil {
+		panic("Channel of captured messages is nil")
+	}
+
+	for item := range outputItems {
+		providers.EntryAdded()
+		saveHarToDb(item.HarEntry, item.ConnectionInfo)
+	}
+}
+
+func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) {
+	// tcpStreamFactory will block on write to channel. Empty channel to unblock.
+	// TODO: Make write to channel optional.
+	for range outboundLinkChannel {
+	}
+}
+
+func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
+	entryBytes, _ := json.Marshal(entry)
+	serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL)
+	entryId := primitive.NewObjectID().Hex()
+	var (
+		resolvedSource      string
+		resolvedDestination string
+	)
+	if k8sResolver != nil {
+		unresolvedSource := connectionInfo.ClientIP
+		resolvedSource = k8sResolver.Resolve(unresolvedSource)
+		if resolvedSource == "" {
+			rlog.Debugf("Cannot find resolved name to source: %s\n", unresolvedSource)
+			if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
+				return
+			}
+		}
+		unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
+		resolvedDestination = k8sResolver.Resolve(unresolvedDestination)
+		if resolvedDestination == "" {
+			rlog.Debugf("Cannot find resolved name to dest: %s\n", unresolvedDestination)
+			if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
+				return
+			}
+		}
+	}
+
+	mizuEntry := models.MizuEntry{
+		EntryId:             entryId,
+		Entry:               string(entryBytes), // simple way to store it and not convert to bytes
+		Service:             serviceName,
+		Url:                 entry.Request.URL,
+		Path:                urlPath,
+		Method:              entry.Request.Method,
+		Status:              entry.Response.Status,
+		RequestSenderIp:     connectionInfo.ClientIP,
+		Timestamp:           entry.StartedDateTime.UnixNano() / int64(time.Millisecond),
+		ResolvedSource:      resolvedSource,
+		ResolvedDestination: resolvedDestination,
+		IsOutgoing:          connectionInfo.IsOutgoing,
+	}
+	mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
+	database.CreateEntry(&mizuEntry)
+
+	baseEntry := models.BaseEntryDetails{}
+	if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil {
+		return
+	}
+	baseEntry.Rules = models.RunValidationRulesState(*entry, serviceName)
+	baseEntry.Latency = entry.Timings.Receive
+	baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry)
+	BroadcastToBrowserClients(baseEntryBytes)
+}
+
+func getServiceNameFromUrl(inputUrl string) (string, string) {
+	parsed, err := url.Parse(inputUrl)
+	utils.CheckErr(err)
+	return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path
+}
+
+func CheckIsServiceIP(address string) bool {
+	return k8sResolver.CheckIsServiceIP(address)
+}
+
+// gives a rough estimate of the size this will take up in the db, good enough for maintaining db size limit accurately
+func getEstimatedEntrySizeBytes(mizuEntry models.MizuEntry) int {
+	sizeBytes := len(mizuEntry.Entry)
+	sizeBytes += len(mizuEntry.EntryId)
+	sizeBytes += len(mizuEntry.Service)
+	sizeBytes += len(mizuEntry.Url)
+	sizeBytes += len(mizuEntry.Method)
+	sizeBytes += len(mizuEntry.RequestSenderIp)
+	sizeBytes += len(mizuEntry.ResolvedDestination)
+	sizeBytes += len(mizuEntry.ResolvedSource)
+	sizeBytes += 8 // Status bytes (sqlite integer is always 8 bytes)
+	sizeBytes += 8 // Timestamp bytes
+	sizeBytes += 8 // SizeBytes bytes
+	sizeBytes += 1 // IsOutgoing bytes
+
+	return sizeBytes
+}

agent/pkg/api/socket_routes.go

@@ -0,0 +1,118 @@
+package api
+
+import (
+	"errors"
+	"github.com/gin-gonic/gin"
+	"github.com/gorilla/websocket"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared/debounce"
+	"net/http"
+	"sync"
+	"time"
+)
+
+type EventHandlers interface {
+	WebSocketConnect(socketId int, isTapper bool)
+	WebSocketDisconnect(socketId int, isTapper bool)
+	WebSocketMessage(socketId int, message []byte)
+}
+
+type SocketConnection struct {
+	connection    *websocket.Conn
+	lock          *sync.Mutex
+	eventHandlers EventHandlers
+	isTapper      bool
+}
+
+var websocketUpgrader = websocket.Upgrader{
+	ReadBufferSize:  1024,
+	WriteBufferSize: 1024,
+}
+
+var websocketIdsLock = sync.Mutex{}
+var connectedWebsockets map[int]*SocketConnection
+var connectedWebsocketIdCounter = 0
+
+func init() {
+	websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
+	connectedWebsockets = make(map[int]*SocketConnection, 0)
+}
+
+func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
+	app.GET("/ws", func(c *gin.Context) {
+		websocketHandler(c.Writer, c.Request, eventHandlers, false)
+	})
+	app.GET("/wsTapper", func(c *gin.Context) {
+		websocketHandler(c.Writer, c.Request, eventHandlers, true)
+	})
+}
+
+func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) {
+	conn, err := websocketUpgrader.Upgrade(w, r, nil)
+	if err != nil {
+		rlog.Errorf("Failed to set websocket upgrade: %v", err)
+		return
+	}
+
+	websocketIdsLock.Lock()
+
+	connectedWebsocketIdCounter++
+	socketId := connectedWebsocketIdCounter
+	connectedWebsockets[socketId] = &SocketConnection{connection: conn, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
+
+	websocketIdsLock.Unlock()
+
+	defer func() {
+		socketCleanup(socketId, connectedWebsockets[socketId])
+	}()
+
+	eventHandlers.WebSocketConnect(socketId, isTapper)
+
+	for {
+		_, msg, err := conn.ReadMessage()
+		if err != nil {
+			rlog.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
+			break
+		}
+		eventHandlers.WebSocketMessage(socketId, msg)
+	}
+}
+
+func socketCleanup(socketId int, socketConnection *SocketConnection) {
+	err := socketConnection.connection.Close()
+	if err != nil {
+		rlog.Errorf("Error closing socket connection for socket id %d: %v\n", socketId, err)
+	}
+
+	websocketIdsLock.Lock()
+	connectedWebsockets[socketId] = nil
+	websocketIdsLock.Unlock()
+
+	socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
+}
+
+var db = debounce.NewDebouncer(time.Second*5, func() {
+	rlog.Error("Successfully sent to socket")
+})
+
+func SendToSocket(socketId int, message []byte) error {
+	socketObj := connectedWebsockets[socketId]
+	if socketObj == nil {
+		return errors.New("Socket is disconnected")
+	}
+
+	var sent = false
+	time.AfterFunc(time.Second*5, func() {
+		if !sent {
+			rlog.Error("Socket timed out")
+			socketCleanup(socketId, socketObj)
+		}
+	})
+
+	socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
+	err := socketObj.connection.WriteMessage(1, message)
+	socketObj.lock.Unlock()
+
+	sent = true
+	return err
+}

agent/pkg/api/socket_server_handlers.go

@@ -0,0 +1,130 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/up9"
+	"sync"
+
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap"
+)
+
+var browserClientSocketUUIDs = make([]int, 0)
+var socketListLock = sync.Mutex{}
+
+type RoutesEventHandlers struct {
+	EventHandlers
+	SocketHarOutChannel chan<- *tap.OutputChannelItem
+}
+
+func init() {
+	go up9.UpdateAnalyzeStatus(BroadcastToBrowserClients)
+}
+
+func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
+	if isTapper {
+		rlog.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
+	} else {
+		rlog.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
+		socketListLock.Lock()
+		browserClientSocketUUIDs = append(browserClientSocketUUIDs, socketId)
+		socketListLock.Unlock()
+	}
+}
+
+func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
+	if isTapper {
+		rlog.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
+	} else {
+		rlog.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
+		socketListLock.Lock()
+		removeSocketUUIDFromBrowserSlice(socketId)
+		socketListLock.Unlock()
+	}
+}
+
+func BroadcastToBrowserClients(message []byte) {
+	for _, socketId := range browserClientSocketUUIDs {
+		go func(socketId int) {
+			err := SendToSocket(socketId, message)
+			if err != nil {
+				rlog.Errorf("error sending message to socket ID %d: %v", socketId, err)
+			}
+		}(socketId)
+	}
+}
+
+func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
+	var socketMessageBase shared.WebSocketMessageMetadata
+	err := json.Unmarshal(message, &socketMessageBase)
+	if err != nil {
+		rlog.Infof("Could not unmarshal websocket message %v\n", err)
+	} else {
+		switch socketMessageBase.MessageType {
+		case shared.WebSocketMessageTypeTappedEntry:
+			var tappedEntryMessage models.WebSocketTappedEntryMessage
+			err := json.Unmarshal(message, &tappedEntryMessage)
+			if err != nil {
+				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+			} else {
+				h.SocketHarOutChannel <- tappedEntryMessage.Data
+			}
+		case shared.WebSocketMessageTypeUpdateStatus:
+			var statusMessage shared.WebSocketStatusMessage
+			err := json.Unmarshal(message, &statusMessage)
+			if err != nil {
+				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+			} else {
+				providers.TapStatus.Pods = statusMessage.TappingStatus.Pods
+				BroadcastToBrowserClients(message)
+			}
+		case shared.WebsocketMessageTypeOutboundLink:
+			var outboundLinkMessage models.WebsocketOutboundLinkMessage
+			err := json.Unmarshal(message, &outboundLinkMessage)
+			if err != nil {
+				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+			} else {
+				handleTLSLink(outboundLinkMessage)
+			}
+		default:
+			rlog.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
+		}
+	}
+}
+
+func handleTLSLink(outboundLinkMessage models.WebsocketOutboundLinkMessage) {
+	resolvedName := k8sResolver.Resolve(outboundLinkMessage.Data.DstIP)
+	if resolvedName != "" {
+		outboundLinkMessage.Data.DstIP = resolvedName
+	} else if outboundLinkMessage.Data.SuggestedResolvedName != "" {
+		outboundLinkMessage.Data.DstIP = outboundLinkMessage.Data.SuggestedResolvedName
+	}
+	cacheKey := fmt.Sprintf("%s -> %s:%d", outboundLinkMessage.Data.Src, outboundLinkMessage.Data.DstIP, outboundLinkMessage.Data.DstPort)
+	_, isInCache := providers.RecentTLSLinks.Get(cacheKey)
+	if isInCache {
+		return
+	} else {
+		providers.RecentTLSLinks.SetDefault(cacheKey, outboundLinkMessage.Data)
+	}
+	marshaledMessage, err := json.Marshal(outboundLinkMessage)
+	if err != nil {
+		rlog.Errorf("Error marshaling outbound link message for broadcasting: %v", err)
+	} else {
+		rlog.Errorf("Broadcasting outboundlink message %s", string(marshaledMessage))
+		BroadcastToBrowserClients(marshaledMessage)
+	}
+}
+
+func removeSocketUUIDFromBrowserSlice(uuidToRemove int) {
+	newUUIDSlice := make([]int, 0, len(browserClientSocketUUIDs))
+	for _, uuid := range browserClientSocketUUIDs {
+		if uuid != uuidToRemove {
+			newUUIDSlice = append(newUUIDSlice, uuid)
+		}
+	}
+	browserClientSocketUUIDs = newUUIDSlice
+}

agent/pkg/controllers/entries_controller.go

@@ -0,0 +1,257 @@
+package controllers
+
+import (
+	"encoding/json"
+	"fmt"
+	"mizuserver/pkg/database"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/up9"
+	"mizuserver/pkg/utils"
+	"mizuserver/pkg/validation"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/google/martian/har"
+	"github.com/romana/rlog"
+)
+
+func GetEntries(c *gin.Context) {
+	entriesFilter := &models.EntriesFilter{}
+
+	if err := c.BindQuery(entriesFilter); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+	err := validation.Validate(entriesFilter)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+
+	order := database.OperatorToOrderMapping[entriesFilter.Operator]
+	operatorSymbol := database.OperatorToSymbolMapping[entriesFilter.Operator]
+	var entries []models.MizuEntry
+	database.GetEntriesTable().
+		Order(fmt.Sprintf("timestamp %s", order)).
+		Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)).
+		Omit("entry"). // remove the "big" entry field
+		Limit(entriesFilter.Limit).
+		Find(&entries)
+
+	if len(entries) > 0 && order == database.OrderDesc {
+		// the entries always order from oldest to newest so we should revers
+		utils.ReverseSlice(entries)
+	}
+
+	baseEntries := make([]models.BaseEntryDetails, 0)
+	for _, data := range entries {
+		harEntry := models.BaseEntryDetails{}
+		if err := models.GetEntry(&data, &harEntry); err != nil {
+			continue
+		}
+		baseEntries = append(baseEntries, harEntry)
+	}
+
+	c.JSON(http.StatusOK, baseEntries)
+}
+
+func GetHARs(c *gin.Context) {
+	entriesFilter := &models.HarFetchRequestQuery{}
+	order := database.OrderDesc
+	if err := c.BindQuery(entriesFilter); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+	err := validation.Validate(entriesFilter)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+
+	var timestampFrom, timestampTo int64
+
+	if entriesFilter.From < 0 {
+		timestampFrom = 0
+	} else {
+		timestampFrom = entriesFilter.From
+	}
+	if entriesFilter.To <= 0 {
+		timestampTo = time.Now().UnixNano() / int64(time.Millisecond)
+	} else {
+		timestampTo = entriesFilter.To
+	}
+
+	var entries []models.MizuEntry
+	database.GetEntriesTable().
+		Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)).
+		Order(fmt.Sprintf("timestamp %s", order)).
+		Find(&entries)
+
+	if len(entries) > 0 {
+		// the entries always order from oldest to newest so we should revers
+		utils.ReverseSlice(entries)
+	}
+
+	harsObject := map[string]*models.ExtendedHAR{}
+
+	for _, entryData := range entries {
+		var harEntry har.Entry
+		_ = json.Unmarshal([]byte(entryData.Entry), &harEntry)
+		if entryData.ResolvedDestination != "" {
+			harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entryData.ResolvedDestination)
+		}
+
+		var fileName string
+		sourceOfEntry := entryData.ResolvedSource
+		if sourceOfEntry != "" {
+			// naively assumes the proper service source is http
+			sourceOfEntry = fmt.Sprintf("http://%s", sourceOfEntry)
+			//replace / from the file name cause they end up creating a corrupted folder
+			fileName = fmt.Sprintf("%s.har", strings.ReplaceAll(sourceOfEntry, "/", "_"))
+		} else {
+			fileName = "unknown_source.har"
+		}
+		if harOfSource, ok := harsObject[fileName]; ok {
+			harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry)
+		} else {
+			var entriesHar []*har.Entry
+			entriesHar = append(entriesHar, &harEntry)
+			harsObject[fileName] = &models.ExtendedHAR{
+				Log: &models.ExtendedLog{
+					Version: "1.2",
+					Creator: &models.ExtendedCreator{
+						Creator: &har.Creator{
+							Name:    "mizu",
+							Version: "0.0.2",
+						},
+					},
+					Entries: entriesHar,
+				},
+			}
+			// leave undefined when no source is present, otherwise modeler assumes source is empty string ""
+			if sourceOfEntry != "" {
+				harsObject[fileName].Log.Creator.Source = &sourceOfEntry
+			}
+		}
+	}
+
+	retObj := map[string][]byte{}
+	for k, v := range harsObject {
+		bytesData, _ := json.Marshal(v)
+		retObj[k] = bytesData
+	}
+	buffer := utils.ZipData(retObj)
+	c.Data(http.StatusOK, "application/octet-stream", buffer.Bytes())
+}
+
+func UploadEntries(c *gin.Context) {
+	rlog.Infof("Upload entries - started\n")
+
+	uploadParams := &models.UploadEntriesRequestQuery{}
+	if err := c.BindQuery(uploadParams); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+	if err := validation.Validate(uploadParams); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+	if up9.GetAnalyzeInfo().IsAnalyzing {
+		c.String(http.StatusBadRequest, "Cannot analyze, mizu is already analyzing")
+		return
+	}
+
+	rlog.Infof("Upload entries - creating token. dest %s\n", uploadParams.Dest)
+	token, err := up9.CreateAnonymousToken(uploadParams.Dest)
+	if err != nil {
+		c.String(http.StatusServiceUnavailable, "Cannot analyze, mizu is already analyzing")
+		return
+	}
+	rlog.Infof("Upload entries - uploading. token: %s model: %s\n", token.Token, token.Model)
+	go up9.UploadEntriesImpl(token.Token, token.Model, uploadParams.Dest, uploadParams.SleepIntervalSec)
+	c.String(http.StatusOK, "OK")
+}
+
+func GetFullEntries(c *gin.Context) {
+	entriesFilter := &models.HarFetchRequestQuery{}
+	if err := c.BindQuery(entriesFilter); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+	err := validation.Validate(entriesFilter)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+
+	var timestampFrom, timestampTo int64
+
+	if entriesFilter.From < 0 {
+		timestampFrom = 0
+	} else {
+		timestampFrom = entriesFilter.From
+	}
+	if entriesFilter.To <= 0 {
+		timestampTo = time.Now().UnixNano() / int64(time.Millisecond)
+	} else {
+		timestampTo = entriesFilter.To
+	}
+
+	entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
+	result := make([]models.FullEntryDetails, 0)
+	for _, data := range entriesArray {
+		harEntry := models.FullEntryDetails{}
+		if err := models.GetEntry(&data, &harEntry); err != nil {
+			continue
+		}
+		result = append(result, harEntry)
+	}
+	c.JSON(http.StatusOK, result)
+}
+
+func GetEntry(c *gin.Context) {
+	var entryData models.MizuEntry
+	database.GetEntriesTable().
+		Where(map[string]string{"entryId": c.Param("entryId")}).
+		First(&entryData)
+
+	fullEntry := models.FullEntryDetails{}
+	if err := models.GetEntry(&entryData, &fullEntry); err != nil {
+		c.JSON(http.StatusInternalServerError, map[string]interface{}{
+			"error": true,
+			"msg":   "Can't get entry details",
+		})
+	}
+	fullEntryWithPolicy := models.FullEntryWithPolicy{}
+	if err := models.GetEntry(&entryData, &fullEntryWithPolicy); err != nil {
+		c.JSON(http.StatusInternalServerError, map[string]interface{}{
+			"error": true,
+			"msg":   "Can't get entry details",
+		})
+	}
+	c.JSON(http.StatusOK, fullEntryWithPolicy)
+}
+
+func DeleteAllEntries(c *gin.Context) {
+	database.GetEntriesTable().
+		Where("1 = 1").
+		Delete(&models.MizuEntry{})
+
+	c.JSON(http.StatusOK, map[string]string{
+		"msg": "Success",
+	})
+
+}
+
+func GetGeneralStats(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.GetGeneralStats())
+}
+
+func GetTappingStatus(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.TapStatus)
+}
+
+func AnalyzeInformation(c *gin.Context) {
+	c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
+}
+
+func GetRecentTLSLinks(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
+}

agent/pkg/controllers/metadata_controller.go

@@ -0,0 +1,13 @@
+package controllers
+
+import (
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/shared"
+	"mizuserver/pkg/version"
+	"net/http"
+)
+
+func GetVersion(c *gin.Context) {
+	resp := shared.VersionResponse{SemVer: version.SemVer}
+	c.JSON(http.StatusOK, resp)
+}

agent/pkg/controllers/resolving_controller.go

@@ -0,0 +1,12 @@
+package controllers
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/holder"
+	"net/http"
+)
+
+func GetCurrentResolvingInformation(c *gin.Context) {
+	c.JSON(http.StatusOK, holder.GetResolver().GetMap())
+}
+

agent/pkg/controllers/status_controller.go

@@ -0,0 +1,32 @@
+package controllers
+
+import (
+	"encoding/json"
+	"github.com/gin-gonic/gin"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared"
+	"mizuserver/pkg/api"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/validation"
+	"net/http"
+)
+
+func PostTappedPods(c *gin.Context) {
+	tapStatus := &shared.TapStatus{}
+	if err := c.Bind(tapStatus); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+	if err := validation.Validate(tapStatus); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+	rlog.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods))
+	providers.TapStatus.Pods = tapStatus.Pods
+	message := shared.CreateWebSocketStatusMessage(*tapStatus)
+	if jsonBytes, err := json.Marshal(message); err != nil {
+		rlog.Errorf("Could not Marshal message %v\n", err)
+	} else {
+		api.BroadcastToBrowserClients(jsonBytes)
+	}
+}

agent/pkg/database/main.go

@@ -0,0 +1,73 @@
+package database
+
+import (
+	"fmt"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/utils"
+	"time"
+)
+
+const (
+	DBPath = "./entries.db"
+	OrderDesc = "desc"
+	OrderAsc  = "asc"
+	LT        = "lt"
+	GT        = "gt"
+)
+
+var (
+	DB *gorm.DB
+	IsDBLocked = false
+	OperatorToSymbolMapping = map[string]string{
+		LT: "<",
+		GT: ">",
+	}
+	OperatorToOrderMapping = map[string]string{
+		LT: OrderDesc,
+		GT: OrderAsc,
+	}
+)
+
+func init() {
+	DB = initDataBase(DBPath)
+	go StartEnforcingDatabaseSize()
+}
+
+func GetEntriesTable() *gorm.DB {
+	return DB.Table("mizu_entries")
+}
+
+func CreateEntry(entry *models.MizuEntry) {
+	if IsDBLocked {
+		return
+	}
+	GetEntriesTable().Create(entry)
+}
+
+func initDataBase(databasePath string) *gorm.DB {
+	temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{
+		Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond},
+	})
+	_ = temp.AutoMigrate(&models.MizuEntry{}) // this will ensure table is created
+	return temp
+}
+
+
+func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []models.MizuEntry {
+	order := OrderDesc
+	var entries []models.MizuEntry
+	GetEntriesTable().
+		Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)).
+		Order(fmt.Sprintf("timestamp %s", order)).
+		Find(&entries)
+
+	if len(entries) > 0 {
+		// the entries always order from oldest to newest so we should revers
+		utils.ReverseSlice(entries)
+	}
+	return entries
+}
+

agent/pkg/database/size_enforcer.go

@@ -0,0 +1,121 @@
+package database
+
+import (
+	"github.com/fsnotify/fsnotify"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/debounce"
+	"github.com/up9inc/mizu/shared/units"
+	"log"
+	"mizuserver/pkg/models"
+	"os"
+	"strconv"
+	"time"
+)
+
+const percentageOfMaxSizeBytesToPrune = 15
+const defaultMaxDatabaseSizeBytes int64 = 200 * 1000 * 1000
+
+func StartEnforcingDatabaseSize() {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Fatalf("Error creating filesystem watcher for db size enforcement: %v\n", err)
+		return
+	}
+
+	maxEntriesDBByteSize, err := getMaxEntriesDBByteSize()
+	if err != nil {
+		log.Fatalf("Error parsing max db size: %v\n", err)
+		return
+	}
+
+	checkFileSizeDebouncer := debounce.NewDebouncer(5*time.Second, func() {
+		checkFileSize(maxEntriesDBByteSize)
+	})
+
+	go func() {
+		for {
+			select {
+			case event, ok := <-watcher.Events:
+				if !ok {
+					return // closed channel
+				}
+				if event.Op == fsnotify.Write {
+					checkFileSizeDebouncer.SetOn()
+				}
+			case err, ok := <-watcher.Errors:
+				if !ok {
+					return // closed channel
+				}
+				rlog.Errorf("filesystem watcher encountered error:%v", err)
+			}
+		}
+	}()
+
+	err = watcher.Add(DBPath)
+	if err != nil {
+		log.Fatalf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err)
+	}
+}
+
+func getMaxEntriesDBByteSize() (int64, error) {
+	maxEntriesDBByteSize := defaultMaxDatabaseSizeBytes
+	var err error
+
+	maxEntriesDBSizeByteSEnvVarValue := os.Getenv(shared.MaxEntriesDBSizeBytesEnvVar)
+	if maxEntriesDBSizeByteSEnvVarValue != "" {
+		maxEntriesDBByteSize, err = strconv.ParseInt(maxEntriesDBSizeByteSEnvVarValue, 10, 64)
+	}
+	return maxEntriesDBByteSize, err
+}
+
+func checkFileSize(maxSizeBytes int64) {
+	fileStat, err := os.Stat(DBPath)
+	if err != nil {
+		rlog.Errorf("Error checking %s file size: %v", DBPath, err)
+	} else {
+		if fileStat.Size() > maxSizeBytes {
+			pruneOldEntries(fileStat.Size())
+		}
+	}
+}
+
+func pruneOldEntries(currentFileSize int64) {
+	// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
+	IsDBLocked = true
+	defer func() { IsDBLocked = false }()
+
+	amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
+
+	rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()
+	if err != nil {
+		rlog.Errorf("Error getting 10000 first db rows: %v", err)
+		return
+	}
+
+	entryIdsToRemove := make([]uint, 0)
+	bytesToBeRemoved := int64(0)
+	for rows.Next() {
+		if bytesToBeRemoved >= amountOfBytesToTrim {
+			break
+		}
+		var entry models.MizuEntry
+		err = DB.ScanRows(rows, &entry)
+		if err != nil {
+			rlog.Errorf("Error scanning db row: %v", err)
+			continue
+		}
+
+		entryIdsToRemove = append(entryIdsToRemove, entry.ID)
+		bytesToBeRemoved += int64(entry.EstimatedSizeBytes)
+	}
+
+	if len(entryIdsToRemove) > 0 {
+		GetEntriesTable().Where(entryIdsToRemove).Delete(models.MizuEntry{})
+		// VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this
+		DB.Exec("VACUUM")
+		rlog.Errorf("Removed %d rows and cleared %s", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
+	} else {
+		rlog.Error("Found no rows to remove when pruning")
+	}
+}

agent/pkg/holder/main.go

@@ -0,0 +1,14 @@
+package holder
+
+import "mizuserver/pkg/resolver"
+
+var k8sResolver *resolver.Resolver
+
+func SetResolver(param *resolver.Resolver) {
+	k8sResolver = param
+}
+
+func GetResolver() *resolver.Resolver {
+	return k8sResolver
+}
+

agent/pkg/models/models.go

@@ -0,0 +1,226 @@
+package models
+
+import (
+	"encoding/json"
+
+	"mizuserver/pkg/rules"
+	"mizuserver/pkg/utils"
+	"time"
+
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap"
+)
+
+type DataUnmarshaler interface {
+	UnmarshalData(*MizuEntry) error
+}
+
+func GetEntry(r *MizuEntry, v DataUnmarshaler) error {
+	return v.UnmarshalData(r)
+}
+
+type MizuEntry struct {
+	ID                  uint `gorm:"primarykey"`
+	CreatedAt           time.Time
+	UpdatedAt           time.Time
+	Entry               string `json:"entry,omitempty" gorm:"column:entry"`
+	EntryId             string `json:"entryId" gorm:"column:entryId"`
+	Url                 string `json:"url" gorm:"column:url"`
+	Method              string `json:"method" gorm:"column:method"`
+	Status              int    `json:"status" gorm:"column:status"`
+	RequestSenderIp     string `json:"requestSenderIp" gorm:"column:requestSenderIp"`
+	Service             string `json:"service" gorm:"column:service"`
+	Timestamp           int64  `json:"timestamp" gorm:"column:timestamp"`
+	Path                string `json:"path" gorm:"column:path"`
+	ResolvedSource      string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
+	ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
+	IsOutgoing          bool   `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"`
+	EstimatedSizeBytes  int    `json:"-" gorm:"column:estimatedSizeBytes"`
+}
+
+type BaseEntryDetails struct {
+	Id              string          `json:"id,omitempty"`
+	Url             string          `json:"url,omitempty"`
+	RequestSenderIp string          `json:"requestSenderIp,omitempty"`
+	Service         string          `json:"service,omitempty"`
+	Path            string          `json:"path,omitempty"`
+	StatusCode      int             `json:"statusCode,omitempty"`
+	Method          string          `json:"method,omitempty"`
+	Timestamp       int64           `json:"timestamp,omitempty"`
+	IsOutgoing      bool            `json:"isOutgoing,omitempty"`
+	Latency         int64           `json:"latency,omitempty"`
+	Rules           ApplicableRules `json:"rules,omitempty"`
+}
+
+type ApplicableRules struct {
+	Latency int64 `json:"latency,omitempty"`
+	Status  bool  `json:"status,omitempty"`
+	NumberOfRules int `json:"numberOfRules,omitempty"`
+}
+
+func NewApplicableRules(status bool, latency int64, number int) ApplicableRules {
+	ar := ApplicableRules{}
+	ar.Status = status
+	ar.Latency = latency
+	ar.NumberOfRules = number
+	return ar
+}
+
+type FullEntryDetails struct {
+	har.Entry
+}
+
+type FullEntryDetailsExtra struct {
+	har.Entry
+}
+
+func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
+	entryUrl := entry.Url
+	service := entry.Service
+	if entry.ResolvedDestination != "" {
+		entryUrl = utils.SetHostname(entryUrl, entry.ResolvedDestination)
+		service = utils.SetHostname(service, entry.ResolvedDestination)
+	}
+	bed.Id = entry.EntryId
+	bed.Url = entryUrl
+	bed.Service = service
+	bed.Path = entry.Path
+	bed.StatusCode = entry.Status
+	bed.Method = entry.Method
+	bed.Timestamp = entry.Timestamp
+	bed.RequestSenderIp = entry.RequestSenderIp
+	bed.IsOutgoing = entry.IsOutgoing
+	return nil
+}
+
+func (fed *FullEntryDetails) UnmarshalData(entry *MizuEntry) error {
+	if err := json.Unmarshal([]byte(entry.Entry), &fed.Entry); err != nil {
+		return err
+	}
+
+	if entry.ResolvedDestination != "" {
+		fed.Entry.Request.URL = utils.SetHostname(fed.Entry.Request.URL, entry.ResolvedDestination)
+	}
+	return nil
+}
+
+func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *MizuEntry) error {
+	if err := json.Unmarshal([]byte(entry.Entry), &fedex.Entry); err != nil {
+		return err
+	}
+
+	if entry.ResolvedSource != "" {
+		fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.ResolvedSource})
+	}
+	if entry.ResolvedDestination != "" {
+		fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.ResolvedDestination})
+		fedex.Entry.Request.URL = utils.SetHostname(fedex.Entry.Request.URL, entry.ResolvedDestination)
+	}
+	return nil
+}
+
+type EntriesFilter struct {
+	Limit     int    `form:"limit" validate:"required,min=1,max=200"`
+	Operator  string `form:"operator" validate:"required,oneof='lt' 'gt'"`
+	Timestamp int64  `form:"timestamp" validate:"required,min=1"`
+}
+
+type UploadEntriesRequestQuery struct {
+	Dest             string `form:"dest"`
+	SleepIntervalSec int    `form:"interval"`
+}
+
+type HarFetchRequestQuery struct {
+	From int64 `form:"from"`
+	To   int64 `form:"to"`
+}
+
+type WebSocketEntryMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *BaseEntryDetails `json:"data,omitempty"`
+}
+
+type WebSocketTappedEntryMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *tap.OutputChannelItem
+}
+
+type WebsocketOutboundLinkMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *tap.OutboundLink
+}
+
+func CreateBaseEntryWebSocketMessage(base *BaseEntryDetails) ([]byte, error) {
+	message := &WebSocketEntryMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeEntry,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+func CreateWebsocketTappedEntryMessage(base *tap.OutputChannelItem) ([]byte, error) {
+	message := &WebSocketTappedEntryMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeTappedEntry,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error) {
+	message := &WebsocketOutboundLinkMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebsocketMessageTypeOutboundLink,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+// ExtendedHAR is the top level object of a HAR log.
+type ExtendedHAR struct {
+	Log *ExtendedLog `json:"log"`
+}
+
+// ExtendedLog is the HAR HTTP request and response log.
+type ExtendedLog struct {
+	// Version number of the HAR format.
+	Version string `json:"version"`
+	// Creator holds information about the log creator application.
+	Creator *ExtendedCreator `json:"creator"`
+	// Entries is a list containing requests and responses.
+	Entries []*har.Entry `json:"entries"`
+}
+
+type ExtendedCreator struct {
+	*har.Creator
+	Source *string `json:"_source"`
+}
+
+type FullEntryWithPolicy struct {
+	RulesMatched []rules.RulesMatched `json:"rulesMatched,omitempty"`
+	Entry        har.Entry            `json:"entry"`
+	Service      string               `json:"service"`
+}
+
+func (fewp *FullEntryWithPolicy) UnmarshalData(entry *MizuEntry) error {
+	if err := json.Unmarshal([]byte(entry.Entry), &fewp.Entry); err != nil {
+		return err
+	}
+
+	_, resultPolicyToSend := rules.MatchRequestPolicy(fewp.Entry, entry.Service)
+	fewp.RulesMatched = resultPolicyToSend
+	fewp.Service = entry.Service
+	return nil
+}
+
+func RunValidationRulesState(harEntry har.Entry, service string) ApplicableRules {
+	numberOfRules, resultPolicyToSend := rules.MatchRequestPolicy(harEntry, service)
+	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend, numberOfRules)
+	ar := NewApplicableRules(statusPolicyToSend, latency, numberOfRules)
+	return ar
+}

agent/pkg/providers/stats_provider.go

@@ -0,0 +1,36 @@
+package providers
+
+import (
+	"reflect"
+	"time"
+)
+
+type GeneralStats struct {
+	EntriesCount        int
+	FirstEntryTimestamp int
+	LastEntryTimestamp  int
+}
+
+var generalStats = GeneralStats{}
+
+func ResetGeneralStats() {
+	generalStats = GeneralStats{}
+}
+
+func GetGeneralStats() GeneralStats {
+	return generalStats
+}
+
+func EntryAdded() {
+	generalStats.EntriesCount++
+
+	currentTimestamp := int(time.Now().Unix())
+
+	if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) {
+		generalStats.FirstEntryTimestamp = currentTimestamp
+	}
+
+	generalStats.LastEntryTimestamp = currentTimestamp
+}
+
+

agent/pkg/providers/stats_provider_test.go

@@ -0,0 +1,35 @@
+package providers_test
+
+import (
+	"fmt"
+	"mizuserver/pkg/providers"
+	"testing"
+)
+
+func TestNoEntryAddedCount(t *testing.T) {
+	entriesStats := providers.GetGeneralStats()
+
+	if entriesStats.EntriesCount != 0 {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesCount)
+	}
+}
+
+func TestEntryAddedCount(t *testing.T) {
+	tests := []int{1, 5, 10, 100, 500, 1000}
+
+	for _, entriesCount := range tests {
+		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
+			for i := 0; i < entriesCount; i++ {
+				providers.EntryAdded()
+			}
+
+			entriesStats := providers.GetGeneralStats()
+
+			if entriesStats.EntriesCount != entriesCount {
+				t.Errorf("unexpected result - expected: %v, actual: %v", entriesCount, entriesStats.EntriesCount)
+			}
+
+			t.Cleanup(providers.ResetGeneralStats)
+		})
+	}
+}

agent/pkg/providers/status_provider.go

@@ -0,0 +1,28 @@
+package providers
+
+import (
+	"github.com/patrickmn/go-cache"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap"
+	"time"
+)
+
+const tlsLinkRetainmentTime = time.Minute * 15
+
+var (
+	TapStatus shared.TapStatus
+	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
+)
+
+func GetAllRecentTLSAddresses() []string {
+	recentTLSLinks := make([]string, 0)
+
+	for _, outboundLinkItem := range RecentTLSLinks.Items() {
+		outboundLink, castOk := outboundLinkItem.Object.(*tap.OutboundLink)
+		if castOk {
+			recentTLSLinks = append(recentTLSLinks, outboundLink.DstIP)
+		}
+	}
+
+	return recentTLSLinks
+}

agent/pkg/resolver/README.md

@@ -32,7 +32,7 @@ Now you will be able to import `github.com/up9inc/mizu/resolver` in any `.go` fi
 errOut := make(chan error, 100)
 k8sResolver, err := resolver.NewFromOutOfCluster("", errOut)
 if err != nil {
-    fmt.Printf("error creating k8s resolver %s", err)
+    rlog.Errorf("error creating k8s resolver %s", err)
 }
 
 ctx, cancel := context.WithCancel(context.Background())
@@ -40,15 +40,15 @@ k8sResolver.Start(ctx)
 
 resolvedName := k8sResolver.Resolve("10.107.251.91") // will always return `nil` in real scenarios as the internal map takes a moment to populate after `Start` is called
 if resolvedName != nil {
-    fmt.Printf("resolved 10.107.251.91=%s", *resolvedName)
+    rlog.Errorf("resolved 10.107.251.91=%s", *resolvedName)
 } else {
-    fmt.Printf("Could not find a resolved name for 10.107.251.91")
+    rlog.Error("Could not find a resolved name for 10.107.251.91")
 }
 
 for {
     select {
         case err := <- errOut:
-            fmt.Printf("name resolving error %s", err)
+            rlog.Errorf("name resolving error %s", err)
     }
 }
 ```

agent/pkg/resolver/loader.go

@@ -0,0 +1,23 @@
+package resolver
+
+import (
+	cmap "github.com/orcaman/concurrent-map"
+	"k8s.io/client-go/kubernetes"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
+	restclient "k8s.io/client-go/rest"
+)
+
+func NewFromInCluster(errOut chan error, namesapce string) (*Resolver, error) {
+	config, err := restclient.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	return &Resolver{clientConfig: config, clientSet: clientset, nameMap: cmap.New(), serviceMap: cmap.New(), errOut: errOut, namespace: namesapce}, nil
+}

agent/pkg/resolver/resolver.go

@@ -0,0 +1,192 @@
+package resolver
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/romana/rlog"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+
+	"github.com/orcaman/concurrent-map"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+)
+
+const (
+	kubClientNullString = "None"
+)
+
+type Resolver struct {
+	clientConfig *restclient.Config
+	clientSet    *kubernetes.Clientset
+	nameMap      cmap.ConcurrentMap
+	serviceMap   cmap.ConcurrentMap
+	isStarted    bool
+	errOut       chan error
+	namespace    string
+}
+
+func (resolver *Resolver) Start(ctx context.Context) {
+	if !resolver.isStarted {
+		resolver.isStarted = true
+
+		go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchServices)
+		go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchEndpoints)
+		go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchPods)
+	}
+}
+
+func (resolver *Resolver) Resolve(name string) string {
+	resolvedName, isFound := resolver.nameMap.Get(name)
+	if !isFound {
+		return ""
+	}
+	return resolvedName.(string)
+}
+
+func (resolver *Resolver) GetMap() cmap.ConcurrentMap {
+	return resolver.nameMap
+}
+
+func (resolver *Resolver) CheckIsServiceIP(address string) bool {
+	_, isFound := resolver.serviceMap.Get(address)
+	return isFound
+}
+
+func (resolver *Resolver) watchPods(ctx context.Context) error {
+	// empty namespace makes the client watch all namespaces
+	watcher, err := resolver.clientSet.CoreV1().Pods(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true})
+	if err != nil {
+		return err
+	}
+	for {
+		select {
+		case event := <-watcher.ResultChan():
+			if event.Object == nil {
+				return errors.New("error in kubectl pod watch")
+			}
+			if event.Type == watch.Deleted {
+				pod := event.Object.(*corev1.Pod)
+				resolver.saveResolvedName(pod.Status.PodIP, "", event.Type)
+			}
+		case <-ctx.Done():
+			watcher.Stop()
+			return nil
+		}
+	}
+}
+
+func (resolver *Resolver) watchEndpoints(ctx context.Context) error {
+	// empty namespace makes the client watch all namespaces
+	watcher, err := resolver.clientSet.CoreV1().Endpoints(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true})
+	if err != nil {
+		return err
+	}
+	for {
+		select {
+		case event := <-watcher.ResultChan():
+			if event.Object == nil {
+				return errors.New("error in kubectl endpoint watch")
+			}
+			endpoint := event.Object.(*corev1.Endpoints)
+			serviceHostname := fmt.Sprintf("%s.%s", endpoint.Name, endpoint.Namespace)
+			if endpoint.Subsets != nil {
+				for _, subset := range endpoint.Subsets {
+					var ports []int32
+					if subset.Ports != nil {
+						for _, portMapping := range subset.Ports {
+							if portMapping.Port > 0 {
+								ports = append(ports, portMapping.Port)
+							}
+						}
+					}
+					if subset.Addresses != nil {
+						for _, address := range subset.Addresses {
+							resolver.saveResolvedName(address.IP, serviceHostname, event.Type)
+							for _, port := range ports {
+								ipWithPort := fmt.Sprintf("%s:%d", address.IP, port)
+								resolver.saveResolvedName(ipWithPort, serviceHostname, event.Type)
+							}
+						}
+					}
+
+				}
+			}
+		case <-ctx.Done():
+			watcher.Stop()
+			return nil
+		}
+	}
+}
+
+func (resolver *Resolver) watchServices(ctx context.Context) error {
+	// empty namespace makes the client watch all namespaces
+	watcher, err := resolver.clientSet.CoreV1().Services(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true})
+	if err != nil {
+		return err
+	}
+	for {
+		select {
+		case event := <-watcher.ResultChan():
+			if event.Object == nil {
+				return errors.New("error in kubectl service watch")
+			}
+
+			service := event.Object.(*corev1.Service)
+			serviceHostname := fmt.Sprintf("%s.%s", service.Name, service.Namespace)
+			if service.Spec.ClusterIP != "" && service.Spec.ClusterIP != kubClientNullString {
+				resolver.saveResolvedName(service.Spec.ClusterIP, serviceHostname, event.Type)
+				resolver.saveServiceIP(service.Spec.ClusterIP, serviceHostname, event.Type)
+			}
+			if service.Status.LoadBalancer.Ingress != nil {
+				for _, ingress := range service.Status.LoadBalancer.Ingress {
+					resolver.saveResolvedName(ingress.IP, serviceHostname, event.Type)
+				}
+			}
+		case <-ctx.Done():
+			watcher.Stop()
+			return nil
+		}
+	}
+}
+
+func (resolver *Resolver) saveResolvedName(key string, resolved string, eventType watch.EventType) {
+	if eventType == watch.Deleted {
+		resolver.nameMap.Remove(key)
+		rlog.Infof("setting %s=nil\n", key)
+	} else {
+		resolver.nameMap.Set(key, resolved)
+		rlog.Infof("setting %s=%s\n", key, resolved)
+	}
+}
+
+func (resolver *Resolver) saveServiceIP(key string, resolved string, eventType watch.EventType) {
+	if eventType == watch.Deleted {
+		resolver.serviceMap.Remove(key)
+	} else {
+		resolver.serviceMap.Set(key, resolved)
+	}
+}
+
+func (resolver *Resolver) infiniteErrorHandleRetryFunc(ctx context.Context, fun func(ctx context.Context) error) {
+	for {
+		err := fun(ctx)
+		if err != nil {
+			resolver.errOut <- err
+
+			var statusError *k8serrors.StatusError
+			if errors.As(err, &statusError) {
+				if statusError.ErrStatus.Reason == metav1.StatusReasonForbidden {
+					rlog.Infof("Resolver loop encountered permission error, aborting event listening - %v\n", err)
+					return
+				}
+			}
+		}
+		if ctx.Err() != nil { // context was cancelled or errored
+			return
+		}
+	}
+}

agent/pkg/routes/entries_routes.go

@@ -0,0 +1,26 @@
+package routes
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/controllers"
+)
+
+// EntriesRoutes defines the group of har entries routes.
+func EntriesRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/api")
+
+	routeGroup.GET("/entries", controllers.GetEntries)        // get entries (base/thin entries)
+	routeGroup.GET("/entries/:entryId", controllers.GetEntry) // get single (full) entry
+	routeGroup.GET("/exportEntries", controllers.GetFullEntries)
+	routeGroup.GET("/uploadEntries", controllers.UploadEntries)
+	routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
+
+	routeGroup.GET("/har", controllers.GetHARs)
+
+	routeGroup.GET("/resetDB", controllers.DeleteAllEntries)     // get single (full) entry
+	routeGroup.GET("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB
+
+	routeGroup.GET("/tapStatus", controllers.GetTappingStatus) // get tapping status
+	routeGroup.GET("/analyzeStatus", controllers.AnalyzeInformation)
+	routeGroup.GET("/recentTLSLinks", controllers.GetRecentTLSLinks)
+}

agent/pkg/routes/metadata_routes.go

@@ -0,0 +1,13 @@
+package routes
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/controllers"
+)
+
+// MetadataRoutes defines the group of metadata routes.
+func MetadataRoutes(app *gin.Engine) {
+	routeGroup := app.Group("/metadata")
+
+	routeGroup.GET("/version", controllers.GetVersion)
+}

agent/pkg/routes/not_found_route.go

@@ -0,0 +1,18 @@
+package routes
+
+import (
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+// NotFoundRoute defines the 404 Error route.
+func NotFoundRoute(app *gin.Engine) {
+	app.Use(
+		func(c *gin.Context) {
+			c.JSON(http.StatusNotFound, map[string]interface{}{
+				"error": true,
+				"msg":   "sorry, endpoint is not found",
+			})
+		},
+	)
+}

agent/pkg/routes/status_routes.go

@@ -0,0 +1,12 @@
+package routes
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/controllers"
+)
+
+func StatusRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/status")
+
+	routeGroup.POST("/tappedPods", controllers.PostTappedPods)
+}

agent/pkg/rules/models.go

@@ -0,0 +1,110 @@
+package rules
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"regexp"
+	"strings"
+
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared"
+	jsonpath "github.com/yalp/jsonpath"
+)
+
+type RulesMatched struct {
+	Matched bool              `json:"matched"`
+	Rule    shared.RulePolicy `json:"rule"`
+}
+
+func appendRulesMatched(rulesMatched []RulesMatched, matched bool, rule shared.RulePolicy) []RulesMatched {
+	return append(rulesMatched, RulesMatched{Matched: matched, Rule: rule})
+}
+
+func ValidatePath(URLFromRule string, URL string) bool {
+	if URLFromRule != "" {
+		matchPath, err := regexp.MatchString(URLFromRule, URL)
+		if err != nil || !matchPath {
+			return false
+		}
+	}
+	return true
+}
+
+func ValidateService(serviceFromRule string, service string) bool {
+	if serviceFromRule != "" {
+		matchService, err := regexp.MatchString(serviceFromRule, service)
+		if err != nil || !matchService {
+			return false
+		}
+	}
+	return true
+}
+
+func MatchRequestPolicy(harEntry har.Entry, service string) (int, []RulesMatched) {
+	enforcePolicy, _ := shared.DecodeEnforcePolicy(fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.RulePolicyFileName))
+	var resultPolicyToSend []RulesMatched
+	for _, rule := range enforcePolicy.Rules {
+		if !ValidatePath(rule.Path, harEntry.Request.URL) || !ValidateService(rule.Service, service) {
+			continue
+		}
+		if rule.Type == "json" {
+			var bodyJsonMap interface{}
+			if err := json.Unmarshal(harEntry.Response.Content.Text, &bodyJsonMap); err != nil {
+				continue
+			}
+			out, err := jsonpath.Read(bodyJsonMap, rule.Key)
+			if err != nil || out == nil {
+				continue
+			}
+			var matchValue bool
+			if reflect.TypeOf(out).Kind() == reflect.String {
+				matchValue, err = regexp.MatchString(rule.Value, out.(string))
+				if err != nil {
+					continue
+				}
+			} else {
+				val := fmt.Sprint(out)
+				matchValue, err = regexp.MatchString(rule.Value, val)
+				if err != nil {
+					continue
+				}
+			}
+			resultPolicyToSend = appendRulesMatched(resultPolicyToSend, matchValue, rule)
+		} else if rule.Type == "header" {
+			for j := range harEntry.Response.Headers {
+				matchKey, err := regexp.MatchString(rule.Key, harEntry.Response.Headers[j].Name)
+				if err != nil {
+					continue
+				}
+				if matchKey {
+					matchValue, err := regexp.MatchString(rule.Value, harEntry.Response.Headers[j].Value)
+					if err != nil {
+						continue
+					}
+					resultPolicyToSend = appendRulesMatched(resultPolicyToSend, matchValue, rule)
+				}
+			}
+		} else {
+			resultPolicyToSend = appendRulesMatched(resultPolicyToSend, true, rule)
+		}
+	}
+	return len(enforcePolicy.Rules), resultPolicyToSend
+}
+
+func PassedValidationRules(rulesMatched []RulesMatched, numberOfRules int) (bool, int64, int) {
+	if len(rulesMatched) == 0 {
+		return false, 0, 0
+	}
+	for _, rule := range rulesMatched {
+		if rule.Matched == false {
+			return false, -1, len(rulesMatched)
+		}
+	}
+	for _, rule := range rulesMatched {
+		if strings.ToLower(rule.Rule.Type) == "latency" {
+			return true, rule.Rule.Latency, len(rulesMatched)
+		}
+	}
+	return true, -1, len(rulesMatched)
+}

agent/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go

@@ -5,7 +5,7 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
-	"mizuserver/pkg/tap"
+	"github.com/up9inc/mizu/tap"
 	"net/url"
 	"strings"
 
@@ -15,8 +15,8 @@ import (
 )
 
 func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, options *shared.TrafficFilteringOptions) {
-	filterHarHeaders(harOutputItem.HarEntry.Request.Headers)
-	filterHarHeaders(harOutputItem.HarEntry.Response.Headers)
+	harOutputItem.HarEntry.Request.Headers = filterHarHeaders(harOutputItem.HarEntry.Request.Headers)
+	harOutputItem.HarEntry.Response.Headers = filterHarHeaders(harOutputItem.HarEntry.Response.Headers)
 
 	harOutputItem.HarEntry.Request.Cookies = make([]har.Cookie, 0, 0)
 	harOutputItem.HarEntry.Response.Cookies = make([]har.Cookie, 0, 0)
@@ -44,12 +44,19 @@ func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, opt
 	}
 }
 
-func filterHarHeaders(headers []har.Header) {
+func filterHarHeaders(headers []har.Header) []har.Header {
+	newHeaders := make([]har.Header, 0)
 	for i, header := range headers {
-		if isFieldNameSensitive(header.Name) {
+		if strings.ToLower(header.Name) == "cookie" {
+			continue
+		} else if isFieldNameSensitive(header.Name) {
+			newHeaders = append(newHeaders, har.Header{Name: header.Name, Value: maskedFieldPlaceholderValue})
 			headers[i].Value = maskedFieldPlaceholderValue
+		} else {
+			newHeaders = append(newHeaders, header)
 		}
 	}
+	return newHeaders
 }
 
 func getContentTypeHeaderValue(headers []har.Header) string {

agent/pkg/up9/main.go

@@ -0,0 +1,201 @@
+package up9
+
+import (
+	"bytes"
+	"compress/zlib"
+	"encoding/json"
+	"fmt"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared"
+	"io/ioutil"
+	"log"
+	"mizuserver/pkg/database"
+	"mizuserver/pkg/models"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+const (
+	AnalyzeCheckSleepTime = 5 * time.Second
+)
+
+type GuestToken struct {
+	Token string `json:"token"`
+	Model string `json:"model"`
+}
+
+type ModelStatus struct {
+	LastMajorGeneration float64 `json:"lastMajorGeneration"`
+}
+
+func getGuestToken(url string, target *GuestToken) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	rlog.Infof("Got token from the server, starting to json decode... status code: %v", resp.StatusCode)
+	return json.NewDecoder(resp.Body).Decode(target)
+}
+
+func CreateAnonymousToken(envPrefix string) (*GuestToken, error) {
+	tokenUrl := fmt.Sprintf("https://trcc.%s/anonymous/token", envPrefix)
+	if strings.HasPrefix(envPrefix, "http") {
+		tokenUrl = fmt.Sprintf("%s/api/token", envPrefix)
+	}
+	token := &GuestToken{}
+	if err := getGuestToken(tokenUrl, token); err != nil {
+		rlog.Infof("Failed to get token, %s", err)
+		return nil, err
+	}
+	return token, nil
+}
+
+func GetRemoteUrl(analyzeDestination string, analyzeToken string) string {
+	return fmt.Sprintf("https://%s/share/%s", analyzeDestination, analyzeToken)
+}
+
+func CheckIfModelReady(analyzeDestination string, analyzeModel string, analyzeToken string) bool {
+	statusUrl, _ := url.Parse(fmt.Sprintf("https://trcc.%s/models/%s/status", analyzeDestination, analyzeModel))
+	req := &http.Request{
+		Method: http.MethodGet,
+		URL:    statusUrl,
+		Header: map[string][]string{
+			"Content-Type": {"application/json"},
+			"Guest-Auth":   {analyzeToken},
+		},
+	}
+	statusResp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return false
+	}
+	defer statusResp.Body.Close()
+
+	target := &ModelStatus{}
+	_ = json.NewDecoder(statusResp.Body).Decode(&target)
+
+	return target.LastMajorGeneration > 0
+}
+
+func GetTrafficDumpUrl(analyzeDestination string, analyzeModel string) *url.URL {
+	strUrl := fmt.Sprintf("https://traffic.%s/dumpTrafficBulk/%s", analyzeDestination, analyzeModel)
+	if strings.HasPrefix(analyzeDestination, "http") {
+		strUrl = fmt.Sprintf("%s/api/workspace/dumpTrafficBulk", analyzeDestination)
+	}
+	postUrl, _ := url.Parse(strUrl)
+	return postUrl
+}
+
+type AnalyzeInformation struct {
+	IsAnalyzing        bool
+	SentCount          int
+	AnalyzedModel      string
+	AnalyzeToken       string
+	AnalyzeDestination string
+}
+
+func (info *AnalyzeInformation) Reset() {
+	info.IsAnalyzing = false
+	info.AnalyzedModel = ""
+	info.AnalyzeToken = ""
+	info.AnalyzeDestination = ""
+	info.SentCount = 0
+}
+
+var analyzeInformation = &AnalyzeInformation{}
+
+func GetAnalyzeInfo() *shared.AnalyzeStatus {
+	return &shared.AnalyzeStatus{
+		IsAnalyzing:   analyzeInformation.IsAnalyzing,
+		RemoteUrl:     GetRemoteUrl(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzeToken),
+		IsRemoteReady: CheckIfModelReady(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzedModel, analyzeInformation.AnalyzeToken),
+		SentCount:     analyzeInformation.SentCount,
+	}
+}
+
+func UploadEntriesImpl(token string, model string, envPrefix string, sleepIntervalSec int) {
+	analyzeInformation.IsAnalyzing = true
+	analyzeInformation.AnalyzedModel = model
+	analyzeInformation.AnalyzeToken = token
+	analyzeInformation.AnalyzeDestination = envPrefix
+	analyzeInformation.SentCount = 0
+
+	sleepTime := time.Second * time.Duration(sleepIntervalSec)
+
+	var timestampFrom int64 = 0
+
+	for {
+		timestampTo := time.Now().UnixNano() / int64(time.Millisecond)
+		rlog.Infof("Getting entries from %v, to %v\n", timestampFrom, timestampTo)
+		entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
+
+		if len(entriesArray) > 0 {
+
+			fullEntriesExtra := make([]models.FullEntryDetailsExtra, 0)
+			for _, data := range entriesArray {
+				harEntry := models.FullEntryDetailsExtra{}
+				if err := models.GetEntry(&data, &harEntry); err != nil {
+					continue
+				}
+				fullEntriesExtra = append(fullEntriesExtra, harEntry)
+			}
+			rlog.Infof("About to upload %v entries\n", len(fullEntriesExtra))
+
+			body, jMarshalErr := json.Marshal(fullEntriesExtra)
+			if jMarshalErr != nil {
+				analyzeInformation.Reset()
+				rlog.Infof("Stopping analyzing")
+				log.Fatal(jMarshalErr)
+			}
+
+			var in bytes.Buffer
+			w := zlib.NewWriter(&in)
+			_, _ = w.Write(body)
+			_ = w.Close()
+			reqBody := ioutil.NopCloser(bytes.NewReader(in.Bytes()))
+
+			req := &http.Request{
+				Method: http.MethodPost,
+				URL:    GetTrafficDumpUrl(envPrefix, model),
+				Header: map[string][]string{
+					"Content-Encoding": {"deflate"},
+					"Content-Type":     {"application/octet-stream"},
+					"Guest-Auth":       {token},
+				},
+				Body: reqBody,
+			}
+
+			if _, postErr := http.DefaultClient.Do(req); postErr != nil {
+				analyzeInformation.Reset()
+				rlog.Info("Stopping analyzing")
+				log.Fatal(postErr)
+			}
+			analyzeInformation.SentCount += len(entriesArray)
+			rlog.Infof("Finish uploading %v entries to %s\n", len(entriesArray), GetTrafficDumpUrl(envPrefix, model))
+
+		} else {
+			rlog.Infof("Nothing to upload")
+		}
+
+		rlog.Infof("Sleeping for %v...\n", sleepTime)
+		time.Sleep(sleepTime)
+		timestampFrom = timestampTo
+	}
+}
+
+func UpdateAnalyzeStatus(callback func(data []byte)) {
+	for {
+		if !analyzeInformation.IsAnalyzing {
+			time.Sleep(AnalyzeCheckSleepTime)
+			continue
+		}
+		analyzeStatus := GetAnalyzeInfo()
+		socketMessage := shared.CreateWebSocketMessageTypeAnalyzeStatus(*analyzeStatus)
+
+		jsonMessage, _ := json.Marshal(socketMessage)
+		callback(jsonMessage)
+		time.Sleep(AnalyzeCheckSleepTime)
+	}
+}

agent/pkg/utils/truncating_logger.go

@@ -0,0 +1,59 @@
+package utils
+
+import (
+	"context"
+	"fmt"
+	"github.com/romana/rlog"
+	"gorm.io/gorm/logger"
+	"gorm.io/gorm/utils"
+	"time"
+)
+
+// TruncatingLogger implements the gorm logger.Interface interface. Its purpose is to act as gorm's logger while truncating logs to a max of 50 characters to minimise the performance impact
+type TruncatingLogger struct {
+	LogLevel      logger.LogLevel
+	SlowThreshold time.Duration
+}
+
+func (truncatingLogger *TruncatingLogger) LogMode(logLevel logger.LogLevel) logger.Interface {
+	truncatingLogger.LogLevel = logLevel
+	return truncatingLogger
+}
+
+func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string, __ ...interface{}) {
+	if truncatingLogger.LogLevel < logger.Info {
+		return
+	}
+	rlog.Errorf("gorm info: %.150s", message)
+}
+
+func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) {
+	if truncatingLogger.LogLevel < logger.Warn {
+		return
+	}
+	rlog.Errorf("gorm warning: %.150s", message)
+}
+
+func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) {
+	if truncatingLogger.LogLevel < logger.Error {
+		return
+	}
+	rlog.Errorf("gorm error: %.150s", message)
+}
+
+func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) {
+	if truncatingLogger.LogLevel == logger.Silent {
+		return
+	}
+	elapsed := time.Since(begin)
+	if err != nil {
+		sql, rows := fc() // copied into every condition as this is a potentially heavy operation best done only when necessary
+		truncatingLogger.Error(ctx, fmt.Sprintf("Error in %s: %v - elapsed: %fs affected rows: %d, sql: %s", utils.FileWithLineNum(), err, elapsed.Seconds(), rows, sql))
+	} else if truncatingLogger.LogLevel >= logger.Warn && elapsed > truncatingLogger.SlowThreshold {
+		sql, rows := fc()
+		truncatingLogger.Warn(ctx, fmt.Sprintf("Slow sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql))
+	} else if truncatingLogger.LogLevel >= logger.Info {
+		sql, rows := fc()
+		truncatingLogger.Info(ctx, fmt.Sprintf("Sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql))
+	}
+}

agent/pkg/utils/utils.go

@@ -1,34 +1,42 @@
 package utils
 
 import (
-	"encoding/json"
-	"fmt"
-	"github.com/gofiber/fiber/v2"
+	"context"
+	"github.com/gin-gonic/gin"
+	"github.com/romana/rlog"
 	"log"
-	"mizuserver/pkg/models"
+	"net/http"
 	"net/url"
 	"os"
 	"os/signal"
 	"reflect"
 	"syscall"
+	"time"
 )
 
 // StartServer starts the server with a graceful shutdown
-func StartServer(app *fiber.App) {
+func StartServer(app *gin.Engine) {
 	signals := make(chan os.Signal, 2)
 	signal.Notify(signals,
 		os.Interrupt,  	  // this catch ctrl + c
 		syscall.SIGTSTP,  // this catch ctrl + z
 	)
 
+	srv := &http.Server{
+		Addr:    ":8080",
+		Handler: app,
+	}
+
 	go func() {
 		_ = <-signals
-		fmt.Println("Shutting down...")
-		_ = app.Shutdown()
+		rlog.Infof("Shutting down...")
+		ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
+		_ = srv.Shutdown(ctx)
+		os.Exit(0)
 	}()
 
 	// Run server.
-	if err := app.Listen(":8899"); err != nil {
+	if err := app.Run(":8899"); err != nil {
 		log.Printf("Oops... Server is not running! Reason: %v", err)
 	}
 }
@@ -61,27 +69,3 @@ func SetHostname(address, newHostname string) string {
 	return replacedUrl.String()
 
 }
-
-func GetResolvedBaseEntry(entry models.MizuEntry) models.BaseEntryDetails {
-	entryUrl := entry.Url
-	service := entry.Service
-	if entry.ResolvedDestination != "" {
-		entryUrl = SetHostname(entryUrl, entry.ResolvedDestination)
-		service = SetHostname(service, entry.ResolvedDestination)
-	}
-	return models.BaseEntryDetails{
-		Id:         entry.EntryId,
-		Url:        entryUrl,
-		Service:    service,
-		Path:       entry.Path,
-		StatusCode: entry.Status,
-		Method:     entry.Method,
-		Timestamp:  entry.Timestamp,
-		RequestSenderIp: entry.RequestSenderIp,
-	}
-}
-
-func GetBytesFromStruct(v interface{}) []byte{
-	a, _ := json.Marshal(v)
-	return a
-}

agent/pkg/version/consts.go

@@ -0,0 +1,8 @@
+package version
+
+var (
+	SemVer         = "0.0.1"
+	Branch         = "develop"
+	GitCommitHash  = "" // this var is overridden using ldflags in makefile when building
+	BuildTimestamp = "" // this var is overridden using ldflags in makefile when building
+)

api/main.go

@@ -1,147 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"flag"
-	"fmt"
-	"github.com/gofiber/fiber/v2"
-	"github.com/gorilla/websocket"
-	"github.com/up9inc/mizu/shared"
-	"mizuserver/pkg/api"
-	"mizuserver/pkg/middleware"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/routes"
-	"mizuserver/pkg/sensitiveDataFiltering"
-	"mizuserver/pkg/tap"
-	"mizuserver/pkg/utils"
-	"os"
-	"os/signal"
-)
-
-var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API")
-var aggregator = flag.Bool("aggregator", false, "Run in aggregator mode with API")
-var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode")
-var aggregatorAddress = flag.String("aggregator-address", "", "Address of mizu collector for tapping")
-
-
-func main() {
-	flag.Parse()
-
-	if !*shouldTap && !*aggregator && !*standalone{
-		panic("One of the flags --tap, --api or --standalone must be provided")
-	}
-
-	if *standalone {
-		harOutputChannel := tap.StartPassiveTapper()
-		filteredHarChannel := make(chan *tap.OutputChannelItem)
-		go filterHarHeaders(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions())
-		go api.StartReadingEntries(filteredHarChannel, nil)
-		hostApi(nil)
-	} else if *shouldTap {
-		if *aggregatorAddress == "" {
-			panic("Aggregator address must be provided with --aggregator-address when using --tap")
-		}
-
-		tapTargets := getTapTargets()
-		if tapTargets != nil {
-			tap.HostAppAddresses = tapTargets
-			fmt.Println("Filtering for the following addresses:", tap.HostAppAddresses)
-		}
-
-		harOutputChannel := tap.StartPassiveTapper()
-		socketConnection, err := shared.ConnectToSocketServer(*aggregatorAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false)
-		if err != nil {
-			panic(fmt.Sprintf("Error connecting to socket server at %s %v", *aggregatorAddress, err))
-		}
-		go pipeChannelToSocket(socketConnection, harOutputChannel)
-	} else if *aggregator {
-		socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000)
-		filteredHarChannel := make(chan *tap.OutputChannelItem)
-		go api.StartReadingEntries(filteredHarChannel, nil)
-		go filterHarHeaders(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions())
-		hostApi(socketHarOutChannel)
-	}
-
-	signalChan := make(chan os.Signal, 1)
-	signal.Notify(signalChan, os.Interrupt)
-	<-signalChan
-
-	fmt.Println("Exiting")
-}
-
-func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
-	app := fiber.New()
-
-
-	middleware.FiberMiddleware(app) // Register Fiber's middleware for app.
-	app.Static("/", "./site")
-
-	//Simple route to know server is running
-	app.Get("/echo", func(c *fiber.Ctx) error {
-		return c.SendString("Hello, World 👋!")
-	})
-	eventHandlers := api.RoutesEventHandlers{
-		SocketHarOutChannel: socketHarOutputChannel,
-	}
-	routes.WebSocketRoutes(app, &eventHandlers)
-	routes.EntriesRoutes(app)
-	routes.NotFoundRoute(app)
-
-	utils.StartServer(app)
-}
-
-
-func getTapTargets() []string {
-	nodeName := os.Getenv(shared.NodeNameEnvVar)
-	var tappedAddressesPerNodeDict map[string][]string
-	err := json.Unmarshal([]byte(os.Getenv(shared.TappedAddressesPerNodeDictEnvVar)), &tappedAddressesPerNodeDict)
-	if err != nil {
-		panic(fmt.Sprintf("env var %s's value of %s is invalid! must be map[string][]string %v", shared.TappedAddressesPerNodeDictEnvVar, tappedAddressesPerNodeDict, err))
-	}
-	return tappedAddressesPerNodeDict[nodeName]
-}
-
-func getTrafficFilteringOptions() *shared.TrafficFilteringOptions {
-	filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar)
-	if filteringOptionsJson == "" {
-		return nil
-	}
-	var filteringOptions shared.TrafficFilteringOptions
-	err := json.Unmarshal([]byte(filteringOptionsJson), &filteringOptions)
-	if err != nil {
-		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.TrafficFilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err))
-	}
-
-	return &filteringOptions
-}
-
-func filterHarHeaders(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) {
-	for message := range inChannel {
-		sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions)
-		outChannel <- message
-	}
-}
-
-func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) {
-	if connection == nil {
-		panic("Websocket connection is nil")
-	}
-
-	if messageDataChannel == nil {
-		panic("Channel of captured messages is nil")
-	}
-
-	for messageData := range messageDataChannel {
-		marshaledData, err := models.CreateWebsocketTappedEntryMessage(messageData)
-		if err != nil {
-			fmt.Printf("error converting message to json %s, (%v,%+v)\n", err, err, err)
-			continue
-		}
-
-		err = connection.WriteMessage(websocket.TextMessage, marshaledData)
-		if err != nil {
-			fmt.Printf("error sending message through socket server %s, (%v,%+v)\n", err, err, err)
-			continue
-		}
-	}
-}

api/pkg/api/main.go

@@ -1,131 +0,0 @@
-package api
-
-import (
-	"bufio"
-	"context"
-	"encoding/json"
-	"fmt"
-	"github.com/google/martian/har"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"mizuserver/pkg/database"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/resolver"
-	"mizuserver/pkg/tap"
-	"mizuserver/pkg/utils"
-	"net/url"
-	"os"
-	"path"
-	"sort"
-	"time"
-)
-
-var k8sResolver *resolver.Resolver
-
-func init() {
-	errOut := make(chan error, 100)
-	res, err := resolver.NewFromInCluster(errOut)
-	if err != nil {
-		fmt.Printf("error creating k8s resolver %s", err)
-		return
-	}
-	ctx := context.Background()
-	res.Start(ctx)
-	go func() {
-		for {
-			select {
-			case err := <-errOut:
-				fmt.Printf("name resolving error %s", err)
-			}
-		}
-	}()
-
-	k8sResolver = res
-}
-
-func StartReadingEntries(harChannel <-chan *tap.OutputChannelItem, workingDir *string) {
-	if workingDir != nil && *workingDir != "" {
-		startReadingFiles(*workingDir)
-	} else {
-		startReadingChannel(harChannel)
-	}
-}
-
-func startReadingFiles(workingDir string) {
-	err := os.MkdirAll(workingDir, os.ModePerm)
-	utils.CheckErr(err)
-
-	for true {
-		dir, _ := os.Open(workingDir)
-		dirFiles, _ := dir.Readdir(-1)
-		sort.Sort(utils.ByModTime(dirFiles))
-
-		if len(dirFiles) == 0 {
-			fmt.Printf("Waiting for new files\n")
-			time.Sleep(3 * time.Second)
-			continue
-		}
-		fileInfo := dirFiles[0]
-		inputFilePath := path.Join(workingDir, fileInfo.Name())
-		file, err := os.Open(inputFilePath)
-		utils.CheckErr(err)
-
-		var inputHar har.HAR
-		decErr := json.NewDecoder(bufio.NewReader(file)).Decode(&inputHar)
-		utils.CheckErr(decErr)
-
-		for _, entry := range inputHar.Log.Entries {
-			time.Sleep(time.Millisecond * 250)
-			saveHarToDb(entry, fileInfo.Name())
-		}
-		rmErr := os.Remove(inputFilePath)
-		utils.CheckErr(rmErr)
-	}
-}
-
-func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) {
-	if outputItems == nil {
-		panic("Channel of captured messages is nil")
-	}
-
-	for item := range outputItems {
-		saveHarToDb(item.HarEntry, item.RequestSenderIp)
-	}
-}
-
-func saveHarToDb(entry *har.Entry, sender string) {
-	entryBytes, _ := json.Marshal(entry)
-	serviceName, urlPath, serviceHostName := getServiceNameFromUrl(entry.Request.URL)
-	entryId := primitive.NewObjectID().Hex()
-	var (
-		resolvedSource      string
-		resolvedDestination string
-	)
-	if k8sResolver != nil {
-		resolvedSource = k8sResolver.Resolve(sender)
-		resolvedDestination = k8sResolver.Resolve(serviceHostName)
-	}
-	mizuEntry := models.MizuEntry{
-		EntryId:             entryId,
-		Entry:               string(entryBytes), // simple way to store it and not convert to bytes
-		Service:             serviceName,
-		Url:                 entry.Request.URL,
-		Path:                urlPath,
-		Method:              entry.Request.Method,
-		Status:              entry.Response.Status,
-		RequestSenderIp:     sender,
-		Timestamp:           entry.StartedDateTime.UnixNano() / int64(time.Millisecond),
-		ResolvedSource:      resolvedSource,
-		ResolvedDestination: resolvedDestination,
-	}
-	database.GetEntriesTable().Create(&mizuEntry)
-
-	baseEntry := utils.GetResolvedBaseEntry(mizuEntry)
-	baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry)
-	broadcastToBrowserClients(baseEntryBytes)
-}
-
-func getServiceNameFromUrl(inputUrl string) (string, string, string) {
-	parsed, err := url.Parse(inputUrl)
-	utils.CheckErr(err)
-	return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path, parsed.Host
-}

api/pkg/api/socket_server_handlers.go

@@ -1,96 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-	"fmt"
-	"github.com/antoniodipinto/ikisocket"
-	"github.com/up9inc/mizu/shared"
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/routes"
-	"mizuserver/pkg/tap"
-)
-
-var browserClientSocketUUIDs = make([]string, 0)
-
-type RoutesEventHandlers struct {
-	routes.EventHandlers
-	SocketHarOutChannel chan<- *tap.OutputChannelItem
-}
-
-
-func (h *RoutesEventHandlers) WebSocketConnect(ep *ikisocket.EventPayload) {
-	if ep.Kws.GetAttribute("is_tapper") == true {
-		fmt.Println(fmt.Sprintf("Websocket Connection event - Tapper connected: %s", ep.SocketUUID))
-	} else {
-		fmt.Println(fmt.Sprintf("Websocket Connection event - Browser socket connected: %s", ep.SocketUUID))
-		browserClientSocketUUIDs = append(browserClientSocketUUIDs, ep.SocketUUID)
-	}
-}
-
-func (h *RoutesEventHandlers) WebSocketDisconnect(ep *ikisocket.EventPayload) {
-	if ep.Kws.GetAttribute("is_tapper") == true {
-		fmt.Println(fmt.Sprintf("Disconnection event - Tapper connected: %s", ep.SocketUUID))
-	} else {
-		fmt.Println(fmt.Sprintf("Disconnection event - Browser socket connected: %s", ep.SocketUUID))
-		removeSocketUUIDFromBrowserSlice(ep.SocketUUID)
-	}
-}
-
-func broadcastToBrowserClients(message []byte) {
-	ikisocket.EmitToList(browserClientSocketUUIDs, message)
-}
-
-func (h *RoutesEventHandlers) WebSocketClose(ep *ikisocket.EventPayload) {
-	if ep.Kws.GetAttribute("is_tapper") == true {
-		fmt.Println(fmt.Sprintf("Websocket Close event - Tapper connected: %s", ep.SocketUUID))
-	} else {
-		fmt.Println(fmt.Sprintf("Websocket  Close event - Browser socket connected: %s", ep.SocketUUID))
-		removeSocketUUIDFromBrowserSlice(ep.SocketUUID)
-	}
-}
-
-func (h *RoutesEventHandlers) WebSocketError(ep *ikisocket.EventPayload) {
-	fmt.Println(fmt.Sprintf("Socket error - Socket uuid : %s %v", ep.SocketUUID, ep.Error))
-}
-
-func (h *RoutesEventHandlers) WebSocketMessage(ep *ikisocket.EventPayload) {
-	var socketMessageBase shared.WebSocketMessageMetadata
-	err := json.Unmarshal(ep.Data, &socketMessageBase)
-	if err != nil {
-		fmt.Printf("Could not unmarshal websocket message %v\n", err)
-	} else {
-		switch socketMessageBase.MessageType {
-		case shared.WebSocketMessageTypeTappedEntry:
-			var tappedEntryMessage models.WebSocketTappedEntryMessage
-			err := json.Unmarshal(ep.Data, &tappedEntryMessage)
-			if err != nil {
-				fmt.Printf("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
-			} else {
-				h.SocketHarOutChannel <- tappedEntryMessage.Data
-			}
-		case shared.WebSocketMessageTypeUpdateStatus:
-			var statusMessage shared.WebSocketStatusMessage
-			err := json.Unmarshal(ep.Data, &statusMessage)
-			if err != nil {
-				fmt.Printf("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
-			} else {
-				controllers.TapStatus = statusMessage.TappingStatus
-				broadcastToBrowserClients(ep.Data)
-			}
-		default:
-			fmt.Printf("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
-		}
-	}
-}
-
-
-func removeSocketUUIDFromBrowserSlice(uuidToRemove string) {
-	newUUIDSlice := make([]string, 0, len(browserClientSocketUUIDs))
-	for _, uuid := range browserClientSocketUUIDs {
-		if uuid != uuidToRemove {
-			newUUIDSlice = append(newUUIDSlice, uuid)
-		}
-	}
-	browserClientSocketUUIDs = newUUIDSlice
-}

api/pkg/controllers/entries_controller.go

@@ -1,166 +0,0 @@
-package controllers
-
-import (
-	"encoding/json"
-	"fmt"
-	"github.com/gofiber/fiber/v2"
-	"github.com/google/martian/har"
-	"mizuserver/pkg/database"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/utils"
-	"mizuserver/pkg/validation"
-)
-
-const (
-	OrderDesc = "desc"
-	OrderAsc  = "asc"
-	LT        = "lt"
-	GT        = "gt"
-)
-
-var (
-	operatorToSymbolMapping = map[string]string{
-		LT: "<",
-		GT: ">",
-	}
-	operatorToOrderMapping = map[string]string{
-		LT: OrderDesc,
-		GT: OrderAsc,
-	}
-)
-
-func GetEntries(c *fiber.Ctx) error {
-	entriesFilter := &models.EntriesFilter{}
-
-	if err := c.QueryParser(entriesFilter); err != nil {
-		return c.Status(fiber.StatusBadRequest).JSON(err)
-	}
-	err := validation.Validate(entriesFilter)
-	if err != nil {
-		return c.Status(fiber.StatusBadRequest).JSON(err)
-	}
-
-	order := operatorToOrderMapping[entriesFilter.Operator]
-	operatorSymbol := operatorToSymbolMapping[entriesFilter.Operator]
-	var entries []models.MizuEntry
-	database.GetEntriesTable().
-		Order(fmt.Sprintf("timestamp %s", order)).
-		Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)).
-		Omit("entry"). // remove the "big" entry field
-		Limit(entriesFilter.Limit).
-		Find(&entries)
-
-	if len(entries) > 0 && order == OrderDesc {
-		// the entries always order from oldest to newest so we should revers
-		utils.ReverseSlice(entries)
-	}
-
-	// Convert to base entries
-	baseEntries := make([]models.BaseEntryDetails, 0, entriesFilter.Limit)
-	for _, entry := range entries {
-		baseEntries = append(baseEntries, utils.GetResolvedBaseEntry(entry))
-	}
-
-	return c.Status(fiber.StatusOK).JSON(baseEntries)
-}
-
-func GetHARs(c *fiber.Ctx) error {
-	entriesFilter := &models.HarFetchRequestBody{}
-	order := OrderDesc
-	if err := c.QueryParser(entriesFilter); err != nil {
-		return c.Status(fiber.StatusBadRequest).JSON(err)
-	}
-	err := validation.Validate(entriesFilter)
-	if err != nil {
-		return c.Status(fiber.StatusBadRequest).JSON(err)
-	}
-
-	var entries []models.MizuEntry
-	database.GetEntriesTable().
-		Order(fmt.Sprintf("timestamp %s", order)).
-		// Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)).
-		Limit(1000).
-		Find(&entries)
-
-	if len(entries) > 0 {
-		// the entries always order from oldest to newest so we should revers
-		utils.ReverseSlice(entries)
-	}
-
-	harsObject := map[string]*models.ExtendedHAR{}
-
-	for _, entryData := range entries {
-		var harEntry har.Entry
-		_ = json.Unmarshal([]byte(entryData.Entry), &harEntry)
-
-		sourceOfEntry := entryData.ResolvedSource
-		fileName := fmt.Sprintf("%s.har", sourceOfEntry)
-		if harOfSource, ok := harsObject[fileName]; ok {
-			harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry)
-		} else {
-			var entriesHar []*har.Entry
-			entriesHar = append(entriesHar, &harEntry)
-			harsObject[fileName] = &models.ExtendedHAR{
-				Log: &models.ExtendedLog{
-					Version: "1.2",
-					Creator: &models.ExtendedCreator{
-						Creator: &har.Creator{
-							Name:    "mizu",
-							Version: "0.0.2",
-						},
-						Source: sourceOfEntry,
-					},
-					Entries: entriesHar,
-				},
-			}
-		}
-	}
-
-	retObj := map[string][]byte{}
-	for k, v := range harsObject {
-		bytesData, _ := json.Marshal(v)
-		retObj[k] = bytesData
-	}
-	buffer := utils.ZipData(retObj)
-	return c.Status(fiber.StatusOK).SendStream(buffer)
-}
-
-func GetEntry(c *fiber.Ctx) error {
-	var entryData models.EntryData
-	database.GetEntriesTable().
-		Select("entry", "resolvedDestination").
-		Where(map[string]string{"entryId": c.Params("entryId")}).
-		First(&entryData)
-
-	var fullEntry har.Entry
-	unmarshallErr := json.Unmarshal([]byte(entryData.Entry), &fullEntry)
-	utils.CheckErr(unmarshallErr)
-
-	if entryData.ResolvedDestination != "" {
-		fullEntry.Request.URL = utils.SetHostname(fullEntry.Request.URL, entryData.ResolvedDestination)
-	}
-
-	return c.Status(fiber.StatusOK).JSON(fullEntry)
-}
-
-func DeleteAllEntries(c *fiber.Ctx) error {
-	database.GetEntriesTable().
-		Where("1 = 1").
-		Delete(&models.MizuEntry{})
-
-	return c.Status(fiber.StatusOK).JSON(fiber.Map{
-		"msg": "Success",
-	})
-
-}
-
-func GetGeneralStats(c *fiber.Ctx) error {
-	sqlQuery := "SELECT count(*) as count, min(timestamp) as min, max(timestamp) as max from mizu_entries"
-	var result struct {
-		Count int
-		Min   int
-		Max   int
-	}
-	database.GetEntriesTable().Raw(sqlQuery).Scan(&result)
-	return c.Status(fiber.StatusOK).JSON(&result)
-}

api/pkg/controllers/status_controller.go

@@ -1,12 +0,0 @@
-package controllers
-
-import (
-	"github.com/gofiber/fiber/v2"
-	"github.com/up9inc/mizu/shared"
-)
-
-var TapStatus shared.TapStatus
-
-func GetTappingStatus(c *fiber.Ctx) error {
-	return c.Status(fiber.StatusOK).JSON(TapStatus)
-}

api/pkg/database/main.go

@@ -1,25 +0,0 @@
-package database
-
-import (
-	"gorm.io/driver/sqlite"
-	"gorm.io/gorm"
-	"mizuserver/pkg/models"
-)
-
-const (
-	DBPath = "./entries.db"
-)
-
-var (
-	DB = initDataBase(DBPath)
-)
-
-func GetEntriesTable() *gorm.DB {
-	return DB.Table("mizu_entries")
-}
-
-func initDataBase(databasePath string) *gorm.DB {
-	temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{})
-	_ = temp.AutoMigrate(&models.MizuEntry{}) // this will ensure table is created
-	return temp
-}

api/pkg/middleware/fiber_middleware.go

@@ -1,18 +0,0 @@
-package middleware
-
-import (
-	"github.com/gofiber/fiber/v2"
-	"github.com/gofiber/fiber/v2/middleware/cors"
-	"github.com/gofiber/fiber/v2/middleware/logger"
-)
-
-// FiberMiddleware provide Fiber's built-in middlewares.
-// See: https://docs.gofiber.io/api/middleware
-func FiberMiddleware(a *fiber.App) {
-	a.Use(
-		// Add CORS to each route.
-		cors.New(),
-		// Add simple logger.
-		logger.New(),
-	)
-}

api/pkg/models/models.go

@@ -1,104 +0,0 @@
-package models
-
-import (
-	"encoding/json"
-	"github.com/google/martian/har"
-	"github.com/up9inc/mizu/shared"
-	"mizuserver/pkg/tap"
-	"time"
-)
-
-type MizuEntry struct {
-	ID                  uint `gorm:"primarykey"`
-	CreatedAt           time.Time
-	UpdatedAt           time.Time
-	Entry               string  `json:"entry,omitempty" gorm:"column:entry"`
-	EntryId             string  `json:"entryId" gorm:"column:entryId"`
-	Url                 string  `json:"url" gorm:"column:url"`
-	Method              string  `json:"method" gorm:"column:method"`
-	Status              int     `json:"status" gorm:"column:status"`
-	RequestSenderIp     string  `json:"requestSenderIp" gorm:"column:requestSenderIp"`
-	Service             string  `json:"service" gorm:"column:service"`
-	Timestamp           int64   `json:"timestamp" gorm:"column:timestamp"`
-	Path                string  `json:"path" gorm:"column:path"`
-	ResolvedSource      string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
-	ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
-}
-
-type BaseEntryDetails struct {
-	Id              string `json:"id,omitempty"`
-	Url             string `json:"url,omitempty"`
-	RequestSenderIp string `json:"requestSenderIp,omitempty"`
-	Service         string `json:"service,omitempty"`
-	Path            string `json:"path,omitempty"`
-	StatusCode      int    `json:"statusCode,omitempty"`
-	Method          string `json:"method,omitempty"`
-	Timestamp       int64  `json:"timestamp,omitempty"`
-}
-
-type EntryData struct {
-	Entry               string  `json:"entry,omitempty"`
-	ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
-}
-
-type EntriesFilter struct {
-	Limit     int    `query:"limit" validate:"required,min=1,max=200"`
-	Operator  string `query:"operator" validate:"required,oneof='lt' 'gt'"`
-	Timestamp int64  `query:"timestamp" validate:"required,min=1"`
-}
-
-type HarFetchRequestBody struct {
-	Limit     int    `query:"limit"`
-}
-
-type WebSocketEntryMessage struct {
-	*shared.WebSocketMessageMetadata
-	Data *BaseEntryDetails `json:"data,omitempty"`
-}
-
-
-type WebSocketTappedEntryMessage struct {
-	*shared.WebSocketMessageMetadata
-	Data *tap.OutputChannelItem
-}
-
-func CreateBaseEntryWebSocketMessage(base *BaseEntryDetails) ([]byte, error) {
-	message := &WebSocketEntryMessage{
-		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
-			MessageType: shared.WebSocketMessageTypeEntry,
-		},
-		Data: base,
-	}
-	return json.Marshal(message)
-}
-
-func CreateWebsocketTappedEntryMessage(base *tap.OutputChannelItem) ([]byte, error) {
-	message := &WebSocketTappedEntryMessage{
-		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
-			MessageType: shared.WebSocketMessageTypeTappedEntry,
-		},
-		Data: base,
-	}
-	return json.Marshal(message)
-}
-
-
-// ExtendedHAR is the top level object of a HAR log.
-type ExtendedHAR struct {
-	Log *ExtendedLog `json:"log"`
-}
-
-// ExtendedLog is the HAR HTTP request and response log.
-type ExtendedLog struct {
-	// Version number of the HAR format.
-	Version string `json:"version"`
-	// Creator holds information about the log creator application.
-	Creator *ExtendedCreator `json:"creator"`
-	// Entries is a list containing requests and responses.
-	Entries []*har.Entry `json:"entries"`
-}
-
-type ExtendedCreator struct {
-	*har.Creator
-	Source         string `json:"_source"`
-}

api/pkg/resolver/loader.go

@@ -1,61 +0,0 @@
-package resolver
-
-import (
-	"k8s.io/client-go/kubernetes"
-	_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
-	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
-	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
-	_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
-	restclient "k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/util/homedir"
-	"path/filepath"
-)
-
-func NewFromInCluster(errOut chan error) (*Resolver, error) {
-	config, err := restclient.InClusterConfig()
-	if err != nil {
-		return nil, err
-	}
-	clientset, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		return nil, err
-	}
-	return &Resolver{clientConfig: config, clientSet: clientset, nameMap: make(map[string]string), errOut: errOut}, nil
-}
-
-func NewFromOutOfCluster(kubeConfigPath string, errOut chan error) (*Resolver, error) {
-	if kubeConfigPath == "" {
-		home := homedir.HomeDir()
-		kubeConfigPath = filepath.Join(home, ".kube", "config")
-	}
-
-	configPathList := filepath.SplitList(kubeConfigPath)
-	configLoadingRules := &clientcmd.ClientConfigLoadingRules{}
-	if len(configPathList) <= 1 {
-		configLoadingRules.ExplicitPath = kubeConfigPath
-	} else {
-		configLoadingRules.Precedence = configPathList
-	}
-	contextName := ""
-	clientConfigLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
-		configLoadingRules,
-		&clientcmd.ConfigOverrides{
-			CurrentContext: contextName,
-		},
-	)
-	clientConfig, err := clientConfigLoader.ClientConfig()
-	if err != nil {
-		return nil, err
-	}
-	clientset, err := kubernetes.NewForConfig(clientConfig)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Resolver{clientConfig: clientConfig, clientSet: clientset, nameMap: make(map[string]string), errOut: errOut}, nil
-}
-
-func NewFromExisting(clientConfig *restclient.Config, clientSet *kubernetes.Clientset, errOut chan error) *Resolver {
-	return &Resolver{clientConfig: clientConfig, clientSet: clientSet, nameMap: make(map[string]string), errOut: errOut}
-}

api/pkg/resolver/resolver.go

@@ -1,169 +0,0 @@
-package resolver
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/watch"
-	"k8s.io/client-go/kubernetes"
-	restclient "k8s.io/client-go/rest"
-)
-const (
-	kubClientNullString = "None"
-)
-
-type Resolver struct {
-	clientConfig  *restclient.Config
-	clientSet *kubernetes.Clientset
-	nameMap map[string]string
-	isStarted bool
-	errOut chan error
-}
-
-func (resolver *Resolver) Start(ctx context.Context) {
-	if !resolver.isStarted {
-		resolver.isStarted = true
-		go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchServices)
-		go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchEndpoints)
-		go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchPods)
-	}
-}
-
-func (resolver *Resolver) Resolve(name string) string {
-	resolvedName, isFound := resolver.nameMap[name]
-	if !isFound {
-		return ""
-	}
-	return resolvedName
-}
-
-func (resolver *Resolver) watchPods(ctx context.Context) error {
-	// empty namespace makes the client watch all namespaces
-	watcher, err := resolver.clientSet.CoreV1().Pods("").Watch(ctx, metav1.ListOptions{Watch: true})
-	if err != nil {
-		return err
-	}
-	for {
-		select {
-			case event := <- watcher.ResultChan():
-				if event.Object == nil {
-					return errors.New("error in kubectl pod watch")
-				}
-				if event.Type == watch.Deleted {
-					pod := event.Object.(*corev1.Pod)
-					resolver.saveResolvedName(pod.Status.PodIP, "", event.Type)
-				}
-			case <- ctx.Done():
-				watcher.Stop()
-				return nil
-		}
-	}
-}
-
-func (resolver *Resolver) watchEndpoints(ctx context.Context) error {
-	// empty namespace makes the client watch all namespaces
-	watcher, err := resolver.clientSet.CoreV1().Endpoints("").Watch(ctx, metav1.ListOptions{Watch: true})
-	if err != nil {
-		return err
-	}
-	for {
-		select {
-			case event := <- watcher.ResultChan():
-				if event.Object == nil {
-					return errors.New("error in kubectl endpoint watch")
-				}
-				endpoint := event.Object.(*corev1.Endpoints)
-				serviceHostname := fmt.Sprintf("%s.%s", endpoint.Name, endpoint.Namespace)
-				if endpoint.Subsets != nil {
-					for _, subset := range endpoint.Subsets {
-						var ports []int32
-						if subset.Ports != nil {
-							for _, portMapping := range subset.Ports {
-								if portMapping.Port > 0 {
-									ports = append(ports, portMapping.Port)
-								}
-							}
-						}
-						if subset.Addresses != nil {
-							for _, address := range subset.Addresses {
-								resolver.saveResolvedName(address.IP, serviceHostname, event.Type)
-								for _, port := range ports {
-									ipWithPort := fmt.Sprintf("%s:%d", address.IP, port)
-									resolver.saveResolvedName(ipWithPort, serviceHostname, event.Type)
-								}
-							}
-						}
-
-					}
-				}
-			case <- ctx.Done():
-				watcher.Stop()
-				return nil
-		}
-	}
-}
-
-func (resolver *Resolver) watchServices(ctx context.Context) error {
-	// empty namespace makes the client watch all namespaces
-	watcher, err := resolver.clientSet.CoreV1().Services("").Watch(ctx, metav1.ListOptions{Watch: true})
-	if err != nil {
-		return err
-	}
-	for {
-		select {
-		case event := <- watcher.ResultChan():
-			if event.Object == nil {
-				return errors.New("error in kubectl service watch")
-			}
-
-			service := event.Object.(*corev1.Service)
-			serviceHostname := fmt.Sprintf("%s.%s", service.Name, service.Namespace)
-			if service.Spec.ClusterIP != "" && service.Spec.ClusterIP != kubClientNullString {
-				resolver.saveResolvedName(service.Spec.ClusterIP, serviceHostname, event.Type)
-			}
-			if service.Status.LoadBalancer.Ingress != nil {
-				for _, ingress := range service.Status.LoadBalancer.Ingress {
-					resolver.saveResolvedName(ingress.IP, serviceHostname, event.Type)
-				}
-			}
-		case <- ctx.Done():
-			watcher.Stop()
-			return nil
-		}
-	}
-}
-
-func (resolver *Resolver) saveResolvedName(key string, resolved string, eventType watch.EventType) {
-	if eventType == watch.Deleted {
-		delete(resolver.nameMap, key)
-		// fmt.Printf("setting %s=nil\n", key)
-	} else {
-		resolver.nameMap[key] = resolved
-		// fmt.Printf("setting %s=%s\n", key, resolved)
-	}
-}
-
-func (resolver *Resolver) infiniteErrorHandleRetryFunc(ctx context.Context, fun func(ctx context.Context) error) {
-	for {
-		err := fun(ctx)
-		if err != nil {
-			resolver.errOut <- err
-
-			var statusError *k8serrors.StatusError
-			if errors.As(err, &statusError) {
-				if statusError.ErrStatus.Reason == metav1.StatusReasonForbidden {
-					fmt.Printf("Resolver loop encountered permission error, aborting event listening - %v\n", err)
-					return
-				}
-			}
-		}
-		if ctx.Err() != nil { // context was cancelled or errored
-			return
-		}
-	}
-}
-

api/pkg/routes/not_found_route.go

@@ -1,15 +0,0 @@
-package routes
-
-import "github.com/gofiber/fiber/v2"
-
-// NotFoundRoute func for describe 404 Error route.
-func NotFoundRoute(fiberApp *fiber.App) {
-	fiberApp.Use(
-		func(c *fiber.Ctx) error {
-			return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
-				"error": true,
-				"msg":   "sorry, endpoint is not found",
-			})
-		},
-	)
-}

api/pkg/routes/public_routes.go

@@ -1,20 +0,0 @@
-package routes
-
-import (
-	"github.com/gofiber/fiber/v2"
-	"mizuserver/pkg/controllers"
-)
-
-// EntriesRoutes func for describe group of public routes.
-func EntriesRoutes(fiberApp *fiber.App) {
-	routeGroup := fiberApp.Group("/api")
-
-	routeGroup.Get("/entries", controllers.GetEntries)        // get entries (base/thin entries)
-	routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry
-
-	routeGroup.Get("/har", controllers.GetHARs)
-	routeGroup.Get("/resetDB", controllers.DeleteAllEntries)     // get single (full) entry
-	routeGroup.Get("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB
-
-	routeGroup.Get("/tapStatus", controllers.GetTappingStatus) // get tapping status
-}

api/pkg/routes/socket_routes.go

@@ -1,31 +0,0 @@
-package routes
-
-import (
-	"github.com/antoniodipinto/ikisocket"
-	"github.com/gofiber/fiber/v2"
-)
-
-type EventHandlers interface {
-	WebSocketConnect(ep *ikisocket.EventPayload)
-	WebSocketDisconnect(ep *ikisocket.EventPayload)
-	WebSocketClose(ep *ikisocket.EventPayload)
-	WebSocketError(ep *ikisocket.EventPayload)
-	WebSocketMessage(ep *ikisocket.EventPayload)
-}
-
-func WebSocketRoutes(app *fiber.App, eventHandlers EventHandlers) {
-	app.Get("/ws", ikisocket.New(func(kws *ikisocket.Websocket) {
-		kws.SetAttribute("is_tapper", false)
-	}))
-
-	app.Get("/wsTapper", ikisocket.New(func(kws *ikisocket.Websocket) {
-		// Tapper clients are handled differently, they don't need to receive new message broadcasts.
-		kws.SetAttribute("is_tapper", true)
-	}))
-
-	ikisocket.On(ikisocket.EventMessage, eventHandlers.WebSocketMessage)
-	ikisocket.On(ikisocket.EventConnect, eventHandlers.WebSocketConnect)
-	ikisocket.On(ikisocket.EventDisconnect, eventHandlers.WebSocketDisconnect)
-	ikisocket.On(ikisocket.EventClose, eventHandlers.WebSocketClose) // This event is called when the server disconnects the user actively with .Close() method
-	ikisocket.On(ikisocket.EventError, eventHandlers.WebSocketError) // On error event
-}

api/pkg/tap/http_matcher.go

@@ -1,209 +0,0 @@
-package tap
-
-import (
-	"fmt"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/orcaman/concurrent-map"
-)
-
-type requestResponsePair struct {
-	Request  httpMessage `json:"request"`
-	Response httpMessage `json:"response"`
-}
-
-type envoyMessageWrapper struct {
-	HttpBufferedTrace requestResponsePair `json:"http_buffered_trace"`
-}
-
-type headerKeyVal struct {
-	Key   string `json:"key"`
-	Value string `json:"value"`
-}
-
-type messageBody struct {
-	Truncated bool   `json:"truncated"`
-	AsBytes   string `json:"as_bytes"`
-}
-
-type httpMessage struct {
-	IsRequest       bool
-	Headers         []headerKeyVal `json:"headers"`
-	HTTPVersion     string         `json:"httpVersion"`
-	Body            messageBody    `json:"body"`
-	captureTime     time.Time
-	orig            interface {}
-	requestSenderIp string
-}
-
-
-// Key is {client_addr}:{client_port}->{dest_addr}:{dest_port}
-type requestResponseMatcher struct {
-	openMessagesMap cmap.ConcurrentMap
-
-}
-
-func createResponseRequestMatcher() requestResponseMatcher {
-	newMatcher := &requestResponseMatcher{openMessagesMap: cmap.New()}
-	return *newMatcher
-}
-
-func (matcher *requestResponseMatcher) registerRequest(ident string, request *http.Request, captureTime time.Time, body string, isHTTP2 bool) *envoyMessageWrapper {
-	split := splitIdent(ident)
-	key := genKey(split)
-
-	messageExtraHeaders := []headerKeyVal{
-		{Key: "x-up9-source", Value: split[0]},
-		{Key: "x-up9-destination", Value: split[1] + ":" + split[3]},
-	}
-
-	requestHTTPMessage := requestToMessage(request, captureTime, body, &messageExtraHeaders, isHTTP2, split[0])
-
-	if response, found := matcher.openMessagesMap.Pop(key); found {
-		// Type assertion always succeeds because all of the map's values are of httpMessage type
-		responseHTTPMessage := response.(*httpMessage)
-		if responseHTTPMessage.IsRequest {
-			SilentError("Request-Duplicate", "Got duplicate request with same identifier\n")
-			return nil
-		}
-		Debug("Matched open Response for %s\n", key)
-		return matcher.preparePair(&requestHTTPMessage, responseHTTPMessage)
-	}
-
-	matcher.openMessagesMap.Set(key, &requestHTTPMessage)
-	Debug("Registered open Request for %s\n", key)
-	return nil
-}
-
-func (matcher *requestResponseMatcher) registerResponse(ident string, response *http.Response, captureTime time.Time, body string, isHTTP2 bool) *envoyMessageWrapper {
-	split := splitIdent(ident)
-	key := genKey(split)
-
-	responseHTTPMessage := responseToMessage(response, captureTime, body, isHTTP2)
-
-	if request, found := matcher.openMessagesMap.Pop(key); found {
-		// Type assertion always succeeds because all of the map's values are of httpMessage type
-		requestHTTPMessage := request.(*httpMessage)
-		if !requestHTTPMessage.IsRequest {
-			SilentError("Response-Duplicate", "Got duplicate response with same identifier\n")
-			return nil
-		}
-		Debug("Matched open Request for %s\n", key)
-		return matcher.preparePair(requestHTTPMessage, &responseHTTPMessage)
-	}
-
-	matcher.openMessagesMap.Set(key, &responseHTTPMessage)
-	Debug("Registered open Response for %s\n", key)
-	return nil
-}
-
-func (matcher *requestResponseMatcher) preparePair(requestHTTPMessage *httpMessage, responseHTTPMessage *httpMessage) *envoyMessageWrapper {
-	matcher.addDuration(requestHTTPMessage, responseHTTPMessage)
-
-	return &envoyMessageWrapper{
-		HttpBufferedTrace: requestResponsePair{
-			Request:  *requestHTTPMessage,
-			Response: *responseHTTPMessage,
-		},
-	}
-}
-
-func requestToMessage(request *http.Request, captureTime time.Time, body string, messageExtraHeaders *[]headerKeyVal, isHTTP2 bool, requestSenderIp string) httpMessage {
-	messageHeaders := make([]headerKeyVal, 0)
-
-	for key, value := range request.Header {
-		messageHeaders = append(messageHeaders, headerKeyVal{Key: key, Value: value[0]})
-	}
-
-	if !isHTTP2 {
-		messageHeaders = append(messageHeaders, headerKeyVal{Key: ":method", Value: request.Method})
-		messageHeaders = append(messageHeaders, headerKeyVal{Key: ":path", Value: request.RequestURI})
-		messageHeaders = append(messageHeaders, headerKeyVal{Key: ":authority", Value: request.Host})
-		messageHeaders = append(messageHeaders, headerKeyVal{Key: ":scheme", Value: "http"})
-	}
-
-	messageHeaders = append(messageHeaders, headerKeyVal{Key: "x-request-start", Value: fmt.Sprintf("%.3f", float64(captureTime.UnixNano()) / float64(1000000000))})
-
-	messageHeaders = append(messageHeaders, *messageExtraHeaders...)
-
-	httpVersion := request.Proto
-
-	requestBody := messageBody{Truncated: false, AsBytes: body}
-
-	return httpMessage{
-		IsRequest:       true,
-		Headers:         messageHeaders,
-		HTTPVersion:     httpVersion,
-		Body:            requestBody,
-		captureTime:     captureTime,
-		orig:            request,
-		requestSenderIp: requestSenderIp,
-	}
-}
-
-func responseToMessage(response *http.Response, captureTime time.Time, body string, isHTTP2 bool) httpMessage {
-	messageHeaders := make([]headerKeyVal, 0)
-
-	for key, value := range response.Header {
-		messageHeaders = append(messageHeaders, headerKeyVal{Key: key, Value: value[0]})
-	}
-
-	if !isHTTP2 {
-		messageHeaders = append(messageHeaders, headerKeyVal{Key: ":status", Value: strconv.Itoa(response.StatusCode)})
-	}
-
-	httpVersion := response.Proto
-
-	requestBody := messageBody{Truncated: false, AsBytes: body}
-
-	return httpMessage{
-		IsRequest:   false,
-		Headers:     messageHeaders,
-		HTTPVersion: httpVersion,
-		Body:        requestBody,
-		captureTime: captureTime,
-		orig:        response,
-	}
-}
-
-func (matcher *requestResponseMatcher) addDuration(requestHTTPMessage *httpMessage, responseHTTPMessage *httpMessage) {
-	durationMs := float64(responseHTTPMessage.captureTime.UnixNano() / 1000000) - float64(requestHTTPMessage.captureTime.UnixNano() / 1000000)
-	if durationMs < 1 {
-		durationMs = 1
-	}
-
-	responseHTTPMessage.Headers  = append(responseHTTPMessage.Headers, headerKeyVal{Key: "x-up9-duration-ms", Value: fmt.Sprintf("%.0f", durationMs)})
-}
-
-func splitIdent(ident string) []string {
-	ident = strings.Replace(ident, "->", " ", -1)
-	return strings.Split(ident, " ")
-}
-
-func genKey(split []string) string {
-	key := fmt.Sprintf("%s:%s->%s:%s,%s", split[0], split[2], split[1], split[3], split[4])
-	return key
-}
-
-func (matcher *requestResponseMatcher) deleteOlderThan(t time.Time) int {
-	keysToPop := make([]string, 0)
-	for item := range matcher.openMessagesMap.IterBuffered() {
-		// Map only contains values of type httpMessage
-		message, _ := item.Val.(*httpMessage)
-
-		if message.captureTime.Before(t) {
-			keysToPop = append(keysToPop, item.Key)
-		}
-	}
-
-	numDeleted := len(keysToPop)
-	
-	for _, key := range keysToPop {
-		_, _ = matcher.openMessagesMap.Pop(key)
-	}
-
-	return numDeleted
-}

api/pkg/tap/stats_tracker.go

@@ -1,35 +0,0 @@
-package tap
-
-import (
-	"sync"
-)
-
-type AppStats struct {
-	matchedMessages int
-}
-
-type StatsTracker struct {
-	stats             AppStats
-	statsMutex	  sync.Mutex
-}
-
-func (st *StatsTracker) incMatchedMessages() {
-	st.statsMutex.Lock()
-	st.stats.matchedMessages++
-	st.statsMutex.Unlock()
-}
-
-func (st *StatsTracker) dumpStats() AppStats {
-	st.statsMutex.Lock()
-
-	stats := AppStats{
-		matchedMessages: st.stats.matchedMessages,
-	}
-
-	st.stats.matchedMessages = 0
-
-	st.statsMutex.Unlock()
-
-	return stats
-}
-

api/pkg/tap/tap_output.go

@@ -1,239 +0,0 @@
-package tap
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"log"
-	"net/http"
-	"time"
-
-	"github.com/gorilla/websocket"
-	"github.com/patrickmn/go-cache"
-)
-
-
-const (
-	// Time allowed to write a message to the peer.
-	writeWait = 10 * time.Second
-
-	// Time allowed to read the next pong message from the peer.
-	pongWait = 60 * time.Second
-
-	// Send pings to peer with this period. Must be less than pongWait.
-	pingPeriod = (pongWait * 9) / 10
-
-	// Maximum message size allowed from peer.
-	maxMessageSize = 512
-)
-
-var (
-	newline                           = []byte{'\n'}
-	space                             = []byte{' '}
-	hub                               *Hub
-	outboundSocketNotifyExpiringCache = cache.New(outboundThrottleCacheExpiryPeriod, outboundThrottleCacheExpiryPeriod)
-)
-
-var upgrader = websocket.Upgrader{
-	ReadBufferSize:  1024,
-	WriteBufferSize: 1024,
-	CheckOrigin: func (_ *http.Request) bool { return true },
-}
-
-// Client is a middleman between the websocket connection and the hub.
-type Client struct {
-	hub *Hub
-
-	// The websocket connection.
-	conn *websocket.Conn
-
-	// Buffered channel of outbound messages.
-	send chan []byte
-}
-
-type OutBoundLinkMessage struct {
-	SourceIP string `json:"sourceIP"`
-	IP string `json:"ip"`
-	Port int `json:"port"`
-	Type string `json:"type"`
-}
-
-
-// readPump pumps messages from the websocket connection to the hub.
-//
-// The application runs readPump in a per-connection goroutine. The application
-// ensures that there is at most one reader on a connection by executing all
-// reads from this goroutine.
-func (c *Client) readPump() {
-	defer func() {
-		c.hub.unregister <- c
-		c.conn.Close()
-	}()
-	c.conn.SetReadLimit(maxMessageSize)
-	c.conn.SetReadDeadline(time.Now().Add(pongWait))
-	c.conn.SetPongHandler(func(string) error { c.conn.SetReadDeadline(time.Now().Add(pongWait)); return nil })
-	for {
-		_, message, err := c.conn.ReadMessage()
-		if err != nil {
-			if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) {
-				log.Printf("error: %v", err)
-			}
-			break
-		}
-		message = bytes.TrimSpace(bytes.Replace(message, newline, space, -1))
-		c.hub.onMessageCallback(message)
-	}
-}
-
-// writePump pumps messages from the hub to the websocket connection.
-//
-// A goroutine running writePump is started for each connection. The
-// application ensures that there is at most one writer to a connection by
-// executing all writes from this goroutine.
-func (c *Client) writePump() {
-	ticker := time.NewTicker(pingPeriod)
-	defer func() {
-		ticker.Stop()
-		c.conn.Close()
-	}()
-	for {
-		select {
-		case message, ok := <-c.send:
-			c.conn.SetWriteDeadline(time.Now().Add(writeWait))
-			if !ok {
-				// The hub closed the channel.
-				c.conn.WriteMessage(websocket.CloseMessage, []byte{})
-				return
-			}
-
-			w, err := c.conn.NextWriter(websocket.TextMessage)
-			if err != nil {
-				return
-			}
-			w.Write(message)
-
-
-			if err := w.Close(); err != nil {
-				return
-			}
-		case <-ticker.C:
-			c.conn.SetWriteDeadline(time.Now().Add(writeWait))
-			if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil {
-				return
-			}
-		}
-	}
-}
-
-type Hub struct {
-	// Registered clients.
-	clients map[*Client]bool
-
-	// Inbound messages from the clients.
-	broadcast chan []byte
-
-	// Register requests from the clients.
-	register chan *Client
-
-	// Unregister requests from clients.
-	unregister chan *Client
-
-	// Handle messages from client
-	onMessageCallback func([]byte)
-
-
-}
-
-func newHub(onMessageCallback func([]byte)) *Hub {
-	return &Hub{
-		broadcast:  make(chan []byte),
-		register:   make(chan *Client),
-		unregister: make(chan *Client),
-		clients:    make(map[*Client]bool),
-		onMessageCallback:  onMessageCallback,
-	}
-}
-
-func (h *Hub) run() {
-	for {
-		select {
-		case client := <-h.register:
-			h.clients[client] = true
-		case client := <-h.unregister:
-			if _, ok := h.clients[client]; ok {
-				delete(h.clients, client)
-				close(client.send)
-			}
-		case message := <-h.broadcast:
-			// matched messages counter is incremented in this thread instead of in multiple http reader
-			// threads in order to reduce contention.
-			statsTracker.incMatchedMessages()
-
-			for client := range h.clients {
-				select {
-				case client.send <- message:
-				default:
-					close(client.send)
-					delete(h.clients, client)
-				}
-			}
-		}
-	}
-}
-
-
-// serveWs handles websocket requests from the peer.
-func serveWs(hub *Hub, w http.ResponseWriter, r *http.Request) {
-	conn, err := upgrader.Upgrade(w, r, nil)
-	if err != nil {
-		log.Println(err)
-		return
-	}
-	client := &Client{hub: hub, conn: conn, send: make(chan []byte, 256)}
-	client.hub.register <- client
-
-	// Allow collection of memory referenced by the caller by doing all work in
-	// new goroutines.
-	go client.writePump()
-	go client.readPump()
-}
-
-func startOutputServer(port string, messageCallback func([]byte)) {
-	hub = newHub(messageCallback)
-	go hub.run()
-	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		serveWs(hub, w, r)
-	})
-	err := http.ListenAndServe("0.0.0.0:" + port, nil)
-	if err != nil {
-		log.Fatal("Output server error: ", err)
-	}
-}
-
-func broadcastReqResPair(reqResJson []byte) {
-	hub.broadcast <- reqResJson
-}
-
-func broadcastOutboundLink(srcIP string, dstIP string, dstPort int) {
-	cacheKey := fmt.Sprintf("%s -> %s:%d", srcIP, dstIP, dstPort)
-	_, isInCache := outboundSocketNotifyExpiringCache.Get(cacheKey)
-	if isInCache {
-		return
-	} else {
-		outboundSocketNotifyExpiringCache.SetDefault(cacheKey, true)
-	}
-
-	socketMessage := OutBoundLinkMessage{
-		SourceIP: srcIP,
-		IP:   dstIP,
-		Port: dstPort,
-		Type: "outboundSocketDetected",
-	}
-
-	jsonStr, err := json.Marshal(socketMessage)
-	if err != nil {
-		log.Printf("error marshalling outbound socket detection object: %v", err)
-	} else {
-		hub.broadcast <- jsonStr
-	}
-}

api/pkg/tap/tcp_stream_factory.go

@@ -1,117 +0,0 @@
-package tap
-
-import (
-	"fmt"
-	"sync"
-
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers" // pulls in all layers decoders
-	"github.com/google/gopacket/reassembly"
-)
-
-/*
- * The TCP factory: returns a new Stream
- * Implements gopacket.reassembly.StreamFactory interface (New)
- * Generates a new tcp stream for each new tcp connection. Closes the stream when the connection closes.
- */
-type tcpStreamFactory struct {
-	wg        sync.WaitGroup
-	doHTTP    bool
-	harWriter *HarWriter
-}
-
-func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.TCP, ac reassembly.AssemblerContext) reassembly.Stream {
-	Debug("* NEW: %s %s\n", net, transport)
-	fsmOptions := reassembly.TCPSimpleFSMOptions{
-		SupportMissingEstablishment: *allowmissinginit,
-	}
-	Debug("Current App Ports: %v\n", appPorts)
-	dstIp := net.Dst().String()
-	dstPort := int(tcp.DstPort)
-
-	if factory.shouldNotifyOnOutboundLink(dstIp, dstPort) {
-		broadcastOutboundLink(net.Src().String(), dstIp, dstPort)
-	}
-	isHTTP := factory.shouldTap(dstIp, dstPort)
-	stream := &tcpStream{
-		net:        net,
-		transport:  transport,
-		isDNS:      tcp.SrcPort == 53 || tcp.DstPort == 53,
-		isHTTP:     isHTTP && factory.doHTTP,
-		reversed:   tcp.SrcPort == 80,
-		tcpstate:   reassembly.NewTCPSimpleFSM(fsmOptions),
-		ident:      fmt.Sprintf("%s:%s", net, transport),
-		optchecker: reassembly.NewTCPOptionCheck(),
-	}
-	if stream.isHTTP {
-		stream.client = httpReader{
-			msgQueue: make(chan httpReaderDataMsg),
-			ident:    fmt.Sprintf("%s %s", net, transport),
-			tcpID: tcpID{
-				srcIP:   net.Src().String(),
-				dstIP:   net.Dst().String(),
-				srcPort: transport.Src().String(),
-				dstPort: transport.Dst().String(),
-			},
-			hexdump:  *hexdump,
-			parent:   stream,
-			isClient: true,
-			harWriter: factory.harWriter,
-		}
-		stream.server = httpReader{
-			msgQueue: make(chan httpReaderDataMsg),
-			ident:    fmt.Sprintf("%s %s", net.Reverse(), transport.Reverse()),
-			tcpID: tcpID{
-				srcIP:   net.Dst().String(),
-				dstIP:   net.Src().String(),
-				srcPort: transport.Dst().String(),
-				dstPort: transport.Src().String(),
-			},
-			hexdump: *hexdump,
-			parent:  stream,
-			harWriter: factory.harWriter,
-		}
-		factory.wg.Add(2)
-		// Start reading from channels stream.client.bytes and stream.server.bytes
-		go stream.client.run(&factory.wg)
-		go stream.server.run(&factory.wg)
-	}
-	return stream
-}
-
-func (factory *tcpStreamFactory) WaitGoRoutines() {
-	factory.wg.Wait()
-}
-
-func (factory *tcpStreamFactory) shouldTap(dstIP string, dstPort int) bool {
-	if hostMode {
-		if inArrayString(HostAppAddresses, fmt.Sprintf("%s:%d", dstIP, dstPort)) == true {
-			return true
-		} else if inArrayString(HostAppAddresses, dstIP) == true {
-			return true
-		}
-		return false
-	} else {
-		isTappedPort := dstPort == 80 || (appPorts != nil && (inArrayInt(appPorts, dstPort)))
-		if !isTappedPort {
-			return false
-		}
-
-		if !*anydirection {
-			isDirectedHere := inArrayString(ownIps, dstIP)
-			if !isDirectedHere {
-				return false
-			}
-		}
-
-		return true
-	}
-}
-
-func (factory *tcpStreamFactory) shouldNotifyOnOutboundLink(dstIP string, dstPort int) bool {
-	if inArrayInt(remoteOnlyOutboundPorts, dstPort) {
-		isDirectedHere := inArrayString(ownIps, dstIP)
-		return !isDirectedHere && !isPrivateIP(dstIP)
-	}
-	return true
-}

api/start.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-./mizuagent -i any -hardump -targets ${TAPPED_ADDRESSES}

build-agent-ci.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+GCP_PROJECT=up9-docker-hub
+REPOSITORY=gcr.io/$GCP_PROJECT
+SERVER_NAME=mizu
+GIT_BRANCH=ci
+
+DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
+SEM_VER=${SEM_VER=0.0.0}
+
+DOCKER_TAGGED_BUILD="$DOCKER_REPO:$SEM_VER"
+
+echo "building $DOCKER_TAGGED_BUILD"
+docker build -t ${DOCKER_TAGGED_BUILD} --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .

build-push-featurebranch.sh

@@ -1,11 +1,15 @@
 #!/bin/bash
 set -e
 
-SERVER_NAME=mizu
 GCP_PROJECT=up9-docker-hub
 REPOSITORY=gcr.io/$GCP_PROJECT
+SERVER_NAME=mizu
 GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
-DOCKER_TAGGED_BUILD=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH:latest
+
+DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
+SEM_VER=${SEM_VER=0.0.0}
+
+DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
 
 if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
 then
@@ -13,8 +17,12 @@ then
   exit 1
 fi
 
-echo "building $DOCKER_TAGGED_BUILD"
-docker build -t "$DOCKER_TAGGED_BUILD" .
+echo "building ${DOCKER_TAGGED_BUILDS[@]}"
+DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
+docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
 
-echo pushing to "$REPOSITORY"
-docker push "$DOCKER_TAGGED_BUILD"
+for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
+do
+  echo pushing "$DOCKER_TAG"
+  docker push "$DOCKER_TAG"
+done

cli/.gitignore

@@ -0,0 +1 @@
+bin

cli/Makefile

@@ -1,6 +1,9 @@
-FOLDER=$(GOOS).$(GOARCH)
+SUFFIX=$(GOOS)_$(GOARCH)
 COMMIT_HASH=$(shell git rev-parse HEAD)
 GIT_BRANCH=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
+GIT_VERSION=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
+BUILD_TIMESTAMP=$(shell date +%s)
+export SEM_VER?=0.0.0
 
 .PHONY: help
 .DEFAULT_GOAL := help
@@ -11,22 +14,31 @@ help: ## This help.
 install:
 	go install mizu.go
 
-build:	## build mizu CLI binary (select platform via GOOS / GOARCH env variables)
-	go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)'" -o bin/$(FOLDER)/mizu mizu.go
+build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
+	go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
+					   -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
+					   -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
+					   -X 'github.com/up9inc/mizu/cli/mizu.SemVer=$(SEM_VER)'" \
+					   -o bin/mizu_$(SUFFIX) mizu.go 
+	(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
 
-build-all:  ## build for all supported platforms
+build-all: ## Build for all supported platforms.
 	@echo "Compiling for every OS and Platform"
+	@mkdir -p bin && echo "SHA256 checksums available for compiled binaries \n\nRun \`shasum -a 256 -c mizu_OS_ARCH.sha256\` to verify\n\n" >  bin/README.md
 	@$(MAKE) build GOOS=darwin GOARCH=amd64
 	@$(MAKE) build GOOS=linux GOARCH=amd64
+	@# $(MAKE) build GOOS=darwin GOARCH=arm64
 	@# $(MAKE) GOOS=windows GOARCH=amd64
 	@# $(MAKE) GOOS=linux GOARCH=386
 	@# $(MAKE) GOOS=windows GOARCH=386
-	@# $(MAKE) GOOS=darwin GOARCH=arm64
 	@# $(MAKE) GOOS=linux GOARCH=arm64
 	@# $(MAKE) GOOS=windows GOARCH=arm64
 	@echo "---------"
 	@find ./bin -ls
 
-clean: ## clean all build artifacts
+clean: ## Clean all build artifacts.
 	go clean
 	rm -rf ./bin/*
+
+test: ## Run cli tests.
+	@go test ./... -coverpkg=./... -race -coverprofile=coverage.out -covermode=atomic

cli/README.md

@@ -1,26 +0,0 @@
-# mizu CLI
-## Usage
-`./mizu {pod_name_regex}`
-
-### Optional Flags
-
-| flag                 | default          | purpose                                                                                                      |
-|----------------------|------------------|--------------------------------------------------------------------------------------------------------------|
-| `--no-gui`           | `false`          | Don't host the web interface (not applicable at the moment)                                                      |
-| `--gui-port`         | `8899`           | local port that web interface will be forwarded to                                                               |
-| `--namespace`        |                  | use namespace different than the one found in kubeconfig                                                     |
-| `--kubeconfig`       |                  | Path to custom kubeconfig file                                                                               |
-
-There are some extra flags defined in code that will show up in `./mizu --help`, these are non functional stubs for now
-
-## Installation
-Make sure your go version is at least 1.11
-1. cd to `mizu/cli`
-2. Run `go mod download` (may take a moment)
-3. Run `go build mizu.go`
-
-Alternatively, you can build+run directly using `go run mizu.go {pod_name_regex}`
-
-
-## Known issues
-* mid-flight port forwarding failures are not detected and no indication will be shown when this occurs

cli/cmd/config.go

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/creasty/defaults"
+	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/telemetry"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"io/ioutil"
+)
+
+var configCmd = &cobra.Command{
+	Use:   "config",
+	Short: "Generate config with default values",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("config", config.Config.Config)
+
+		template, err := config.GetConfigWithDefaults()
+		if err != nil {
+			logger.Log.Errorf("Failed generating config with defaults %v", err)
+			return nil
+		}
+		if config.Config.Config.Regenerate {
+			data := []byte(template)
+			if err := ioutil.WriteFile(config.GetConfigFilePath(), data, 0644); err != nil {
+				logger.Log.Errorf("Failed writing config %v", err)
+				return nil
+			}
+			logger.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, config.GetConfigFilePath())))
+		} else {
+			logger.Log.Debugf("Writing template config.\n%v", template)
+			fmt.Printf("%v", template)
+		}
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(configCmd)
+
+	defaultConfigConfig := configStructs.ConfigConfig{}
+	defaults.Set(&defaultConfigConfig)
+
+	configCmd.Flags().BoolP(configStructs.RegenerateConfigName, "r", defaultConfigConfig.Regenerate, fmt.Sprintf("Regenerate the config file with default values %s", config.GetConfigFilePath()))
+}

cli/cmd/fetch.go

@@ -1,21 +1,26 @@
 package cmd
 
 import (
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/mizu/version"
+	"github.com/up9inc/mizu/cli/telemetry"
 )
 
-type MizuFetchOptions struct {
-	Limit        uint16
-	Directory	 string
-}
-
-var mizuFetchOptions = MizuFetchOptions{}
-
 var fetchCmd = &cobra.Command{
 	Use:   "fetch",
 	Short: "Download recorded traffic to files",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		RunMizuFetch(&mizuFetchOptions)
+		go telemetry.ReportRun("fetch", config.Config.Fetch)
+
+		if isCompatible, err := version.CheckVersionCompatibility(config.Config.Fetch.GuiPort); err != nil {
+			return err
+		} else if !isCompatible {
+			return nil
+		}
+		RunMizuFetch()
 		return nil
 	},
 }
@@ -23,6 +28,11 @@ var fetchCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(fetchCmd)
 
-	fetchCmd.Flags().Uint16VarP(&mizuFetchOptions.Limit, "limit", "l", 1000, "Provide a custom limit for entries to fetch")
-	fetchCmd.Flags().StringVarP(&mizuFetchOptions.Directory, "directory", "d", ".", "Provide a custom directory for fetched entries")
+	defaultFetchConfig := configStructs.FetchConfig{}
+	defaults.Set(&defaultFetchConfig)
+
+	fetchCmd.Flags().StringP(configStructs.DirectoryFetchName, "d", defaultFetchConfig.Directory, "Provide a custom directory for fetched entries")
+	fetchCmd.Flags().Int(configStructs.FromTimestampFetchName, defaultFetchConfig.FromTimestamp, "Custom start timestamp for fetched entries")
+	fetchCmd.Flags().Int(configStructs.ToTimestampFetchName, defaultFetchConfig.ToTimestamp, "Custom end timestamp fetched entries")
+	fetchCmd.Flags().Uint16P(configStructs.GuiPortFetchName, "p", defaultFetchConfig.GuiPort, "Provide a custom port for the web interface webserver")
 }

cli/cmd/fetchRunner.go

@@ -4,6 +4,9 @@ import (
 	"archive/zip"
 	"bytes"
 	"fmt"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
 	"io"
 	"io/ioutil"
 	"log"
@@ -13,8 +16,9 @@ import (
 	"strings"
 )
 
-func RunMizuFetch(fetch *MizuFetchOptions) {
-	resp, err :=  http.Get(fmt.Sprintf("http://localhost:8899/api/har?limit=%v", fetch.Limit))
+func RunMizuFetch() {
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Fetch.GuiPort)
+	resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, config.Config.Fetch.FromTimestamp, config.Config.Fetch.ToTimestamp))
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -30,8 +34,8 @@ func RunMizuFetch(fetch *MizuFetchOptions) {
 	if err != nil {
 		log.Fatal(err)
 	}
-	_ = Unzip(zipReader, fetch.Directory)
 
+	_ = Unzip(zipReader, config.Config.Fetch.Directory)
 }
 
 func Unzip(reader *zip.Reader, dest string) error {
@@ -53,7 +57,7 @@ func Unzip(reader *zip.Reader, dest string) error {
 		path := filepath.Join(dest, f.Name)
 
 		// Check for ZipSlip (Directory traversal)
-		if !strings.HasPrefix(path, filepath.Clean(dest) + string(os.PathSeparator)) {
+		if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) {
 			return fmt.Errorf("illegal file path: %s", path)
 		}
 
@@ -61,7 +65,7 @@ func Unzip(reader *zip.Reader, dest string) error {
 			_ = os.MkdirAll(path, f.Mode())
 		} else {
 			_ = os.MkdirAll(filepath.Dir(path), f.Mode())
-            fmt.Print("writing HAR file [ ", path, " ] .. ")
+			logger.Log.Infof("writing HAR file [ %v ]", path)
 			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
 			if err != nil {
 				return err
@@ -70,7 +74,7 @@ func Unzip(reader *zip.Reader, dest string) error {
 				if err := f.Close(); err != nil {
 					panic(err)
 				}
-                fmt.Println(" done")
+				logger.Log.Info(" done")
 			}()
 
 			_, err = io.Copy(f, rc)
@@ -90,5 +94,3 @@ func Unzip(reader *zip.Reader, dest string) error {
 
 	return nil
 }
-
-

cli/cmd/logs.go

@@ -0,0 +1,49 @@
+package cmd
+
+import (
+	"context"
+	"github.com/creasty/defaults"
+	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/errormessage"
+	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu/fsUtils"
+	"github.com/up9inc/mizu/cli/telemetry"
+)
+
+var logsCmd = &cobra.Command{
+	Use:   "logs",
+	Short: "Create a zip file with logs for Github issue or troubleshoot",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("logs", config.Config.Logs)
+
+		kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath)
+		if err != nil {
+			return nil
+		}
+		ctx, _ := context.WithCancel(context.Background())
+
+		if validationErr := config.Config.Logs.Validate(); validationErr != nil {
+			return errormessage.FormatError(validationErr)
+		}
+
+		logger.Log.Debugf("Using file path %s", config.Config.Logs.FilePath())
+
+		if dumpLogsErr := fsUtils.DumpLogs(kubernetesProvider, ctx, config.Config.Logs.FilePath()); dumpLogsErr != nil {
+			logger.Log.Errorf("Failed dump logs %v", dumpLogsErr)
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(logsCmd)
+
+	defaultLogsConfig := configStructs.LogsConfig{}
+	defaults.Set(&defaultLogsConfig)
+
+	logsCmd.Flags().StringP(configStructs.FileLogsName, "f", defaultLogsConfig.FileStr, "Path for zip file (default current <pwd>\\mizu_logs.zip)")
+}

cli/cmd/root.go

@@ -1,7 +1,12 @@
 package cmd
 
 import (
+	"fmt"
 	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 )
 
 var rootCmd = &cobra.Command{
@@ -9,6 +14,21 @@ var rootCmd = &cobra.Command{
 	Short: "A web traffic viewer for kubernetes",
 	Long: `A web traffic viewer for kubernetes
 Further info is available at https://github.com/up9inc/mizu`,
+	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		if err := fsUtils.EnsureDir(mizu.GetMizuFolderPath()); err != nil {
+			logger.Log.Errorf("Failed to use mizu folder, %v", err)
+		}
+		logger.InitLogger()
+		if err := config.InitConfig(cmd); err != nil {
+			logger.Log.Fatal(err)
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringSlice(config.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", config.SetCommandName))
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.

cli/cmd/tap.go

@@ -2,44 +2,55 @@ package cmd
 
 import (
 	"errors"
-	"fmt"
-	"github.com/up9inc/mizu/cli/mizu"
-	"regexp"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/telemetry"
+	"os"
 
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/errormessage"
+	"github.com/up9inc/mizu/cli/uiUtils"
 )
 
-type MizuTapOptions struct {
-	GuiPort                uint16
-	Namespace              string
-	AllNamespaces          bool
-	KubeConfigPath         string
-	MizuImage              string
-	MizuPodPort            uint16
-	PlainTextFilterRegexes []string
-}
-
-
-var mizuTapOptions = &MizuTapOptions{}
+const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic for further analysis and enriched presentation options.`
 
 var tapCmd = &cobra.Command{
 	Use:   "tap [POD REGEX]",
 	Short: "Record ingoing traffic of a kubernetes pod",
 	Long: `Record the ingoing traffic of a kubernetes pod.
- Supported protocols are HTTP and gRPC.`,
+Supported protocols are HTTP and gRPC.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		if len(args) == 0 {
-			return errors.New("POD REGEX argument is required")
+		go telemetry.ReportRun("tap", config.Config.Tap)
+		RunMizuTap()
+		return nil
+	},
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		if len(args) == 1 {
+			config.Config.Tap.PodRegexStr = args[0]
 		} else if len(args) > 1 {
 			return errors.New("unexpected number of arguments")
 		}
 
-		regex, err := regexp.Compile(args[0])
-		if err != nil {
-			return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], err))
+		if err := config.Config.Validate(); err != nil {
+			return errormessage.FormatError(err)
+		}
+
+		if err := config.Config.Tap.Validate(); err != nil {
+			return errormessage.FormatError(err)
+		}
+
+		logger.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", config.Config.Tap.HumanMaxEntriesDBSize)
+
+		if config.Config.Tap.Analysis {
+			logger.Log.Infof(analysisMessageToConfirm)
+			if !uiUtils.AskForConfirmation("Would you like to proceed [Y/n]: ") {
+				logger.Log.Infof("You can always run mizu without analysis, aborting")
+				os.Exit(0)
+			}
 		}
 
-		RunMizuTap(regex, mizuTapOptions)
 		return nil
 	},
 }
@@ -47,11 +58,17 @@ var tapCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(tapCmd)
 
-	tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
-	tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector")
-	tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces")
-	tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
-	tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector")
-	tapCmd.Flags().Uint16VarP(&mizuTapOptions.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod")
-	tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "regex-masking", "r", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies")
+	defaultTapConfig := configStructs.TapConfig{}
+	defaults.Set(&defaultTapConfig)
+
+	tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
+	tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")
+	tapCmd.Flags().Bool(configStructs.AnalysisTapName, defaultTapConfig.Analysis, "Uploads traffic to UP9 for further analysis (Beta)")
+	tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
+	tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
+	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
+	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
+	tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any")
+	tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
+	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file with policy rules")
 }

cli/cmd/tapRunner.go

@@ -1,169 +1,396 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
-	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu/fsUtils"
+	"github.com/up9inc/mizu/cli/mizu/goUtils"
+	"github.com/up9inc/mizu/cli/mizu/version"
+	"github.com/up9inc/mizu/cli/telemetry"
+	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
+	"path"
 	"regexp"
+	"strings"
 	"syscall"
 	"time"
 
-	core "k8s.io/api/core/v1"
-
-	"github.com/up9inc/mizu/cli/debounce"
+	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/kubernetes"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/debounce"
+	yaml "gopkg.in/yaml.v3"
+	core "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
-var mizuServiceAccountExists bool
-var aggregatorService *core.Service
-
 const (
+	cleanupTimeout     = time.Minute
 	updateTappersDelay = 5 * time.Second
 )
 
-var currentlyTappedPods []core.Pod
+type tapState struct {
+	apiServerService         *core.Service
+	currentlyTappedPods      []core.Pod
+	mizuServiceAccountExists bool
+	doNotRemoveConfigMap     bool
+}
 
-func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
-	mizuApiFilteringOptions, err := getMizuApiFilteringOptions(tappingOptions)
+var state tapState
+
+func RunMizuTap() {
+	mizuApiFilteringOptions, err := getMizuApiFilteringOptions()
 	if err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error parsing regex-masking: %v", errormessage.FormatError(err)))
+		return
+	}
+	var mizuValidationRules string
+	if config.Config.Tap.EnforcePolicyFile != "" {
+		mizuValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile)
+		if err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading policy file: %v", errormessage.FormatError(err)))
+			return
+		}
+	}
+
+	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath)
+	if err != nil {
+		logger.Log.Error(err)
 		return
 	}
 
-	kubernetesProvider := kubernetes.NewProvider(tappingOptions.KubeConfigPath)
-
-	defer cleanUpMizuResources(kubernetesProvider)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel() // cancel will be called when this function exits
 
-	if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegexQuery); err != nil {
-		return
+	targetNamespaces := getNamespaces(kubernetesProvider)
+
+	var namespacesStr string
+	if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
+		namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(targetNamespaces, "\", \""))
 	} else {
-		currentlyTappedPods = matchingPods
+		namespacesStr = "all namespaces"
 	}
+	version.CheckNewerVersion()
+	logger.Log.Infof("Tapping pods in %s", namespacesStr)
 
-	nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
-	if err != nil {
+	if err, _ := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespaces); err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error getting pods by regex: %v", errormessage.FormatError(err)))
 		return
 	}
 
-	if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions, mizuApiFilteringOptions); err != nil {
+	if len(state.currentlyTappedPods) == 0 {
+		var suggestionStr string
+		if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
+			suggestionStr = ". Select a different namespace with -n or tap all namespaces with -A"
+		}
+		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr))
+	}
+
+	if config.Config.Tap.DryRun {
 		return
 	}
 
-	go portForwardApiPod(ctx, kubernetesProvider, cancel, tappingOptions) // TODO convert this to job for built in pod ttl or have the running app handle this
-	go watchPodsForTapping(ctx, kubernetesProvider, cancel, podRegexQuery, tappingOptions)
-	go syncApiStatus(ctx, cancel, tappingOptions)
+	nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
+
+	defer cleanUpMizu(kubernetesProvider)
+	if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
+		return
+	}
+
+	go goUtils.HandleExcWrapper(createProxyToApiServerPod, ctx, kubernetesProvider, cancel)
+	go goUtils.HandleExcWrapper(watchPodsForTapping, ctx, kubernetesProvider, targetNamespaces, cancel)
 
 	//block until exit signal or error
 	waitForFinish(ctx, cancel)
-
-	// TODO handle incoming traffic from tapper using a channel
 }
 
-func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
-	if err := createMizuAggregator(ctx, kubernetesProvider, tappingOptions, mizuApiFilteringOptions); err != nil {
+func readValidationRules(file string) (string, error) {
+	rules, err := shared.DecodeEnforcePolicy(file)
+	if err != nil {
+		return "", err
+	}
+	newContent, _ := yaml.Marshal(&rules)
+	return string(newContent), nil
+}
+
+func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, mizuApiFilteringOptions *shared.TrafficFilteringOptions, mizuValidationRules string) error {
+	if !config.Config.IsNsRestrictedMode() {
+		if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
+			return err
+		}
+	}
+
+	if err := createMizuApiServer(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
 		return err
 	}
 
-	if err := createMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
+	if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
 		return err
 	}
 
+	if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules); err != nil {
+		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err)))
+		state.doNotRemoveConfigMap = true
+	} else if mizuValidationRules == "" {
+		state.doNotRemoveConfigMap = true
+	}
+
 	return nil
 }
 
-func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
+func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, data string) error {
+	err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName, data)
+	return err
+}
+
+func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider) error {
+	_, err := kubernetesProvider.CreateNamespace(ctx, config.Config.MizuResourcesNamespace)
+	return err
+}
+
+func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
 	var err error
 
-	mizuServiceAccountExists = createRBACIfNecessary(ctx, kubernetesProvider)
-	_, err = kubernetesProvider.CreateMizuAggregatorPod(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.MizuImage, mizuServiceAccountExists, mizuApiFilteringOptions)
+	state.mizuServiceAccountExists, err = createRBACIfNecessary(ctx, kubernetesProvider)
 	if err != nil {
-		fmt.Printf("Error creating mizu collector pod: %v\n", err)
-		return err
+		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err)))
 	}
 
-	aggregatorService, err = kubernetesProvider.CreateService(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, mizu.AggregatorPodName)
+	var serviceAccountName string
+	if state.mizuServiceAccountExists {
+		serviceAccountName = mizu.ServiceAccountName
+	} else {
+		serviceAccountName = ""
+	}
+
+	opts := &kubernetes.ApiServerOptions{
+		Namespace:               config.Config.MizuResourcesNamespace,
+		PodName:                 mizu.ApiServerPodName,
+		PodImage:                config.Config.AgentImage,
+		ServiceAccountName:      serviceAccountName,
+		IsNamespaceRestricted:   config.Config.IsNsRestrictedMode(),
+		MizuApiFilteringOptions: mizuApiFilteringOptions,
+		MaxEntriesDBSizeBytes:   config.Config.Tap.MaxEntriesDBSizeBytes(),
+		Resources:               config.Config.Tap.ApiServerResources,
+		ImagePullPolicy:         config.Config.ImagePullPolicy(),
+	}
+	_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, opts)
 	if err != nil {
-		fmt.Printf("Error creating mizu collector service: %v\n", err)
 		return err
 	}
+	logger.Log.Debugf("Successfully created API server pod: %s", mizu.ApiServerPodName)
+
+	state.apiServerService, err = kubernetesProvider.CreateService(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName, mizu.ApiServerPodName)
+	if err != nil {
+		return err
+	}
+	logger.Log.Debugf("Successfully created service: %s", mizu.ApiServerPodName)
 
 	return nil
 }
 
-func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.TrafficFilteringOptions, error) {
-	if tappingOptions.PlainTextFilterRegexes == nil || len(tappingOptions.PlainTextFilterRegexes) == 0 {
-		return nil, nil
-	}
+func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
+	var compiledRegexSlice []*shared.SerializableRegexp
 
-	compiledRegexSlice := make([]*shared.SerializableRegexp, 0)
-	for _, regexStr := range tappingOptions.PlainTextFilterRegexes {
-		compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr)
-		if err != nil {
-			fmt.Printf("Regex %s is invalid: %v", regexStr, err)
-			return nil, err
+	if config.Config.Tap.PlainTextFilterRegexes != nil && len(config.Config.Tap.PlainTextFilterRegexes) > 0 {
+		compiledRegexSlice = make([]*shared.SerializableRegexp, 0)
+		for _, regexStr := range config.Config.Tap.PlainTextFilterRegexes {
+			compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr)
+			if err != nil {
+				return nil, err
+			}
+			compiledRegexSlice = append(compiledRegexSlice, compiledRegex)
 		}
-		compiledRegexSlice = append(compiledRegexSlice, compiledRegex)
 	}
 
-	return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice}, nil
+	return &shared.TrafficFilteringOptions{
+		PlainTextMaskingRegexes:      compiledRegexSlice,
+		HealthChecksUserAgentHeaders: config.Config.Tap.HealthChecksUserAgentHeaders,
+		DisableRedaction:             config.Config.Tap.DisableRedaction,
+	}, nil
 }
 
-func createMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error {
-	if err := kubernetesProvider.ApplyMizuTapperDaemonSet(
-		ctx,
-		mizu.ResourcesNamespace,
-		mizu.TapperDaemonSetName,
-		tappingOptions.MizuImage,
-		mizu.TapperPodName,
-		fmt.Sprintf("%s.%s.svc.cluster.local", aggregatorService.Name, aggregatorService.Namespace),
-		nodeToTappedPodIPMap,
-		mizuServiceAccountExists,
-	); err != nil {
-		fmt.Printf("Error creating mizu tapper daemonset: %v\n", err)
-		return err
+func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error {
+	if len(nodeToTappedPodIPMap) > 0 {
+		var serviceAccountName string
+		if state.mizuServiceAccountExists {
+			serviceAccountName = mizu.ServiceAccountName
+		} else {
+			serviceAccountName = ""
+		}
+
+		if err := kubernetesProvider.ApplyMizuTapperDaemonSet(
+			ctx,
+			config.Config.MizuResourcesNamespace,
+			mizu.TapperDaemonSetName,
+			config.Config.AgentImage,
+			mizu.TapperPodName,
+			fmt.Sprintf("%s.%s.svc.cluster.local", state.apiServerService.Name, state.apiServerService.Namespace),
+			nodeToTappedPodIPMap,
+			serviceAccountName,
+			config.Config.Tap.TapOutgoing(),
+			config.Config.Tap.TapperResources,
+			config.Config.ImagePullPolicy(),
+		); err != nil {
+			return err
+		}
+		logger.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap))
+	} else {
+		if err := kubernetesProvider.RemoveDaemonSet(ctx, config.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
+			return err
+		}
 	}
 
 	return nil
 }
 
+func cleanUpMizu(kubernetesProvider *kubernetes.Provider) {
+	telemetry.ReportAPICalls(config.Config.Tap.GuiPort)
+	cleanUpMizuResources(kubernetesProvider)
+}
+
 func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
-	fmt.Printf("\nRemoving mizu resources\n")
+	removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
+	defer cancel()
 
-	removalCtx, _ := context.WithTimeout(context.Background(), 5*time.Second)
-	if err := kubernetesProvider.RemovePod(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil {
-		fmt.Printf("Error removing Pod %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err)
+	if config.Config.DumpLogs {
+		mizuDir := mizu.GetMizuFolderPath()
+		filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
+		if err := fsUtils.DumpLogs(kubernetesProvider, removalCtx, filePath); err != nil {
+			logger.Log.Errorf("Failed dump logs %v", err)
+		}
 	}
-	if err := kubernetesProvider.RemoveService(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil {
-		fmt.Printf("Error removing Service %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err)
+
+	logger.Log.Infof("\nRemoving mizu resources\n")
+
+	if !config.Config.IsNsRestrictedMode() {
+		if err := kubernetesProvider.RemoveNamespace(removalCtx, config.Config.MizuResourcesNamespace); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Namespace %s: %v", config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			return
+		}
+	} else {
+		if err := kubernetesProvider.RemovePod(removalCtx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Pod %s in namespace %s: %v", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+		}
+
+		if err := kubernetesProvider.RemoveService(removalCtx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service %s in namespace %s: %v", mizu.ApiServerPodName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+		}
+
+		if err := kubernetesProvider.RemoveDaemonSet(removalCtx, config.Config.MizuResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing DaemonSet %s in namespace %s: %v", mizu.TapperDaemonSetName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+		}
+
+		if !state.doNotRemoveConfigMap {
+			if err := kubernetesProvider.RemoveConfigMap(removalCtx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing ConfigMap %s in namespace %s: %v", mizu.ConfigMapName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			}
+		}
+
 	}
-	if err := kubernetesProvider.RemoveDaemonSet(removalCtx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
-		fmt.Printf("Error removing DaemonSet %s in namespace %s: %s (%v,%+v)\n", mizu.TapperDaemonSetName, mizu.ResourcesNamespace, err, err, err)
+
+	if state.mizuServiceAccountExists {
+		if !config.Config.IsNsRestrictedMode() {
+			if err := kubernetesProvider.RemoveNonNamespacedResources(removalCtx, mizu.ClusterRoleName, mizu.ClusterRoleBindingName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing non-namespaced resources: %v", errormessage.FormatError(err)))
+				return
+			}
+		} else {
+			if err := kubernetesProvider.RemoveServicAccount(removalCtx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Service Account %s in namespace %s: %v", mizu.ServiceAccountName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+				return
+			}
+
+			if err := kubernetesProvider.RemoveRole(removalCtx, config.Config.MizuResourcesNamespace, mizu.RoleName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing Role %s in namespace %s: %v", mizu.RoleName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			}
+
+			if err := kubernetesProvider.RemoveRoleBinding(removalCtx, config.Config.MizuResourcesNamespace, mizu.RoleBindingName); err != nil {
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error removing RoleBinding %s in namespace %s: %v", mizu.RoleBindingName, config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+			}
+		}
+	}
+
+	if !config.Config.IsNsRestrictedMode() {
+		waitUntilNamespaceDeleted(removalCtx, cancel, kubernetesProvider)
 	}
 }
 
-func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, podRegex *regexp.Regexp, tappingOptions *MizuTapOptions) {
-	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, getNamespace(tappingOptions, kubernetesProvider)), podRegex)
+func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
+	// Call cancel if a terminating signal was received. Allows user to skip the wait.
+	go func() {
+		waitForFinish(ctx, cancel)
+	}()
+
+	if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, config.Config.MizuResourcesNamespace); err != nil {
+		switch {
+		case ctx.Err() == context.Canceled:
+			// Do nothing. User interrupted the wait.
+		case err == wait.ErrWaitTimeout:
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", config.Config.MizuResourcesNamespace))
+		default:
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error while waiting for Namespace %s to be deleted: %v", config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
+		}
+	}
+}
+
+func reportTappedPods() {
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort)
+	tappedPodsUrl := fmt.Sprintf("http://%s/status/tappedPods", mizuProxiedUrl)
+
+	podInfos := make([]shared.PodInfo, 0)
+	for _, pod := range state.currentlyTappedPods {
+		podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
+	}
+	tapStatus := shared.TapStatus{Pods: podInfos}
+
+	if jsonValue, err := json.Marshal(tapStatus); err != nil {
+		logger.Log.Debugf("[ERROR] failed Marshal the tapped pods %v", err)
+	} else {
+		if response, err := http.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+			logger.Log.Debugf("[ERROR] failed sending to API server the tapped pods %v", err)
+		} else if response.StatusCode != 200 {
+			logger.Log.Debugf("[ERROR] failed sending to API server the tapped pods, response status code %v", response.StatusCode)
+		} else {
+			logger.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos))
+		}
+	}
+}
+
+func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, targetNamespaces []string, cancel context.CancelFunc) {
+	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, targetNamespaces, config.Config.Tap.PodRegex())
 
 	restartTappers := func() {
-		if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex); err != nil {
-			fmt.Printf("Error getting pods by regex: %s (%v,%+v)\n", err, err, err)
-			cancel()
-		} else {
-			currentlyTappedPods = matchingPods
-		}
-
-		nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
+		err, changeFound := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespaces)
 		if err != nil {
-			fmt.Printf("Error building node to ips map: %s (%v,%+v)\n", err, err, err)
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Failed to update currently tapped pods: %v", err))
 			cancel()
 		}
 
-		if err := createMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
-			fmt.Printf("Error updating daemonset: %s (%v,%+v)\n", err, err, err)
+		if !changeFound {
+			logger.Log.Debugf("Nothing changed update tappers not needed")
+			return
+		}
+
+		reportTappedPods()
+
+		nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
+		if err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error building node to ips map: %v", errormessage.FormatError(err)))
+			cancel()
+		}
+		if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating daemonset: %v", errormessage.FormatError(err)))
 			cancel()
 		}
 	}
@@ -171,98 +398,187 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 
 	for {
 		select {
-		case newTarget := <-added:
-			fmt.Printf("+%s\n", newTarget.Name)
-
-		case removedTarget := <-removed:
-			fmt.Printf("-%s\n", removedTarget.Name)
+		case pod := <-added:
+			logger.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
 			restartTappersDebouncer.SetOn()
-
-		case modifiedTarget := <-modified:
+		case pod := <-removed:
+			logger.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
+			restartTappersDebouncer.SetOn()
+		case pod := <-modified:
+			logger.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
 			// Act only if the modified pod has already obtained an IP address.
 			// After filtering for IPs, on a normal pod restart this includes the following events:
 			// - Pod deletion
 			// - Pod reaches start state
 			// - Pod reaches ready state
 			// Ready/unready transitions might also trigger this event.
-			if modifiedTarget.Status.PodIP != "" {
+			if pod.Status.PodIP != "" {
 				restartTappersDebouncer.SetOn()
 			}
 
-		case <-errorChan:
+		case err := <-errorChan:
+			logger.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err)
+			restartTappersDebouncer.Cancel()
 			// TODO: Does this also perform cleanup?
 			cancel()
 
 		case <-ctx.Done():
+			logger.Log.Debugf("Watching pods loop, context done, stopping `restart tappers debouncer`")
+			restartTappersDebouncer.Cancel()
 			return
 		}
 	}
 }
 
-func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
-	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.AggregatorPodName))
-	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, mizu.ResourcesNamespace), podExactRegex)
+func updateCurrentlyTappedPods(kubernetesProvider *kubernetes.Provider, ctx context.Context, targetNamespaces []string) (error, bool) {
+	changeFound := false
+	if matchingPods, err := kubernetesProvider.ListAllRunningPodsMatchingRegex(ctx, config.Config.Tap.PodRegex(), targetNamespaces); err != nil {
+		return err, false
+	} else {
+		podsToTap := excludeMizuPods(matchingPods)
+		addedPods, removedPods := getPodArrayDiff(state.currentlyTappedPods, podsToTap)
+		for _, addedPod := range addedPods {
+			changeFound = true
+			logger.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", addedPod.Name))
+		}
+		for _, removedPod := range removedPods {
+			changeFound = true
+			logger.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedPod.Name))
+		}
+		state.currentlyTappedPods = podsToTap
+	}
+
+	return nil, changeFound
+}
+
+func excludeMizuPods(pods []core.Pod) []core.Pod {
+	mizuPrefixRegex := regexp.MustCompile("^" + mizu.MizuResourcesPrefix)
+
+	nonMizuPods := make([]core.Pod, 0)
+	for _, pod := range pods {
+		if !mizuPrefixRegex.MatchString(pod.Name) {
+			nonMizuPods = append(nonMizuPods, pod)
+		}
+	}
+
+	return nonMizuPods
+}
+
+func getPodArrayDiff(oldPods []core.Pod, newPods []core.Pod) (added []core.Pod, removed []core.Pod) {
+	added = getMissingPods(newPods, oldPods)
+	removed = getMissingPods(oldPods, newPods)
+
+	return added, removed
+}
+
+//returns pods present in pods1 array and missing in pods2 array
+func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod {
+	missingPods := make([]core.Pod, 0)
+	for _, pod1 := range pods1 {
+		var found = false
+		for _, pod2 := range pods2 {
+			if pod1.UID == pod2.UID {
+				found = true
+				break
+			}
+		}
+		if !found {
+			missingPods = append(missingPods, pod1)
+		}
+	}
+	return missingPods
+}
+
+func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName))
+	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
 	isPodReady := false
-	var portForward *kubernetes.PortForward
+	timeAfter := time.After(25 * time.Second)
 	for {
 		select {
+		case <-ctx.Done():
+			logger.Log.Debugf("Watching API Server pod loop, ctx done")
+			return
 		case <-added:
+			logger.Log.Debugf("Watching API Server pod loop, added")
 			continue
 		case <-removed:
-			fmt.Printf("%s removed\n", mizu.AggregatorPodName)
+			logger.Log.Infof("%s removed", mizu.ApiServerPodName)
 			cancel()
 			return
 		case modifiedPod := <-modified:
-			if modifiedPod.Status.Phase == "Running" && !isPodReady {
-				isPodReady = true
-				var err error
-				portForward, err = kubernetes.NewPortForward(kubernetesProvider, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.GuiPort, tappingOptions.MizuPodPort, cancel)
-				fmt.Printf("Web interface is now available at http://localhost:%d\n", tappingOptions.GuiPort)
-				if err != nil {
-					fmt.Printf("error forwarding port to pod %s\n", err)
-					cancel()
-				}
+			if modifiedPod == nil {
+				logger.Log.Debugf("Watching API Server pod loop, modifiedPod with nil")
+				continue
 			}
-
-		case <-time.After(25 * time.Second):
+			logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
+			if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
+				isPodReady = true
+				go startProxyReportErrorIfAny(kubernetesProvider, cancel)
+				logger.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort))
+				time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
+				requestForAnalysis()
+				reportTappedPods()
+			}
+		case <-timeAfter:
 			if !isPodReady {
-				fmt.Printf("error: %s pod was not ready in time", mizu.AggregatorPodName)
+				logger.Log.Errorf(uiUtils.Error, "Mizu API server was not ready in time")
 				cancel()
 			}
-
 		case <-errorChan:
+			logger.Log.Debugf("[ERROR] Agent creation, watching %v namespace", config.Config.MizuResourcesNamespace)
 			cancel()
-
-		case <-ctx.Done():
-			if portForward != nil {
-				portForward.Stop()
-			}
-			return
 		}
 	}
 }
 
-func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
-	mizuRBACExists, err := kubernetesProvider.DoesMizuRBACExist(ctx, mizu.ResourcesNamespace)
+func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+	err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
 	if err != nil {
-		fmt.Printf("warning: could not ensure mizu rbac resources exist %v\n", err)
-		return false
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+
+			"Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName))
+		cancel()
 	}
-	if !mizuRBACExists {
-		var versionString = mizu.Version
-		if mizu.GitCommitHash != "" {
-			versionString += "-" + mizu.GitCommitHash
-		}
-		err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.ResourcesNamespace, versionString)
-		if err != nil {
-			fmt.Printf("warning: could not create mizu rbac resources %v\n", err)
-			return false
-		}
-	}
-	return true
 }
 
-func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) (map[string][]string, error) {
+func requestForAnalysis() {
+	if !config.Config.Tap.Analysis {
+		return
+	}
+
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort)
+	urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(config.Config.Tap.AnalysisDestination), config.Config.Tap.SleepIntervalSec)
+	u, parseErr := url.ParseRequestURI(urlPath)
+	if parseErr != nil {
+		logger.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr)
+	}
+
+	logger.Log.Debugf("Sending get request to %v", u.String())
+	if response, requestErr := http.Get(u.String()); requestErr != nil {
+		logger.Log.Errorf("Failed to notify agent for analysis, err: %v", requestErr)
+	} else if response.StatusCode != 200 {
+		logger.Log.Errorf("Failed to notify agent for analysis, status code: %v", response.StatusCode)
+	} else {
+		logger.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis")
+	}
+}
+
+func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) (bool, error) {
+	if !config.Config.IsNsRestrictedMode() {
+		err := kubernetesProvider.CreateMizuRBAC(ctx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.ClusterRoleName, mizu.ClusterRoleBindingName, mizu.RBACVersion)
+		if err != nil {
+			return false, err
+		}
+	} else {
+		err := kubernetesProvider.CreateMizuRBACNamespaceRestricted(ctx, config.Config.MizuResourcesNamespace, mizu.ServiceAccountName, mizu.RoleName, mizu.RoleBindingName, mizu.RBACVersion)
+		if err != nil {
+			return false, err
+		}
+	}
+	return true, nil
+}
+
+func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) map[string][]string {
 	nodeToTappedPodIPMap := make(map[string][]string, 0)
 	for _, pod := range tappedPods {
 		existingList := nodeToTappedPodIPMap[pod.Spec.NodeName]
@@ -272,7 +588,7 @@ func getNodeHostToTappedPodIpsMap(tappedPods []core.Pod) (map[string][]string, e
 			nodeToTappedPodIPMap[pod.Spec.NodeName] = append(nodeToTappedPodIPMap[pod.Spec.NodeName], pod.Status.PodIP)
 		}
 	}
-	return nodeToTappedPodIPMap, nil
+	return nodeToTappedPodIPMap
 }
 
 func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
@@ -288,34 +604,12 @@ func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
 	}
 }
 
-func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
-	controlSocket, err := mizu.CreateControlSocket(fmt.Sprintf("ws://localhost:%d/ws", tappingOptions.GuiPort))
-	if err != nil {
-		fmt.Printf("error establishing control socket connection %s\n", err)
-		cancel()
-	}
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		default:
-			err = controlSocket.SendNewTappedPodsListMessage(currentlyTappedPods)
-			if err != nil {
-				fmt.Printf("error Sending message via control socket %s\n", err)
-			}
-			time.Sleep(10 * time.Second)
-		}
-	}
-
-}
-
-func getNamespace(tappingOptions *MizuTapOptions, kubernetesProvider *kubernetes.Provider) string {
-	if tappingOptions.AllNamespaces {
-		return mizu.K8sAllNamespaces
-	} else if len(tappingOptions.Namespace) > 0 {
-		return tappingOptions.Namespace
+func getNamespaces(kubernetesProvider *kubernetes.Provider) []string {
+	if config.Config.Tap.AllNamespaces {
+		return []string{mizu.K8sAllNamespaces}
+	} else if len(config.Config.Tap.Namespaces) > 0 {
+		return config.Config.Tap.Namespaces
 	} else {
-		return kubernetesProvider.CurrentNamespace()
+		return []string{kubernetesProvider.CurrentNamespace()}
 	}
 }

cli/cmd/version.go

@@ -1,21 +1,42 @@
 package cmd
 
 import (
-	"fmt"
-	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/telemetry"
+	"strconv"
+	"time"
 
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/mizu"
 )
 
 var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print version info",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Printf("%s (%s) %s\n", mizu.Version, mizu.Branch, mizu.GitCommitHash)
+		go telemetry.ReportRun("version", config.Config.Version)
+
+		if config.Config.Version.DebugInfo {
+			timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
+			logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
+			logger.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
+
+		} else {
+			logger.Log.Infof("Version: %s (%s)", mizu.SemVer, mizu.Branch)
+		}
 		return nil
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(versionCmd)
+
+	defaultVersionConfig := configStructs.VersionConfig{}
+	defaults.Set(&defaultVersionConfig)
+
+	versionCmd.Flags().BoolP(configStructs.DebugInfoVersionName, "d", defaultVersionConfig.DebugInfo, "Provide all information about version")
+
 }

cli/cmd/view.go

@@ -1,13 +1,18 @@
 package cmd
 
 import (
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/telemetry"
 )
 
 var viewCmd = &cobra.Command{
 	Use:   "view",
 	Short: "Open GUI in browser",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("view", config.Config.View)
 		runMizuView()
 		return nil
 	},
@@ -15,4 +20,9 @@ var viewCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(viewCmd)
+
+	defaultViewConfig := configStructs.ViewConfig{}
+	defaults.Set(&defaultViewConfig)
+
+	viewCmd.Flags().Uint16P(configStructs.GuiPortViewName, "p", defaultViewConfig.GuiPort, "Provide a custom port for the web interface webserver")
 }

cli/cmd/viewRunner.go

@@ -3,31 +3,55 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/version"
 	"net/http"
 )
 
 func runMizuView() {
-	kubernetesProvider := kubernetes.NewProvider("")
+	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath)
+	if err != nil {
+		logger.Log.Error(err)
+		return
+	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	exists, err := kubernetesProvider.DoesServicesExist(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName)
+	exists, err := kubernetesProvider.DoesServicesExist(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
 	if err != nil {
-		panic(err)
+		logger.Log.Errorf("Failed to found mizu service %v", err)
+		cancel()
+		return
 	}
 	if !exists {
-		fmt.Printf("The %s service not found\n", mizu.AggregatorPodName)
+		logger.Log.Infof("%s service not found, you should run `mizu tap` command first", mizu.ApiServerPodName)
+		cancel()
 		return
 	}
 
-	_, err = http.Get("http://localhost:8899/")
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.View.GuiPort)
+	_, err = http.Get(fmt.Sprintf("http://%s/", mizuProxiedUrl))
 	if err == nil {
-		fmt.Printf("Found a running service %s and open port 8899\n", mizu.AggregatorPodName)
+		logger.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, config.Config.View.GuiPort)
 		return
 	}
-	fmt.Printf("Found service %s, creating port forwarding to 8899\n", mizu.AggregatorPodName)
-	portForwardApiPod(ctx, kubernetesProvider, cancel, &MizuTapOptions{GuiPort: 8899, MizuPodPort: 8899})
+	logger.Log.Debugf("Found service %s, creating k8s proxy", mizu.ApiServerPodName)
+
+	go startProxyReportErrorIfAny(kubernetesProvider, cancel)
+
+	logger.Log.Infof("Mizu is available at  http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.View.GuiPort))
+	if isCompatible, err := version.CheckVersionCompatibility(config.Config.View.GuiPort); err != nil {
+		logger.Log.Errorf("Failed to check versions compatibility %v", err)
+		cancel()
+		return
+	} else if !isCompatible {
+		cancel()
+		return
+	}
+
+	waitForFinish(ctx, cancel)
 }

cli/config/config.go

@@ -0,0 +1,337 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+	"github.com/up9inc/mizu/cli/mizu"
+	"io/ioutil"
+	"os"
+	"path"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/creasty/defaults"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"gopkg.in/yaml.v3"
+)
+
+const (
+	Separator      = "="
+	SetCommandName = "set"
+	FieldNameTag   = "yaml"
+	ReadonlyTag    = "readonly"
+)
+
+var (
+	Config = ConfigStruct{}
+	cmdName string
+)
+
+func (config *ConfigStruct) Validate() error {
+	if config.IsNsRestrictedMode() {
+		if config.Tap.AllNamespaces || len(config.Tap.Namespaces) != 1 || config.Tap.Namespaces[0] != config.MizuResourcesNamespace {
+			return fmt.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+
+				"You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, MizuResourcesNamespaceConfigName)
+		}
+	}
+
+	return nil
+}
+
+func InitConfig(cmd *cobra.Command) error {
+	cmdName = cmd.Name()
+
+	if err := defaults.Set(&Config); err != nil {
+		return err
+	}
+
+	if err := mergeConfigFile(); err != nil {
+		return fmt.Errorf("invalid config, %w\n" +
+			"you can regenerate the file by removing it (%v) and using `mizu config -r`", err, GetConfigFilePath())
+	}
+
+	cmd.Flags().Visit(initFlag)
+
+	finalConfigPrettified, _ := uiUtils.PrettyJson(Config)
+	logger.Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified)
+
+	return nil
+}
+
+func GetConfigWithDefaults() (string, error) {
+	defaultConf := ConfigStruct{}
+	if err := defaults.Set(&defaultConf); err != nil {
+		return "", err
+	}
+
+	configElem := reflect.ValueOf(&defaultConf).Elem()
+	setZeroForReadonlyFields(configElem)
+
+	return uiUtils.PrettyYaml(defaultConf)
+}
+
+func GetConfigFilePath() string {
+	return path.Join(mizu.GetMizuFolderPath(), "config.yaml")
+}
+
+func mergeConfigFile() error {
+	reader, openErr := os.Open(GetConfigFilePath())
+	if openErr != nil {
+		return nil
+	}
+
+	buf, readErr := ioutil.ReadAll(reader)
+	if readErr != nil {
+		return readErr
+	}
+
+	if err := yaml.Unmarshal(buf, &Config); err != nil {
+		return err
+	}
+	logger.Log.Debugf("Found config file, merged to default options")
+
+	return nil
+}
+
+func initFlag(f *pflag.Flag) {
+	configElemValue := reflect.ValueOf(&Config).Elem()
+
+	flagPath := []string {cmdName, f.Name}
+
+	sliceValue, isSliceValue := f.Value.(pflag.SliceValue)
+	if !isSliceValue {
+		if err := mergeFlagValue(configElemValue, flagPath, strings.Join(flagPath, "."), f.Value.String()); err != nil {
+			logger.Log.Warningf(uiUtils.Warning, err)
+		}
+		return
+	}
+
+	if f.Name == SetCommandName {
+		if err := mergeSetFlag(configElemValue, sliceValue.GetSlice()); err != nil {
+			logger.Log.Warningf(uiUtils.Warning, err)
+		}
+		return
+	}
+
+	if err := mergeFlagValues(configElemValue, flagPath, strings.Join(flagPath, "."), sliceValue.GetSlice()); err != nil {
+		logger.Log.Warningf(uiUtils.Warning, err)
+	}
+}
+
+func mergeSetFlag(configElemValue reflect.Value, setValues []string) error {
+	var setErrors []string
+	setMap := map[string][]string{}
+
+	for _, setValue := range setValues {
+		if !strings.Contains(setValue, Separator) {
+			setErrors = append(setErrors, fmt.Sprintf("Ignoring set argument %s (set argument format: <flag name>=<flag value>)", setValue))
+			continue
+		}
+
+		split := strings.SplitN(setValue, Separator, 2)
+		argumentKey, argumentValue := split[0], split[1]
+
+		setMap[argumentKey] = append(setMap[argumentKey], argumentValue)
+	}
+
+	for argumentKey, argumentValues := range setMap {
+		flagPath := strings.Split(argumentKey, ".")
+
+		if len(argumentValues) > 1 {
+			if err := mergeFlagValues(configElemValue, flagPath, argumentKey, argumentValues); err != nil {
+				setErrors = append(setErrors, fmt.Sprintf("%v", err))
+			}
+		} else {
+			if err := mergeFlagValue(configElemValue, flagPath, argumentKey, argumentValues[0]); err != nil {
+				setErrors = append(setErrors, fmt.Sprintf("%v", err))
+			}
+		}
+	}
+
+	if len(setErrors) > 0 {
+		return fmt.Errorf(strings.Join(setErrors, "\n"))
+	}
+
+	return nil
+}
+
+func mergeFlagValue(configElemValue reflect.Value, flagPath []string, fullFlagName string, flagValue string) error {
+	mergeFunction := func(flagName string, currentFieldStruct reflect.StructField, currentFieldElemValue reflect.Value, currentElemValue reflect.Value) error {
+		currentFieldKind := currentFieldStruct.Type.Kind()
+
+		if currentFieldKind == reflect.Slice {
+			return mergeFlagValues(currentElemValue, []string{flagName}, fullFlagName, []string{flagValue})
+		}
+
+		parsedValue, err := getParsedValue(currentFieldKind, flagValue)
+		if err != nil {
+			return fmt.Errorf("invalid value %s for flag name %s, expected %s", flagValue, flagName, currentFieldKind)
+		}
+
+		currentFieldElemValue.Set(parsedValue)
+		return nil
+	}
+
+	return mergeFlag(configElemValue, flagPath, fullFlagName, mergeFunction)
+}
+
+func mergeFlagValues(configElemValue reflect.Value, flagPath []string, fullFlagName string, flagValues []string) error {
+	mergeFunction := func(flagName string, currentFieldStruct reflect.StructField, currentFieldElemValue reflect.Value, currentElemValue reflect.Value) error {
+		currentFieldKind := currentFieldStruct.Type.Kind()
+
+		if currentFieldKind != reflect.Slice {
+			return fmt.Errorf("invalid values %s for flag name %s, expected %s", strings.Join(flagValues, ","), flagName, currentFieldKind)
+		}
+
+		flagValueKind := currentFieldStruct.Type.Elem().Kind()
+
+		parsedValues := reflect.MakeSlice(reflect.SliceOf(currentFieldStruct.Type.Elem()), 0, 0)
+		for _, flagValue := range flagValues {
+			parsedValue, err := getParsedValue(flagValueKind, flagValue)
+			if err != nil {
+				return fmt.Errorf("invalid value %s for flag name %s, expected %s", flagValue, flagName, flagValueKind)
+			}
+
+			parsedValues = reflect.Append(parsedValues, parsedValue)
+		}
+
+		currentFieldElemValue.Set(parsedValues)
+		return nil
+	}
+
+	return mergeFlag(configElemValue, flagPath, fullFlagName, mergeFunction)
+}
+
+func mergeFlag(currentElemValue reflect.Value, currentFlagPath []string, fullFlagName string, mergeFunction func(flagName string, currentFieldStruct reflect.StructField, currentFieldElemValue reflect.Value, currentElemValue reflect.Value) error) error {
+	if len(currentFlagPath) == 0 {
+		return fmt.Errorf("flag \"%s\" not found", fullFlagName)
+	}
+
+	for i := 0; i < currentElemValue.NumField(); i++ {
+		currentFieldStruct := currentElemValue.Type().Field(i)
+		currentFieldElemValue := currentElemValue.FieldByName(currentFieldStruct.Name)
+
+		if currentFieldStruct.Type.Kind() == reflect.Struct && getFieldNameByTag(currentFieldStruct) == currentFlagPath[0] {
+			return mergeFlag(currentFieldElemValue, currentFlagPath[1:], fullFlagName, mergeFunction)
+		}
+
+		if len(currentFlagPath) > 1 || getFieldNameByTag(currentFieldStruct) != currentFlagPath[0] {
+			continue
+		}
+
+		return mergeFunction(currentFlagPath[0], currentFieldStruct, currentFieldElemValue, currentElemValue)
+	}
+
+	return fmt.Errorf("flag \"%s\" not found", fullFlagName)
+}
+
+func getFieldNameByTag(field reflect.StructField) string {
+	return strings.Split(field.Tag.Get(FieldNameTag), ",")[0]
+}
+
+func getParsedValue(kind reflect.Kind, value string) (reflect.Value, error) {
+	switch kind {
+	case reflect.String:
+		return reflect.ValueOf(value), nil
+	case reflect.Bool:
+		boolArgumentValue, err := strconv.ParseBool(value)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(boolArgumentValue), nil
+	case reflect.Int:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(int(intArgumentValue)), nil
+	case reflect.Int8:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 8)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(int8(intArgumentValue)), nil
+	case reflect.Int16:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 16)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(int16(intArgumentValue)), nil
+	case reflect.Int32:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 32)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(int32(intArgumentValue)), nil
+	case reflect.Int64:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(intArgumentValue), nil
+	case reflect.Uint:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uint(uintArgumentValue)), nil
+	case reflect.Uint8:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 8)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uint8(uintArgumentValue)), nil
+	case reflect.Uint16:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 16)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uint16(uintArgumentValue)), nil
+	case reflect.Uint32:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 32)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uint32(uintArgumentValue)), nil
+	case reflect.Uint64:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uintArgumentValue), nil
+	}
+
+	return reflect.ValueOf(nil), errors.New("value to parse does not match type")
+}
+
+func setZeroForReadonlyFields(currentElem reflect.Value) {
+	for i := 0; i < currentElem.NumField(); i++ {
+		currentField := currentElem.Type().Field(i)
+		currentFieldByName := currentElem.FieldByName(currentField.Name)
+
+		if currentField.Type.Kind() == reflect.Struct {
+			setZeroForReadonlyFields(currentFieldByName)
+			continue
+		}
+
+		if _, ok := currentField.Tag.Lookup(ReadonlyTag); ok {
+			currentFieldByName.Set(reflect.Zero(currentField.Type))
+		}
+	}
+}

cli/config/configStruct.go

@@ -0,0 +1,39 @@
+package config
+
+import (
+	"fmt"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/mizu"
+	v1 "k8s.io/api/core/v1"
+)
+
+const (
+	MizuResourcesNamespaceConfigName = "mizu-resources-namespace"
+)
+
+type ConfigStruct struct {
+	Tap                    configStructs.TapConfig     `yaml:"tap"`
+	Fetch                  configStructs.FetchConfig   `yaml:"fetch"`
+	Version                configStructs.VersionConfig `yaml:"version"`
+	View                   configStructs.ViewConfig    `yaml:"view"`
+	Logs                   configStructs.LogsConfig    `yaml:"logs"`
+	Config                 configStructs.ConfigConfig  `yaml:"config,omitempty"`
+	AgentImage             string                      `yaml:"agent-image,omitempty" readonly:""`
+	ImagePullPolicyStr     string                      `yaml:"image-pull-policy" default:"Always"`
+	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
+	Telemetry              bool                        `yaml:"telemetry" default:"true"`
+	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`
+	KubeConfigPath         string                      `yaml:"kube-config-path"`
+}
+
+func (config *ConfigStruct) SetDefaults() {
+	config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer)
+}
+
+func (config *ConfigStruct) ImagePullPolicy() v1.PullPolicy {
+	return v1.PullPolicy(config.ImagePullPolicyStr)
+}
+
+func (config *ConfigStruct) IsNsRestrictedMode() bool {
+	return config.MizuResourcesNamespace != "mizu" // Notice "mizu" string must match the default MizuResourcesNamespace
+}

cli/config/configStructs/configConfig.go

@@ -0,0 +1,9 @@
+package configStructs
+
+const (
+	RegenerateConfigName = "regenerate"
+)
+
+type ConfigConfig struct {
+	Regenerate bool `yaml:"regenerate,omitempty" default:"false" readonly:""`
+}

cli/config/configStructs/fetchConfig.go

@@ -0,0 +1,15 @@
+package configStructs
+
+const (
+	DirectoryFetchName     = "directory"
+	FromTimestampFetchName = "from"
+	ToTimestampFetchName   = "to"
+	GuiPortFetchName       = "gui-port"
+)
+
+type FetchConfig struct {
+	Directory     string `yaml:"directory" default:"."`
+	FromTimestamp int    `yaml:"from" default:"0"`
+	ToTimestamp   int    `yaml:"to" default:"0"`
+	GuiPort       uint16 `yaml:"gui-port" default:"8899"`
+}

cli/config/configStructs/logsConfig.go

@@ -0,0 +1,35 @@
+package configStructs
+
+import (
+	"fmt"
+	"os"
+	"path"
+)
+
+const (
+	FileLogsName = "file"
+)
+
+type LogsConfig struct {
+	FileStr string `yaml:"file"`
+}
+
+func (config *LogsConfig) Validate() error {
+	if config.FileStr == "" {
+		_, err := os.Getwd()
+		if err != nil {
+			return fmt.Errorf("failed to get PWD, %v (try using `mizu logs -f <full path dest zip file>)`", err)
+		}
+	}
+
+	return nil
+}
+
+func (config *LogsConfig) FilePath() string {
+	if config.FileStr == "" {
+		pwd, _ := os.Getwd()
+		return path.Join(pwd, "mizu_logs.zip")
+	}
+
+	return config.FileStr
+}

cli/config/configStructs/tapConfig.go

@@ -0,0 +1,87 @@
+package configStructs
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/up9inc/mizu/shared/units"
+)
+
+const (
+	GuiPortTapName                = "gui-port"
+	NamespacesTapName             = "namespaces"
+	AnalysisTapName               = "analysis"
+	AllNamespacesTapName          = "all-namespaces"
+	PlainTextFilterRegexesTapName = "regex-masking"
+	DisableRedactionTapName       = "no-redact"
+	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
+	DirectionTapName              = "direction"
+	DryRunTapName                 = "dry-run"
+	EnforcePolicyFile             = "test-rules"
+)
+
+type TapConfig struct {
+	AnalysisDestination          string    `yaml:"dest" default:"up9.app"`
+	SleepIntervalSec             int       `yaml:"upload-interval" default:"10"`
+	PodRegexStr                  string    `yaml:"regex" default:".*"`
+	GuiPort                      uint16    `yaml:"gui-port" default:"8899"`
+	Namespaces                   []string  `yaml:"namespaces"`
+	Analysis                     bool      `yaml:"analysis" default:"false"`
+	AllNamespaces                bool      `yaml:"all-namespaces" default:"false"`
+	PlainTextFilterRegexes       []string  `yaml:"regex-masking"`
+	HealthChecksUserAgentHeaders []string  `yaml:"ignored-user-agents"`
+	DisableRedaction             bool      `yaml:"no-redact" default:"false"`
+	HumanMaxEntriesDBSize        string    `yaml:"max-entries-db-size" default:"200MB"`
+	Direction                    string    `yaml:"direction" default:"in"`
+	DryRun                       bool      `yaml:"dry-run" default:"false"`
+	EnforcePolicyFile            string    `yaml:"test-rules"`
+	ApiServerResources           Resources `yaml:"api-server-resources"`
+	TapperResources              Resources `yaml:"tapper-resources"`
+}
+
+type Resources struct {
+	CpuLimit       string `yaml:"cpu-limit" default:"750m"`
+	MemoryLimit    string `yaml:"memory-limit" default:"1Gi"`
+	CpuRequests    string `yaml:"cpu-requests" default:"50m"`
+	MemoryRequests string `yaml:"memory-requests" default:"50Mi"`
+}
+
+func (config *TapConfig) PodRegex() *regexp.Regexp {
+	podRegex, _ := regexp.Compile(config.PodRegexStr)
+	return podRegex
+}
+
+func (config *TapConfig) TapOutgoing() bool {
+	directionLowerCase := strings.ToLower(config.Direction)
+	if directionLowerCase == "any" {
+		return true
+	}
+
+	return false
+}
+
+func (config *TapConfig) MaxEntriesDBSizeBytes() int64 {
+	maxEntriesDBSizeBytes, _ := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
+	return maxEntriesDBSizeBytes
+}
+
+func (config *TapConfig) Validate() error {
+	_, compileErr := regexp.Compile(config.PodRegexStr)
+	if compileErr != nil {
+		return errors.New(fmt.Sprintf("%s is not a valid regex %s", config.PodRegexStr, compileErr))
+	}
+
+	_, parseHumanDataSizeErr := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
+	if parseHumanDataSizeErr != nil {
+		return errors.New(fmt.Sprintf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize))
+	}
+
+	directionLowerCase := strings.ToLower(config.Direction)
+	if directionLowerCase != "any" && directionLowerCase != "in" {
+		return errors.New(fmt.Sprintf("%s is not a valid value for flag --%s. Acceptable values are in/any.", config.Direction, DirectionTapName))
+	}
+
+	return nil
+}