Tapper stats in stats tracker (#166)

* Config grooming and several general fixes

.github/workflows/publish-docker.yml

@@ -1,39 +0,0 @@
-name: publish-docker
-on:
-  push:
-    branches:
-      - 'develop'
-      - 'main'
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get base image name
-        shell: bash
-        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
-        id: base_image_step
-      - name: Docker meta
-        id: meta
-        uses: crazy-max/ghaction-docker-meta@v2
-        with:
-          images: ${{ steps.base_image_step.outputs.image }}
-          tags: |
-            type=sha
-            type=raw,${{ github.sha }}
-            type=raw,latest
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          registry: gcr.io
-          username: _json_key
-          password: ${{ secrets.GCR_JSON_KEY }}
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-

.github/workflows/publish.yml

@@ -1,9 +1,9 @@
-name: public-cli
+name: publish
 on:
   push:
     branches:
-      - develop
-      - main
+      - 'develop'
+      - 'main'
 jobs:
   docker:
     runs-on: ubuntu-latest
@@ -27,14 +27,48 @@ jobs:
         with:
           releaseType: ${{ steps.condval.outputs.value }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Get base image name
+      - name: Get version parameters
         shell: bash
         run: |
           echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
           echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
         id: version_parameters
+      - name: Get base image name
+        shell: bash
+        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
+        id: base_image_step
+      - name: Docker meta
+        id: meta
+        uses: crazy-max/ghaction-docker-meta@v2
+        with:
+          images: ${{ steps.base_image_step.outputs.image }}
+          tags: |
+            type=sha
+            type=raw,${{ github.sha }}
+            type=raw,${{ steps.versioning.outputs.version }}
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          registry: gcr.io
+          username: _json_key
+          password: ${{ secrets.GCR_JSON_KEY }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            SEM_VER=${{ steps.versioning.outputs.version }}
+            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
+            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
+            COMMIT_HASH=${{ github.sha }}
       - name: Build and Push CLI
         run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
+      - shell: bash
+        run: |
+          echo '${{ steps.versioning.outputs.version }}' >> cli/bin/version.txt
       - name: publish
         uses: ncipollo/release-action@v1
         with:
@@ -44,3 +78,4 @@ jobs:
           tag: ${{ steps.versioning.outputs.version }}
           prerelease: ${{ github.ref != 'refs/heads/main' }}
           bodyFile: 'cli/bin/README.md'
+

Dockerfile

@@ -13,21 +13,30 @@ ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 
 RUN apk add libpcap-dev gcc g++ make
 
-# Move to api working directory (/api-build).
-WORKDIR /app/api-build
+# Move to agent working directory (/agent-build).
+WORKDIR /app/agent-build
 
-COPY api/go.mod api/go.sum ./
+COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
 COPY tap/go.mod tap/go.mod ../tap/
 RUN go mod download
 # cheap trick to make the build faster (As long as go.mod wasn't changes)
 RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
 
-# Copy and build api code
+ARG COMMIT_HASH
+ARG GIT_BRANCH
+ARG BUILD_TIMESTAMP
+ARG SEM_VER
+
+# Copy and build agent code
 COPY shared ../shared
 COPY tap ../tap
-COPY api .
-RUN go build -ldflags="-s -w" -o mizuagent .
+COPY agent .
+RUN go build -ldflags="-s -w \
+     -X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
+     -X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
+     -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
+     -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
 
 FROM alpine:3.13.5
@@ -36,10 +45,13 @@ RUN apk add bash libpcap-dev tcpdump
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
-COPY --from=builder ["/app/api-build/mizuagent", "."]
+COPY --from=builder ["/app/agent-build/mizuagent", "."]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 
-COPY api/start.sh .
+COPY agent/start.sh .
+
+# gin-gonic runs in debug mode without this
+ENV GIN_MODE=release
 
 # this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
 ENTRYPOINT "/app/mizuagent"

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui api cli tap docker
+.PHONY: help ui agent cli tap docker
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -19,34 +19,35 @@ help: ## This help.
 TS_SUFFIX="$(shell date '+%s')"
 GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
 BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
+export SEM_VER?=0.0.0
 
-ui: ## build UI
+ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
 	@ls -l ui/build  
 
-cli: # build CLI
+cli: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build
 
-api: ## build API server
-	@(echo "building API server .." )
-	@(cd api; go build -o build/apiserver main.go)
-	@ls -l api/build
+agent: ## Build agent.
+	@(echo "building mizu agent .." )
+	@(cd agent; go build -o build/mizuagent main.go)
+	@ls -l agent/build
 
 #tap: ## build tap binary
 #	@(cd tap; go build -o build/tap ./src)
 #	@ls -l tap/build
 
-docker: ## build Docker image 
+docker: ## Build Docker image.
 	@(echo "building docker image" )
 	./build-push-featurebranch.sh
 
-push: push-docker push-cli ## build and publish Mizu docker image & CLI
+push: push-docker push-cli ## Build and publish agent docker image & CLI.
 
-push-docker: 
+push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
 	./build-push-featurebranch.sh
 
-push-cli:
+push-cli: ## Build and publish CLI.
 	@echo "publishing CLI .. "
 	@cd cli; $(MAKE) build-all
 	@echo "publishing file ${OUTPUT_FILE} .."
@@ -55,17 +56,17 @@ push-cli:
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
 
-clean: clean-ui clean-api clean-cli clean-docker ## Clean all build artifacts
+clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
 
-clean-ui: 
+clean-ui: ## Clean UI.
 	@(rm -rf ui/build ; echo "UI cleanup done" )
 
-clean-api: 
-	@(rm -rf api/build ; echo "api cleanup done" )
+clean-agent: ## Clean agent.
+	@(rm -rf agent/build ; echo "agent cleanup done" )
 
-clean-cli: 
+clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
-clean-docker: 
+clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 

README.md

@@ -1,6 +1,17 @@
-# 水 mizu
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
+# The API Traffic Viewer for Kubernetes
+
 A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined.
 
+![Simple UI](assets/mizu-ui.png)
+
+## Features
+
+- Simple and powerful CLI
+- Real time view of all HTTP requests, REST and gRPC API calls
+- No installation or code instrumentation
+- Works completely on premises (on-prem)
+
 ## Download
 
 Download `mizu` for your platform and operating system
@@ -26,12 +37,126 @@ SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/
 ### Development (unstable) build
 Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page.
 
+## Prerequisites
+1. Set `KUBECONFIG` environment variable to your kubernetes configuration. If this is not set, mizu assumes that configuration is at `${HOME}/.kube/config`
+2. mizu needs following permissions on your kubernetes cluster to run
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - watch
+  - create
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services/proxy
+  verbs:
+  - get
+```
+3. Optionally, for resolving traffic ip to kubernetes service name, mizu needs below permissions
+
+```yaml
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  - apps
+  - extensions
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
+  - create
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterroles
+  verbs:
+  - list
+  - create
+  - delete
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterrolebindings
+  verbs:
+  - list
+  - create
+  - delete
+```
+
+See `examples/roles` for example `clusterroles`. 
+
 ## How to run
 
-1. Find pod you'd like to tap to in your Kubernetes cluster
+1. Find pods you'd like to tap to in your Kubernetes cluster
 2. Run `mizu tap PODNAME` or `mizu tap REGEX` 
-3. Open browser on `http://localhost:8899` as instructed .. 
-4. Watch the WebAPI traffic flowing ..
+3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI .. 
+4. Watch the API traffic flowing ..
 5. Type ^C to stop
 
 ## Examples

agent/README.md

@@ -1,5 +1,5 @@
-# mizu API server
-API server for MIZU
+# mizu agent
+Agent for MIZU (API server and tapper)
 Basic APIs:
 * /fetch - retrieve traffic data
 * /stats - retrieve statistics of collected data
@@ -14,7 +14,7 @@ Basic APIs:
 
 ### Connecting
 1. Start mizu using the cli with the debug image `mizu tap --mizu-image gcr.io/up9-docker-hub/mizu/debug:latest {tapped_pod_name}`
-2. Forward the debug port using `kubectl port-forward -n default mizu-collector 2345:2345`
+2. Forward the debug port using `kubectl port-forward -n default mizu-api-server 2345:2345`
 3. Run the run/debug configuration you've created earlier in Intellij.
 
 <small>Do note that dlv won't start the api until a debugger connects to it.</small>

agent/go.mod

@@ -3,17 +3,16 @@ module mizuserver
 go 1.16
 
 require (
-	github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a
 	github.com/beevik/etree v1.1.0
 	github.com/djherbis/atime v1.0.0
-	github.com/fasthttp/websocket v1.4.3-beta.1 // indirect
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/gin-contrib/static v0.0.1
+	github.com/gin-gonic/gin v1.7.2
 	github.com/go-playground/locales v0.13.0
 	github.com/go-playground/universal-translator v0.17.0
 	github.com/go-playground/validator/v10 v10.5.0
-	github.com/gofiber/fiber/v2 v2.8.0
 	github.com/google/martian v2.1.0+incompatible
 	github.com/gorilla/websocket v1.4.2
-	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
@@ -23,7 +22,7 @@ require (
 	k8s.io/api v0.21.0
 	k8s.io/apimachinery v0.21.0
 	k8s.io/client-go v0.21.0
-	github.com/fsnotify/fsnotify v1.4.9
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 )
 
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared

agent/go.sum

@@ -41,15 +41,12 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
-github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc=
-github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
-github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a h1:76llBleIE3fkdqaJFDzdirtiYhQPdIQem8H8r2iwA1Q=
-github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a/go.mod h1:QvDfsDQDmGxUsvEeWabVZ5pp2FMXpOkwQV0L6SE6cp0=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
 github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
 github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
+github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M=
+github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -69,14 +66,18 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y=
-github.com/fasthttp/websocket v1.4.3-beta.1 h1:stc4P2aoxYKsdmbe1AJ5mAm73Fxc1NOgrZpPftvZIXQ=
-github.com/fasthttp/websocket v1.4.3-beta.1/go.mod h1:JGrgLaT02bL9NuJkZbHN8mVV2tkCJZQh7yJ5/XCXO2g=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-contrib/static v0.0.1 h1:JVxuvHPuUfkoul12N7dtQw7KRn/pSMq7Ue1Va9Swm1U=
+github.com/gin-contrib/static v0.0.1/go.mod h1:CSxeF+wep05e0kCOsqWdAWbSszmc31zTIbD8TvWl7Hs=
+github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/gin-gonic/gin v1.7.2 h1:Tg03T9yM2xa8j6I3Z3oqLaQRSmKvxPd6g/2HJ6zICFA=
+github.com/gin-gonic/gin v1.7.2/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -96,6 +97,8 @@ github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8c
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E=
 github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
@@ -124,12 +127,6 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe
 github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
 github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
-github.com/gofiber/fiber/v2 v2.1.3/go.mod h1:MMiSv1HrDkN8Pv7NeVDYK+T/lwXOEKAvPBbLvJPCEfA=
-github.com/gofiber/fiber/v2 v2.7.1/go.mod h1:f8BRRIMjMdRyt2qmJ/0Sea3j3rwwfufPrh9WNBRiVZ0=
-github.com/gofiber/fiber/v2 v2.8.0 h1:BdWvZmg/WY/Vjtjm38aXOp1Lks1BhuyS2b7lSWSPAzk=
-github.com/gofiber/fiber/v2 v2.8.0/go.mod h1:Ah3IJikrKNRepl/HuVawppS25X7FWohwfCSRn7kJG28=
-github.com/gofiber/websocket/v2 v2.0.3 h1:nqPGHB4LQhxKX5KJUjayOd2xiiENieS/dn6TPfCL8uk=
-github.com/gofiber/websocket/v2 v2.0.3/go.mod h1:/OTEImCxORKE5unw0dWqJYovid6vZF+wB1W0aaMKs2M=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -190,7 +187,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
@@ -202,6 +198,7 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -210,13 +207,7 @@ github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaR
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4=
-github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -226,13 +217,14 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
-github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
-github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ=
 github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -255,6 +247,7 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@@ -269,9 +262,6 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
 github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
-github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY=
-github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c h1:KKqhycXW1WVNkX7r4ekTV2gFkbhdyihlWD8c0/FiWmk=
-github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -283,25 +273,17 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
-github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
-github.com/valyala/fasthttp v1.15.1/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
-github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
-github.com/valyala/fasthttp v1.18.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A=
-github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5NcFs=
-github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU=
-github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc=
-github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
+github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
+github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
@@ -377,11 +359,8 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758 h1:aEpZnXcAmXkd6AvLb2OPt+EN1Zu/8Ne3pCqPjja5PXY=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -420,17 +399,15 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ=
 golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

agent/main.go

@@ -4,34 +4,34 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"github.com/gofiber/fiber/v2"
-	"github.com/gofiber/fiber/v2/middleware/cors"
+	"github.com/gin-contrib/static"
+	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	"github.com/romana/rlog"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
 	"mizuserver/pkg/api"
-	"mizuserver/pkg/middleware"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/routes"
 	"mizuserver/pkg/sensitiveDataFiltering"
 	"mizuserver/pkg/utils"
+	"net/http"
 	"os"
 	"os/signal"
 	"strings"
 )
 
 var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API")
-var aggregator = flag.Bool("aggregator", false, "Run in aggregator mode with API")
+var apiServer = flag.Bool("api-server", false, "Run in API server mode with API")
 var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode")
-var aggregatorAddress = flag.String("aggregator-address", "", "Address of mizu collector for tapping")
+var apiServerAddress = flag.String("api-server-address", "", "Address of mizu API server")
 
 func main() {
 	flag.Parse()
 	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
 	tapOpts := &tap.TapOpts{HostMode: hostMode}
 
-	if !*shouldTap && !*aggregator && !*standalone {
+	if !*shouldTap && !*apiServer && !*standalone {
 		panic("One of the flags --tap, --api or --standalone must be provided")
 	}
 
@@ -45,8 +45,8 @@ func main() {
 
 		hostApi(nil)
 	} else if *shouldTap {
-		if *aggregatorAddress == "" {
-			panic("Aggregator address must be provided with --aggregator-address when using --tap")
+		if *apiServerAddress == "" {
+			panic("API server address must be provided with --api-server-address when using --tap")
 		}
 
 		tapTargets := getTapTargets()
@@ -57,14 +57,14 @@ func main() {
 
 		harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts)
 
-		socketConnection, err := shared.ConnectToSocketServer(*aggregatorAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false)
+		socketConnection, err := shared.ConnectToSocketServer(*apiServerAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false)
 		if err != nil {
-			panic(fmt.Sprintf("Error connecting to socket server at %s %v", *aggregatorAddress, err))
+			panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
 		}
 
-		go pipeChannelToSocket(socketConnection, harOutputChannel)
-		go api.StartReadingOutbound(outboundLinkOutputChannel)
-	} else if *aggregator {
+		go pipeTapChannelToSocket(socketConnection, harOutputChannel)
+		go pipeOutboundLinksChannelToSocket(socketConnection, outboundLinkOutputChannel)
+	} else if *apiServer {
 		socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000)
 		filteredHarChannel := make(chan *tap.OutputChannelItem)
 
@@ -82,30 +82,44 @@ func main() {
 }
 
 func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
-	app := fiber.New()
+	app := gin.Default()
 
-	app.Use(cors.New(cors.Config{
-		AllowOrigins: "*",
-		AllowMethods: "*",
-		AllowHeaders: "*",
-	}))
-	middleware.FiberMiddleware(app) // Register Fiber's middleware for app.
-	app.Static("/", "./site")
-
-	//Simple route to know server is running
-	app.Get("/echo", func(c *fiber.Ctx) error {
-		return c.SendString("Hello, World 👋!")
+	app.GET("/echo", func(c *gin.Context) {
+		c.String(http.StatusOK, "Here is Mizu agent")
 	})
+
 	eventHandlers := api.RoutesEventHandlers{
 		SocketHarOutChannel: socketHarOutputChannel,
 	}
-	routes.WebSocketRoutes(app, &eventHandlers)
+
+	app.Use(static.ServeRoot("/", "./site"))
+	app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
+
+	api.WebSocketRoutes(app, &eventHandlers)
 	routes.EntriesRoutes(app)
+	routes.MetadataRoutes(app)
+	routes.StatusRoutes(app)
 	routes.NotFoundRoute(app)
 
 	utils.StartServer(app)
 }
 
+func CORSMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
+		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
+		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
+		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")
+
+		if c.Request.Method == "OPTIONS" {
+			c.AbortWithStatus(204)
+			return
+		}
+
+		c.Next()
+	}
+}
+
 func getTapTargets() []string {
 	nodeName := os.Getenv(shared.NodeNameEnvVar)
 	var tappedAddressesPerNodeDict map[string][]string
@@ -142,7 +156,9 @@ func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *ta
 			continue
 		}
 
-		sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions)
+		if !filterOptions.DisableRedaction {
+			sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions)
+		}
 
 		outChannel <- message
 	}
@@ -162,7 +178,7 @@ func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool {
 	return false
 }
 
-func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) {
+func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) {
 	if connection == nil {
 		panic("Websocket connection is nil")
 	}
@@ -185,3 +201,21 @@ func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *
 		}
 	}
 }
+
+func pipeOutboundLinksChannelToSocket(connection *websocket.Conn, outboundLinkChannel <-chan *tap.OutboundLink) {
+	for outboundLink := range outboundLinkChannel {
+		if outboundLink.SuggestedProtocol == tap.TLSProtocol {
+			marshaledData, err := models.CreateWebsocketOutboundLinkMessage(outboundLink)
+			if err != nil {
+				rlog.Infof("Error converting outbound link to json %s, (%v,%+v)", err, err, err)
+				continue
+			}
+
+			err = connection.WriteMessage(websocket.TextMessage, marshaledData)
+			if err != nil {
+				rlog.Infof("error sending outbound link message through socket server %s, (%v,%+v)", err, err, err)
+				continue
+			}
+		}
+	}
+}

agent/pkg/api/main.go

@@ -88,9 +88,9 @@ func startReadingFiles(workingDir string) {
 		for _, entry := range inputHar.Log.Entries {
 			time.Sleep(time.Millisecond * 250)
 			connectionInfo := &tap.ConnectionInfo{
-				ClientIP: fileInfo.Name(),
+				ClientIP:   fileInfo.Name(),
 				ClientPort: "",
-				ServerIP: "",
+				ServerIP:   "",
 				ServerPort: "",
 				IsOutgoing: false,
 			}
@@ -118,7 +118,6 @@ func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) {
 	}
 }
 
-
 func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
 	entryBytes, _ := json.Marshal(entry)
 	serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL)
@@ -161,14 +160,14 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
 		IsOutgoing:          connectionInfo.IsOutgoing,
 	}
 	mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
-	database.GetEntriesTable().Create(&mizuEntry)
+	database.CreateEntry(&mizuEntry)
 
 	baseEntry := models.BaseEntryDetails{}
 	if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil {
 		return
 	}
 	baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry)
-	broadcastToBrowserClients(baseEntryBytes)
+	BroadcastToBrowserClients(baseEntryBytes)
 }
 
 func getServiceNameFromUrl(inputUrl string) (string, string) {
@@ -196,6 +195,5 @@ func getEstimatedEntrySizeBytes(mizuEntry models.MizuEntry) int {
 	sizeBytes += 8 // SizeBytes bytes
 	sizeBytes += 1 // IsOutgoing bytes
 
-
 	return sizeBytes
 }

agent/pkg/api/socket_routes.go

@@ -0,0 +1,118 @@
+package api
+
+import (
+	"errors"
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/gorilla/websocket"
+	"github.com/up9inc/mizu/shared/debounce"
+	"net/http"
+	"sync"
+	"time"
+)
+
+type EventHandlers interface {
+	WebSocketConnect(socketId int, isTapper bool)
+	WebSocketDisconnect(socketId int, isTapper bool)
+	WebSocketMessage(socketId int, message []byte)
+}
+
+type SocketConnection struct {
+	connection    *websocket.Conn
+	lock          *sync.Mutex
+	eventHandlers EventHandlers
+	isTapper      bool
+}
+
+var websocketUpgrader = websocket.Upgrader{
+	ReadBufferSize:  1024,
+	WriteBufferSize: 1024,
+}
+
+var websocketIdsLock = sync.Mutex{}
+var connectedWebsockets map[int]*SocketConnection
+var connectedWebsocketIdCounter = 0
+
+func init() {
+	websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
+	connectedWebsockets = make(map[int]*SocketConnection, 0)
+}
+
+func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
+	app.GET("/ws", func(c *gin.Context) {
+		websocketHandler(c.Writer, c.Request, eventHandlers, false)
+	})
+	app.GET("/wsTapper", func(c *gin.Context) {
+		websocketHandler(c.Writer, c.Request, eventHandlers, true)
+	})
+}
+
+func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) {
+	conn, err := websocketUpgrader.Upgrade(w, r, nil)
+	if err != nil {
+		fmt.Println("Failed to set websocket upgrade: %+v", err)
+		return
+	}
+
+	websocketIdsLock.Lock()
+
+	connectedWebsocketIdCounter++
+	socketId := connectedWebsocketIdCounter
+	connectedWebsockets[socketId] = &SocketConnection{connection: conn, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
+
+	websocketIdsLock.Unlock()
+
+	defer func() {
+		socketCleanup(socketId, connectedWebsockets[socketId])
+	}()
+
+	eventHandlers.WebSocketConnect(socketId, isTapper)
+
+	for {
+		_, msg, err := conn.ReadMessage()
+		if err != nil {
+			fmt.Printf("Conn err: %v\n", err)
+			break
+		}
+		eventHandlers.WebSocketMessage(socketId, msg)
+	}
+}
+
+func socketCleanup(socketId int, socketConnection *SocketConnection) {
+	err := socketConnection.connection.Close()
+	if err != nil {
+		fmt.Printf("Error closing socket connection for socket id %d: %v\n", socketId, err)
+	}
+
+	websocketIdsLock.Lock()
+	connectedWebsockets[socketId] = nil
+	websocketIdsLock.Unlock()
+
+	socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
+}
+
+var db = debounce.NewDebouncer(time.Second*5, func() {
+	fmt.Println("Successfully sent to socket")
+})
+
+func SendToSocket(socketId int, message []byte) error {
+	socketObj := connectedWebsockets[socketId]
+	if socketObj == nil {
+		return errors.New("Socket is disconnected")
+	}
+
+	var sent = false
+	time.AfterFunc(time.Second*5, func() {
+		if !sent {
+			fmt.Println("Socket timed out")
+			socketCleanup(socketId, socketObj)
+		}
+	})
+
+	socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
+	err := socketObj.connection.WriteMessage(1, message)
+	socketObj.lock.Unlock()
+
+	sent = true
+	return err
+}

agent/pkg/api/socket_server_handlers.go

@@ -0,0 +1,129 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/up9"
+	"sync"
+)
+
+var browserClientSocketUUIDs = make([]int, 0)
+var socketListLock = sync.Mutex{}
+
+type RoutesEventHandlers struct {
+	EventHandlers
+	SocketHarOutChannel chan<- *tap.OutputChannelItem
+}
+
+func init() {
+	go up9.UpdateAnalyzeStatus(BroadcastToBrowserClients)
+}
+
+func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
+	if isTapper {
+		rlog.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
+	} else {
+		rlog.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
+		socketListLock.Lock()
+		browserClientSocketUUIDs = append(browserClientSocketUUIDs, socketId)
+		socketListLock.Unlock()
+	}
+}
+
+func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
+	if isTapper {
+		rlog.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
+	} else {
+		rlog.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
+		socketListLock.Lock()
+		removeSocketUUIDFromBrowserSlice(socketId)
+		socketListLock.Unlock()
+	}
+}
+
+func BroadcastToBrowserClients(message []byte) {
+	for _, socketId := range browserClientSocketUUIDs {
+		go func(socketId int) {
+			err := SendToSocket(socketId, message)
+			if err != nil {
+				fmt.Printf("error sending message to socket ID %d: %v", socketId, err)
+			}
+		}(socketId)
+	}
+}
+
+func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
+	var socketMessageBase shared.WebSocketMessageMetadata
+	err := json.Unmarshal(message, &socketMessageBase)
+	if err != nil {
+		rlog.Infof("Could not unmarshal websocket message %v\n", err)
+	} else {
+		switch socketMessageBase.MessageType {
+		case shared.WebSocketMessageTypeTappedEntry:
+			var tappedEntryMessage models.WebSocketTappedEntryMessage
+			err := json.Unmarshal(message, &tappedEntryMessage)
+			if err != nil {
+				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+			} else {
+				h.SocketHarOutChannel <- tappedEntryMessage.Data
+			}
+		case shared.WebSocketMessageTypeUpdateStatus:
+			var statusMessage shared.WebSocketStatusMessage
+			err := json.Unmarshal(message, &statusMessage)
+			if err != nil {
+				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+			} else {
+				providers.TapStatus.Pods = statusMessage.TappingStatus.Pods
+				BroadcastToBrowserClients(message)
+			}
+		case shared.WebsocketMessageTypeOutboundLink:
+			var outboundLinkMessage models.WebsocketOutboundLinkMessage
+			err := json.Unmarshal(message, &outboundLinkMessage)
+			if err != nil {
+				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+			} else {
+				handleTLSLink(outboundLinkMessage)
+			}
+		default:
+			rlog.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
+		}
+	}
+}
+
+func handleTLSLink(outboundLinkMessage models.WebsocketOutboundLinkMessage) {
+	resolvedName := k8sResolver.Resolve(outboundLinkMessage.Data.DstIP)
+	if resolvedName != "" {
+		outboundLinkMessage.Data.DstIP = resolvedName
+	} else if outboundLinkMessage.Data.SuggestedResolvedName != "" {
+		outboundLinkMessage.Data.DstIP = outboundLinkMessage.Data.SuggestedResolvedName
+	}
+	cacheKey := fmt.Sprintf("%s -> %s:%d", outboundLinkMessage.Data.Src, outboundLinkMessage.Data.DstIP, outboundLinkMessage.Data.DstPort)
+	_, isInCache := providers.RecentTLSLinks.Get(cacheKey)
+	if isInCache {
+		return
+	} else {
+		providers.RecentTLSLinks.SetDefault(cacheKey, outboundLinkMessage.Data)
+	}
+	marshaledMessage, err := json.Marshal(outboundLinkMessage)
+	if err != nil {
+		rlog.Errorf("Error marshaling outbound link message for broadcasting: %v", err)
+	} else {
+		fmt.Printf("Broadcasting outboundlink message %s\n", string(marshaledMessage))
+		BroadcastToBrowserClients(marshaledMessage)
+	}
+}
+
+func removeSocketUUIDFromBrowserSlice(uuidToRemove int) {
+	newUUIDSlice := make([]int, 0, len(browserClientSocketUUIDs))
+	for _, uuid := range browserClientSocketUUIDs {
+		if uuid != uuidToRemove {
+			newUUIDSlice = append(newUUIDSlice, uuid)
+		}
+	}
+	browserClientSocketUUIDs = newUUIDSlice
+}

agent/pkg/controllers/entries_controller.go

@@ -3,27 +3,29 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/gofiber/fiber/v2"
+	"github.com/gin-gonic/gin"
 	"github.com/google/martian/har"
 	"github.com/romana/rlog"
 	"mizuserver/pkg/database"
 	"mizuserver/pkg/models"
+	"mizuserver/pkg/providers"
 	"mizuserver/pkg/up9"
 	"mizuserver/pkg/utils"
 	"mizuserver/pkg/validation"
+	"net/http"
 	"strings"
 	"time"
 )
 
-func GetEntries(c *fiber.Ctx) error {
+func GetEntries(c *gin.Context) {
 	entriesFilter := &models.EntriesFilter{}
 
-	if err := c.QueryParser(entriesFilter); err != nil {
-		return c.Status(fiber.StatusBadRequest).JSON(err)
+	if err := c.BindQuery(entriesFilter); err != nil {
+		c.JSON(http.StatusBadRequest, err)
 	}
 	err := validation.Validate(entriesFilter)
 	if err != nil {
-		return c.Status(fiber.StatusBadRequest).JSON(err)
+		c.JSON(http.StatusBadRequest, err)
 	}
 
 	order := database.OperatorToOrderMapping[entriesFilter.Operator]
@@ -50,18 +52,18 @@ func GetEntries(c *fiber.Ctx) error {
 		baseEntries = append(baseEntries, harEntry)
 	}
 
-	return c.Status(fiber.StatusOK).JSON(baseEntries)
+	c.JSON(http.StatusOK, baseEntries)
 }
 
-func GetHARs(c *fiber.Ctx) error {
+func GetHARs(c *gin.Context) {
 	entriesFilter := &models.HarFetchRequestBody{}
 	order := database.OrderDesc
-	if err := c.QueryParser(entriesFilter); err != nil {
-		return c.Status(fiber.StatusBadRequest).JSON(err)
+	if err := c.BindQuery(entriesFilter); err != nil {
+		c.JSON(http.StatusBadRequest, err)
 	}
 	err := validation.Validate(entriesFilter)
 	if err != nil {
-		return c.Status(fiber.StatusBadRequest).JSON(err)
+		c.JSON(http.StatusBadRequest, err)
 	}
 
 	var timestampFrom, timestampTo int64
@@ -137,40 +139,45 @@ func GetHARs(c *fiber.Ctx) error {
 		retObj[k] = bytesData
 	}
 	buffer := utils.ZipData(retObj)
-	return c.Status(fiber.StatusOK).SendStream(buffer)
+	c.Data(http.StatusOK, "application/octet-stream", buffer.Bytes())
 }
 
-func UploadEntries(c *fiber.Ctx) error {
+func UploadEntries(c *gin.Context) {
 	rlog.Infof("Upload entries - started\n")
 
 	uploadRequestBody := &models.UploadEntriesRequestBody{}
-	if err := c.QueryParser(uploadRequestBody); err != nil {
-		return c.Status(fiber.StatusBadRequest).JSON(err)
+	if err := c.BindQuery(uploadRequestBody); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
 	}
 	if err := validation.Validate(uploadRequestBody); err != nil {
-		return c.Status(fiber.StatusBadRequest).JSON(err)
+		c.JSON(http.StatusBadRequest, err)
+		return
 	}
 	if up9.GetAnalyzeInfo().IsAnalyzing {
-		return c.Status(fiber.StatusBadRequest).SendString("Cannot analyze, mizu is already analyzing")
+		c.String(http.StatusBadRequest, "Cannot analyze, mizu is already analyzing")
+		return
 	}
+
 	rlog.Infof("Upload entries - creating token. dest %s\n", uploadRequestBody.Dest)
 	token, err := up9.CreateAnonymousToken(uploadRequestBody.Dest)
 	if err != nil {
-		return c.Status(fiber.StatusServiceUnavailable).SendString("Can't get token")
+		c.String(http.StatusServiceUnavailable, "Cannot analyze, mizu is already analyzing")
+		return
 	}
 	rlog.Infof("Upload entries - uploading. token: %s model: %s\n", token.Token, token.Model)
 	go up9.UploadEntriesImpl(token.Token, token.Model, uploadRequestBody.Dest, uploadRequestBody.SleepIntervalSec)
-	return c.Status(fiber.StatusOK).SendString("OK")
+	c.String(http.StatusOK, "OK")
 }
 
-func GetFullEntries(c *fiber.Ctx) error {
+func GetFullEntries(c *gin.Context) {
 	entriesFilter := &models.HarFetchRequestBody{}
-	if err := c.QueryParser(entriesFilter); err != nil {
-		return c.Status(fiber.StatusBadRequest).JSON(err)
+	if err := c.BindQuery(entriesFilter); err != nil {
+		c.JSON(http.StatusBadRequest, err)
 	}
 	err := validation.Validate(entriesFilter)
 	if err != nil {
-		return c.Status(fiber.StatusBadRequest).JSON(err)
+		c.JSON(http.StatusBadRequest, err)
 	}
 
 	var timestampFrom, timestampTo int64
@@ -195,38 +202,37 @@ func GetFullEntries(c *fiber.Ctx) error {
 		}
 		result = append(result, harEntry)
 	}
-
-	return c.Status(fiber.StatusOK).JSON(result)
+	c.JSON(http.StatusOK, result)
 }
 
-func GetEntry(c *fiber.Ctx) error {
+func GetEntry(c *gin.Context) {
 	var entryData models.MizuEntry
 	database.GetEntriesTable().
-		Where(map[string]string{"entryId": c.Params("entryId")}).
+		Where(map[string]string{"entryId": c.Param("entryId")}).
 		First(&entryData)
 
 	fullEntry := models.FullEntryDetails{}
 	if err := models.GetEntry(&entryData, &fullEntry); err != nil {
-		return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
+		c.JSON(http.StatusInternalServerError, map[string]interface{}{
 			"error": true,
 			"msg":   "Can't get entry details",
 		})
 	}
-	return c.Status(fiber.StatusOK).JSON(fullEntry)
+	c.JSON(http.StatusOK, fullEntry)
 }
 
-func DeleteAllEntries(c *fiber.Ctx) error {
+func DeleteAllEntries(c *gin.Context) {
 	database.GetEntriesTable().
 		Where("1 = 1").
 		Delete(&models.MizuEntry{})
 
-	return c.Status(fiber.StatusOK).JSON(fiber.Map{
+	c.JSON(http.StatusOK, map[string]string{
 		"msg": "Success",
 	})
 
 }
 
-func GetGeneralStats(c *fiber.Ctx) error {
+func GetGeneralStats(c *gin.Context) {
 	sqlQuery := "SELECT count(*) as count, min(timestamp) as min, max(timestamp) as max from mizu_entries"
 	var result struct {
 		Count int
@@ -234,5 +240,17 @@ func GetGeneralStats(c *fiber.Ctx) error {
 		Max   int
 	}
 	database.GetEntriesTable().Raw(sqlQuery).Scan(&result)
-	return c.Status(fiber.StatusOK).JSON(&result)
+	c.JSON(http.StatusOK, result)
+}
+
+func GetTappingStatus(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.TapStatus)
+}
+
+func AnalyzeInformation(c *gin.Context) {
+	c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
+}
+
+func GetRecentTLSLinks(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
 }

agent/pkg/controllers/metadata_controller.go

@@ -0,0 +1,13 @@
+package controllers
+
+import (
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/shared"
+	"mizuserver/pkg/version"
+	"net/http"
+)
+
+func GetVersion(c *gin.Context) {
+	resp := shared.VersionResponse{SemVer: version.SemVer}
+	c.JSON(http.StatusOK, resp)
+}

agent/pkg/controllers/resolving_controller.go

@@ -0,0 +1,12 @@
+package controllers
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/holder"
+	"net/http"
+)
+
+func GetCurrentResolvingInformation(c *gin.Context) {
+	c.JSON(http.StatusOK, holder.GetResolver().GetMap())
+}
+

agent/pkg/controllers/status_controller.go

@@ -0,0 +1,32 @@
+package controllers
+
+import (
+	"encoding/json"
+	"github.com/gin-gonic/gin"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/shared"
+	"mizuserver/pkg/api"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/validation"
+	"net/http"
+)
+
+func PostTappedPods(c *gin.Context) {
+	tapStatus := &shared.TapStatus{}
+	if err := c.Bind(tapStatus); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+	if err := validation.Validate(tapStatus); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+	rlog.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods))
+	providers.TapStatus.Pods = tapStatus.Pods
+	message := shared.CreateWebSocketStatusMessage(*tapStatus)
+	if jsonBytes, err := json.Marshal(message); err != nil {
+		rlog.Errorf("Could not Marshal message %v\n", err)
+	} else {
+		api.BroadcastToBrowserClients(jsonBytes)
+	}
+}

agent/pkg/database/main.go

@@ -12,11 +12,6 @@ import (
 
 const (
 	DBPath = "./entries.db"
-)
-
- var DB *gorm.DB
-
-const (
 	OrderDesc = "desc"
 	OrderAsc  = "asc"
 	LT        = "lt"
@@ -24,6 +19,8 @@ const (
 )
 
 var (
+	DB *gorm.DB
+	IsDBLocked = false
 	OperatorToSymbolMapping = map[string]string{
 		LT: "<",
 		GT: ">",
@@ -43,6 +40,13 @@ func GetEntriesTable() *gorm.DB {
 	return DB.Table("mizu_entries")
 }
 
+func CreateEntry(entry *models.MizuEntry) {
+	if IsDBLocked {
+		return
+	}
+	GetEntriesTable().Create(entry)
+}
+
 func initDataBase(databasePath string) *gorm.DB {
 	temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{
 		Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond},

agent/pkg/database/size_enforcer.go

@@ -62,7 +62,7 @@ func getMaxEntriesDBByteSize() (int64, error) {
 	maxEntriesDBByteSize := defaultMaxDatabaseSizeBytes
 	var err error
 
-	maxEntriesDBSizeByteSEnvVarValue := os.Getenv(shared.MaxEntriesDBSizeByteSEnvVar)
+	maxEntriesDBSizeByteSEnvVarValue := os.Getenv(shared.MaxEntriesDBSizeBytesEnvVar)
 	if maxEntriesDBSizeByteSEnvVarValue != "" {
 		maxEntriesDBByteSize, err = strconv.ParseInt(maxEntriesDBSizeByteSEnvVarValue, 10, 64)
 	}
@@ -81,6 +81,10 @@ func checkFileSize(maxSizeBytes int64) {
 }
 
 func pruneOldEntries(currentFileSize int64) {
+	// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
+	IsDBLocked = true
+	defer func() {IsDBLocked = false}()
+
 	amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
 
 	rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()

agent/pkg/models/models.go

@@ -113,8 +113,8 @@ type EntriesFilter struct {
 }
 
 type UploadEntriesRequestBody struct {
-	Dest             string `query:"dest"`
-	SleepIntervalSec int    `query:"interval"`
+	Dest             string `form:"dest"`
+	SleepIntervalSec int    `form:"interval"`
 }
 
 type HarFetchRequestBody struct {
@@ -132,6 +132,11 @@ type WebSocketTappedEntryMessage struct {
 	Data *tap.OutputChannelItem
 }
 
+type WebsocketOutboundLinkMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *tap.OutboundLink
+}
+
 func CreateBaseEntryWebSocketMessage(base *BaseEntryDetails) ([]byte, error) {
 	message := &WebSocketEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
@@ -152,6 +157,16 @@ func CreateWebsocketTappedEntryMessage(base *tap.OutputChannelItem) ([]byte, err
 	return json.Marshal(message)
 }
 
+func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error) {
+	message := &WebsocketOutboundLinkMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebsocketMessageTypeOutboundLink,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
 // ExtendedHAR is the top level object of a HAR log.
 type ExtendedHAR struct {
 	Log *ExtendedLog `json:"log"`

agent/pkg/providers/status_provider.go

@@ -0,0 +1,28 @@
+package providers
+
+import (
+	"github.com/patrickmn/go-cache"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap"
+	"time"
+)
+
+const tlsLinkRetainmentTime = time.Minute * 15
+
+var (
+	TapStatus shared.TapStatus
+	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
+)
+
+func GetAllRecentTLSAddresses() []string {
+	recentTLSLinks := make([]string, 0)
+
+	for _, outboundLinkItem := range RecentTLSLinks.Items() {
+		outboundLink, castOk := outboundLinkItem.Object.(*tap.OutboundLink)
+		if castOk {
+			recentTLSLinks = append(recentTLSLinks, outboundLink.DstIP)
+		}
+	}
+
+	return recentTLSLinks
+}

agent/pkg/resolver/loader.go

@@ -0,0 +1,22 @@
+package resolver
+
+import (
+	"k8s.io/client-go/kubernetes"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
+	restclient "k8s.io/client-go/rest"
+)
+
+func NewFromInCluster(errOut chan error) (*Resolver, error) {
+	config, err := restclient.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	return &Resolver{clientConfig: config, clientSet: clientset, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}, nil
+}

agent/pkg/routes/entries_routes.go

@@ -0,0 +1,26 @@
+package routes
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/controllers"
+)
+
+// EntriesRoutes defines the group of har entries routes.
+func EntriesRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/api")
+
+	routeGroup.GET("/entries", controllers.GetEntries)        // get entries (base/thin entries)
+	routeGroup.GET("/entries/:entryId", controllers.GetEntry) // get single (full) entry
+	routeGroup.GET("/exportEntries", controllers.GetFullEntries)
+	routeGroup.GET("/uploadEntries", controllers.UploadEntries)
+	routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
+
+	routeGroup.GET("/har", controllers.GetHARs)
+
+	routeGroup.GET("/resetDB", controllers.DeleteAllEntries)     // get single (full) entry
+	routeGroup.GET("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB
+
+	routeGroup.GET("/tapStatus", controllers.GetTappingStatus) // get tapping status
+	routeGroup.GET("/analyzeStatus", controllers.AnalyzeInformation)
+	routeGroup.GET("/recentTLSLinks", controllers.GetRecentTLSLinks)
+}

agent/pkg/routes/metadata_routes.go

@@ -0,0 +1,13 @@
+package routes
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/controllers"
+)
+
+// MetadataRoutes defines the group of metadata routes.
+func MetadataRoutes(app *gin.Engine) {
+	routeGroup := app.Group("/metadata")
+
+	routeGroup.GET("/version", controllers.GetVersion)
+}

agent/pkg/routes/not_found_route.go

@@ -0,0 +1,18 @@
+package routes
+
+import (
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+// NotFoundRoute defines the 404 Error route.
+func NotFoundRoute(app *gin.Engine) {
+	app.Use(
+		func(c *gin.Context) {
+			c.JSON(http.StatusNotFound, map[string]interface{}{
+				"error": true,
+				"msg":   "sorry, endpoint is not found",
+			})
+		},
+	)
+}

agent/pkg/routes/status_routes.go

@@ -0,0 +1,12 @@
+package routes
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/controllers"
+)
+
+func StatusRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/status")
+
+	routeGroup.POST("/tappedPods", controllers.PostTappedPods)
+}

agent/pkg/utils/utils.go

@@ -1,32 +1,42 @@
 package utils
 
 import (
-	"github.com/gofiber/fiber/v2"
+	"context"
+	"github.com/gin-gonic/gin"
 	"github.com/romana/rlog"
 	"log"
+	"net/http"
 	"net/url"
 	"os"
 	"os/signal"
 	"reflect"
 	"syscall"
+	"time"
 )
 
 // StartServer starts the server with a graceful shutdown
-func StartServer(app *fiber.App) {
+func StartServer(app *gin.Engine) {
 	signals := make(chan os.Signal, 2)
 	signal.Notify(signals,
 		os.Interrupt,  	  // this catch ctrl + c
 		syscall.SIGTSTP,  // this catch ctrl + z
 	)
 
+	srv := &http.Server{
+		Addr:    ":8080",
+		Handler: app,
+	}
+
 	go func() {
 		_ = <-signals
 		rlog.Infof("Shutting down...")
-		_ = app.Shutdown()
+		ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
+		_ = srv.Shutdown(ctx)
+		os.Exit(0)
 	}()
 
 	// Run server.
-	if err := app.Listen(":8899"); err != nil {
+	if err := app.Run(":8899"); err != nil {
 		log.Printf("Oops... Server is not running! Reason: %v", err)
 	}
 }

agent/pkg/version/consts.go

@@ -0,0 +1,8 @@
+package version
+
+var (
+	SemVer         = "0.0.1"
+	Branch         = "develop"
+	GitCommitHash  = "" // this var is overridden using ldflags in makefile when building
+	BuildTimestamp = "" // this var is overridden using ldflags in makefile when building
+)

api/pkg/api/socket_server_handlers.go

@@ -1,99 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-	"github.com/antoniodipinto/ikisocket"
-	"github.com/romana/rlog"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap"
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/routes"
-	"mizuserver/pkg/up9"
-)
-
-var browserClientSocketUUIDs = make([]string, 0)
-
-type RoutesEventHandlers struct {
-	routes.EventHandlers
-	SocketHarOutChannel chan<- *tap.OutputChannelItem
-}
-
-func init() {
-	go up9.UpdateAnalyzeStatus(broadcastToBrowserClients)
-}
-
-func (h *RoutesEventHandlers) WebSocketConnect(ep *ikisocket.EventPayload) {
-	if ep.Kws.GetAttribute("is_tapper") == true {
-		rlog.Infof("Websocket Connection event - Tapper connected: %s", ep.SocketUUID)
-	} else {
-		rlog.Infof("Websocket Connection event - Browser socket connected: %s", ep.SocketUUID)
-		browserClientSocketUUIDs = append(browserClientSocketUUIDs, ep.SocketUUID)
-	}
-}
-
-func (h *RoutesEventHandlers) WebSocketDisconnect(ep *ikisocket.EventPayload) {
-	if ep.Kws.GetAttribute("is_tapper") == true {
-		rlog.Infof("Disconnection event - Tapper connected: %s", ep.SocketUUID)
-	} else {
-		rlog.Infof("Disconnection event - Browser socket connected: %s", ep.SocketUUID)
-		removeSocketUUIDFromBrowserSlice(ep.SocketUUID)
-	}
-}
-
-func broadcastToBrowserClients(message []byte) {
-	ikisocket.EmitToList(browserClientSocketUUIDs, message)
-}
-
-func (h *RoutesEventHandlers) WebSocketClose(ep *ikisocket.EventPayload) {
-	if ep.Kws.GetAttribute("is_tapper") == true {
-		rlog.Infof("Websocket Close event - Tapper connected: %s", ep.SocketUUID)
-	} else {
-		rlog.Infof("Websocket  Close event - Browser socket connected: %s", ep.SocketUUID)
-		removeSocketUUIDFromBrowserSlice(ep.SocketUUID)
-	}
-}
-
-func (h *RoutesEventHandlers) WebSocketError(ep *ikisocket.EventPayload) {
-	rlog.Infof("Socket error - Socket uuid : %s %v", ep.SocketUUID, ep.Error)
-}
-
-func (h *RoutesEventHandlers) WebSocketMessage(ep *ikisocket.EventPayload) {
-	var socketMessageBase shared.WebSocketMessageMetadata
-	err := json.Unmarshal(ep.Data, &socketMessageBase)
-	if err != nil {
-		rlog.Infof("Could not unmarshal websocket message %v\n", err)
-	} else {
-		switch socketMessageBase.MessageType {
-		case shared.WebSocketMessageTypeTappedEntry:
-			var tappedEntryMessage models.WebSocketTappedEntryMessage
-			err := json.Unmarshal(ep.Data, &tappedEntryMessage)
-			if err != nil {
-				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
-			} else {
-				h.SocketHarOutChannel <- tappedEntryMessage.Data
-			}
-		case shared.WebSocketMessageTypeUpdateStatus:
-			var statusMessage shared.WebSocketStatusMessage
-			err := json.Unmarshal(ep.Data, &statusMessage)
-			if err != nil {
-				rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
-			} else {
-				controllers.TapStatus = statusMessage.TappingStatus
-				broadcastToBrowserClients(ep.Data)
-			}
-		default:
-			rlog.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
-		}
-	}
-}
-
-func removeSocketUUIDFromBrowserSlice(uuidToRemove string) {
-	newUUIDSlice := make([]string, 0, len(browserClientSocketUUIDs))
-	for _, uuid := range browserClientSocketUUIDs {
-		if uuid != uuidToRemove {
-			newUUIDSlice = append(newUUIDSlice, uuid)
-		}
-	}
-	browserClientSocketUUIDs = newUUIDSlice
-}

api/pkg/controllers/resolving_controller.go

@@ -1,11 +0,0 @@
-package controllers
-
-import (
-	"github.com/gofiber/fiber/v2"
-	"mizuserver/pkg/holder"
-)
-
-func GetCurrentResolvingInformation(c *fiber.Ctx) error {
-	return c.Status(fiber.StatusOK).JSON(holder.GetResolver().GetMap())
-}
-

api/pkg/controllers/status_controller.go

@@ -1,17 +0,0 @@
-package controllers
-
-import (
-	"github.com/gofiber/fiber/v2"
-	"github.com/up9inc/mizu/shared"
-	"mizuserver/pkg/up9"
-)
-
-var TapStatus shared.TapStatus
-
-func GetTappingStatus(c *fiber.Ctx) error {
-	return c.Status(fiber.StatusOK).JSON(TapStatus)
-}
-
-func AnalyzeInformation(c *fiber.Ctx) error {
-	return c.Status(fiber.StatusOK).JSON(up9.GetAnalyzeInfo())
-}

api/pkg/middleware/fiber_middleware.go

@@ -1,18 +0,0 @@
-package middleware
-
-import (
-	"github.com/gofiber/fiber/v2"
-	"github.com/gofiber/fiber/v2/middleware/cors"
-	"github.com/gofiber/fiber/v2/middleware/logger"
-)
-
-// FiberMiddleware provide Fiber's built-in middlewares.
-// See: https://docs.gofiber.io/api/middleware
-func FiberMiddleware(a *fiber.App) {
-	a.Use(
-		// Add CORS to each route.
-		cors.New(),
-		// Add simple logger.
-		logger.New(),
-	)
-}

api/pkg/resolver/loader.go

@@ -1,61 +0,0 @@
-package resolver
-
-import (
-	"k8s.io/client-go/kubernetes"
-	_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
-	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
-	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
-	_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
-	restclient "k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/util/homedir"
-	"path/filepath"
-)
-
-func NewFromInCluster(errOut chan error) (*Resolver, error) {
-	config, err := restclient.InClusterConfig()
-	if err != nil {
-		return nil, err
-	}
-	clientset, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		return nil, err
-	}
-	return &Resolver{clientConfig: config, clientSet: clientset, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}, nil
-}
-
-func NewFromOutOfCluster(kubeConfigPath string, errOut chan error) (*Resolver, error) {
-	if kubeConfigPath == "" {
-		home := homedir.HomeDir()
-		kubeConfigPath = filepath.Join(home, ".kube", "config")
-	}
-
-	configPathList := filepath.SplitList(kubeConfigPath)
-	configLoadingRules := &clientcmd.ClientConfigLoadingRules{}
-	if len(configPathList) <= 1 {
-		configLoadingRules.ExplicitPath = kubeConfigPath
-	} else {
-		configLoadingRules.Precedence = configPathList
-	}
-	contextName := ""
-	clientConfigLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
-		configLoadingRules,
-		&clientcmd.ConfigOverrides{
-			CurrentContext: contextName,
-		},
-	)
-	clientConfig, err := clientConfigLoader.ClientConfig()
-	if err != nil {
-		return nil, err
-	}
-	clientset, err := kubernetes.NewForConfig(clientConfig)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Resolver{clientConfig: clientConfig, clientSet: clientset, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}, nil
-}
-
-func NewFromExisting(clientConfig *restclient.Config, clientSet *kubernetes.Clientset, errOut chan error) *Resolver {
-	return &Resolver{clientConfig: clientConfig, clientSet: clientSet, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}
-}

api/pkg/routes/not_found_route.go

@@ -1,15 +0,0 @@
-package routes
-
-import "github.com/gofiber/fiber/v2"
-
-// NotFoundRoute func for describe 404 Error route.
-func NotFoundRoute(fiberApp *fiber.App) {
-	fiberApp.Use(
-		func(c *fiber.Ctx) error {
-			return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
-				"error": true,
-				"msg":   "sorry, endpoint is not found",
-			})
-		},
-	)
-}

api/pkg/routes/public_routes.go

@@ -1,25 +0,0 @@
-package routes
-
-import (
-	"github.com/gofiber/fiber/v2"
-	"mizuserver/pkg/controllers"
-)
-
-// EntriesRoutes func for describe group of public routes.
-func EntriesRoutes(fiberApp *fiber.App) {
-	routeGroup := fiberApp.Group("/api")
-
-	routeGroup.Get("/entries", controllers.GetEntries)        // get entries (base/thin entries)
-	routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry
-	routeGroup.Get("/exportEntries", controllers.GetFullEntries)
-	routeGroup.Get("/uploadEntries", controllers.UploadEntries)
-	routeGroup.Get("/resolving", controllers.GetCurrentResolvingInformation)
-
-	routeGroup.Get("/har", controllers.GetHARs)
-
-	routeGroup.Get("/resetDB", controllers.DeleteAllEntries)     // get single (full) entry
-	routeGroup.Get("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB
-
-	routeGroup.Get("/tapStatus", controllers.GetTappingStatus) // get tapping status
-	routeGroup.Get("/analyzeStatus", controllers.AnalyzeInformation)
-}

api/pkg/routes/socket_routes.go

@@ -1,31 +0,0 @@
-package routes
-
-import (
-	"github.com/antoniodipinto/ikisocket"
-	"github.com/gofiber/fiber/v2"
-)
-
-type EventHandlers interface {
-	WebSocketConnect(ep *ikisocket.EventPayload)
-	WebSocketDisconnect(ep *ikisocket.EventPayload)
-	WebSocketClose(ep *ikisocket.EventPayload)
-	WebSocketError(ep *ikisocket.EventPayload)
-	WebSocketMessage(ep *ikisocket.EventPayload)
-}
-
-func WebSocketRoutes(app *fiber.App, eventHandlers EventHandlers) {
-	app.Get("/ws", ikisocket.New(func(kws *ikisocket.Websocket) {
-		kws.SetAttribute("is_tapper", false)
-	}))
-
-	app.Get("/wsTapper", ikisocket.New(func(kws *ikisocket.Websocket) {
-		// Tapper clients are handled differently, they don't need to receive new message broadcasts.
-		kws.SetAttribute("is_tapper", true)
-	}))
-
-	ikisocket.On(ikisocket.EventMessage, eventHandlers.WebSocketMessage)
-	ikisocket.On(ikisocket.EventConnect, eventHandlers.WebSocketConnect)
-	ikisocket.On(ikisocket.EventDisconnect, eventHandlers.WebSocketDisconnect)
-	ikisocket.On(ikisocket.EventClose, eventHandlers.WebSocketClose) // This event is called when the server disconnects the user actively with .Close() method
-	ikisocket.On(ikisocket.EventError, eventHandlers.WebSocketError) // On error event
-}

build-push-featurebranch.sh

@@ -5,7 +5,9 @@ SERVER_NAME=mizu
 GCP_PROJECT=up9-docker-hub
 REPOSITORY=gcr.io/$GCP_PROJECT
 GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
-DOCKER_TAGGED_BUILD=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH:latest
+SEM_VER=${SEM_VER=0.0.0}
+DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
+DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
 
 if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
 then
@@ -13,8 +15,12 @@ then
   exit 1
 fi
 
-echo "building $DOCKER_TAGGED_BUILD"
-docker build -t "$DOCKER_TAGGED_BUILD" .
+echo "building ${DOCKER_TAGGED_BUILDS[@]}"
+DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
+docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
 
-echo pushing to "$REPOSITORY"
-docker push "$DOCKER_TAGGED_BUILD"
+for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
+do
+        echo pushing "$DOCKER_TAG"
+        docker push "$DOCKER_TAG"
+done

cli/.gitignore

@@ -0,0 +1 @@
+bin

cli/Makefile

@@ -3,6 +3,7 @@ COMMIT_HASH=$(shell git rev-parse HEAD)
 GIT_BRANCH=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
 GIT_VERSION=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
 BUILD_TIMESTAMP=$(shell date +%s)
+export SEM_VER?=0.0.0
 
 .PHONY: help
 .DEFAULT_GOAL := help
@@ -13,7 +14,7 @@ help: ## This help.
 install:
 	go install mizu.go
 
-build:	## build mizu CLI binary (select platform via GOOS / GOARCH env variables)
+build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
 	go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
@@ -21,7 +22,7 @@ build:	## build mizu CLI binary (select platform via GOOS / GOARCH env variables
 					   -o bin/mizu_$(SUFFIX) mizu.go 
 	(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
 
-build-all:  ## build for all supported platforms
+build-all: ## Build for all supported platforms.
 	@echo "Compiling for every OS and Platform"
 	@mkdir -p bin && echo "SHA256 checksums available for compiled binaries \n\nRun \`shasum -a 256 -c mizu_OS_ARCH.sha256\` to verify\n\n" >  bin/README.md
 	@$(MAKE) build GOOS=darwin GOARCH=amd64
@@ -35,6 +36,6 @@ build-all:  ## build for all supported platforms
 	@echo "---------"
 	@find ./bin -ls
 
-clean: ## clean all build artifacts
+clean: ## Clean all build artifacts.
 	go clean
 	rm -rf ./bin/*

cli/cmd/config.go

@@ -0,0 +1,40 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"io/ioutil"
+)
+
+var regenerateFile bool
+
+var configCmd = &cobra.Command{
+	Use:   "config",
+	Short: "Generate config with default values",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		template, err := mizu.GetConfigWithDefaults()
+		if err != nil {
+			mizu.Log.Errorf("Failed generating config with defaults %v", err)
+			return nil
+		}
+		if regenerateFile {
+			data := []byte(template)
+			if err := ioutil.WriteFile(mizu.GetConfigFilePath(), data, 0644); err != nil {
+				mizu.Log.Errorf("Failed writing config %v", err)
+				return nil
+			}
+			mizu.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, mizu.GetConfigFilePath())))
+		} else {
+			mizu.Log.Debugf("Writing template config.\n%v", template)
+			fmt.Printf("%v", template)
+		}
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(configCmd)
+	configCmd.Flags().BoolVarP(&regenerateFile, "regenerate", "r", false, fmt.Sprintf("Regenerate the config file with default values %s", mizu.GetConfigFilePath()))
+}

cli/cmd/fetch.go

@@ -1,23 +1,23 @@
 package cmd
 
 import (
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/configStructs"
 )
 
-type MizuFetchOptions struct {
-	FromTimestamp int64
-	ToTimestamp   int64
-	Directory     string
-	MizuPort      uint16
-}
-
-var mizuFetchOptions = MizuFetchOptions{}
-
 var fetchCmd = &cobra.Command{
 	Use:   "fetch",
 	Short: "Download recorded traffic to files",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		RunMizuFetch(&mizuFetchOptions)
+		go mizu.ReportRun("fetch", mizu.Config.Fetch)
+		if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.Fetch.MizuPort); err != nil {
+			return err
+		} else if !isCompatible {
+			return nil
+		}
+		RunMizuFetch()
 		return nil
 	},
 }
@@ -25,8 +25,11 @@ var fetchCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(fetchCmd)
 
-	fetchCmd.Flags().StringVarP(&mizuFetchOptions.Directory, "directory", "d", ".", "Provide a custom directory for fetched entries")
-	fetchCmd.Flags().Int64Var(&mizuFetchOptions.FromTimestamp, "from", 0, "Custom start timestamp for fetched entries")
-	fetchCmd.Flags().Int64Var(&mizuFetchOptions.ToTimestamp, "to", 0, "Custom end timestamp fetched entries")
-	fetchCmd.Flags().Uint16VarP(&mizuFetchOptions.MizuPort, "port", "p", 8899, "Custom port for mizu")
+	defaultFetchConfig := configStructs.FetchConfig{}
+	defaults.Set(&defaultFetchConfig)
+
+	fetchCmd.Flags().StringP(configStructs.DirectoryFetchName, "d", defaultFetchConfig.Directory, "Provide a custom directory for fetched entries")
+	fetchCmd.Flags().Int(configStructs.FromTimestampFetchName, defaultFetchConfig.FromTimestamp, "Custom start timestamp for fetched entries")
+	fetchCmd.Flags().Int(configStructs.ToTimestampFetchName, defaultFetchConfig.ToTimestamp, "Custom end timestamp fetched entries")
+	fetchCmd.Flags().Uint16P(configStructs.MizuPortFetchName, "p", defaultFetchConfig.MizuPort, "Custom port for mizu")
 }

cli/cmd/fetchRunner.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/up9inc/mizu/cli/kubernetes"
+	"github.com/up9inc/mizu/cli/mizu"
 	"io"
 	"io/ioutil"
 	"log"
@@ -14,9 +15,9 @@ import (
 	"strings"
 )
 
-func RunMizuFetch(fetch *MizuFetchOptions) {
-	mizuProxiedUrl := kubernetes.GetMizuCollectorProxiedHostAndPath(fetch.MizuPort)
-	resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, fetch.FromTimestamp, fetch.ToTimestamp))
+func RunMizuFetch() {
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Fetch.MizuPort)
+	resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, mizu.Config.Fetch.FromTimestamp, mizu.Config.Fetch.ToTimestamp))
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -32,8 +33,8 @@ func RunMizuFetch(fetch *MizuFetchOptions) {
 	if err != nil {
 		log.Fatal(err)
 	}
-	_ = Unzip(zipReader, fetch.Directory)
 
+	_ = Unzip(zipReader, mizu.Config.Fetch.Directory)
 }
 
 func Unzip(reader *zip.Reader, dest string) error {
@@ -63,7 +64,7 @@ func Unzip(reader *zip.Reader, dest string) error {
 			_ = os.MkdirAll(path, f.Mode())
 		} else {
 			_ = os.MkdirAll(filepath.Dir(path), f.Mode())
-			fmt.Print("writing HAR file [ ", path, " ] .. ")
+			mizu.Log.Infof("writing HAR file [ %v ]", path)
 			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
 			if err != nil {
 				return err
@@ -72,7 +73,7 @@ func Unzip(reader *zip.Reader, dest string) error {
 				if err := f.Close(); err != nil {
 					panic(err)
 				}
-				fmt.Println(" done")
+				mizu.Log.Info(" done")
 			}()
 
 			_, err = io.Copy(f, rc)

cli/cmd/root.go

@@ -1,7 +1,10 @@
 package cmd
 
 import (
+	"errors"
+	"fmt"
 	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/mizu"
 )
 
 var rootCmd = &cobra.Command{
@@ -9,6 +12,18 @@ var rootCmd = &cobra.Command{
 	Short: "A web traffic viewer for kubernetes",
 	Long: `A web traffic viewer for kubernetes
 Further info is available at https://github.com/up9inc/mizu`,
+	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		if err := mizu.InitConfig(cmd); err != nil {
+			mizu.Log.Errorf("Invalid config, Exit %s", err)
+			return errors.New(fmt.Sprintf("%v", err))
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringSlice(mizu.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", mizu.SetCommandName))
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.

cli/cmd/tap.go

@@ -2,41 +2,15 @@ package cmd
 
 import (
 	"errors"
-	"fmt"
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/configStructs"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/units"
 	"os"
-	"regexp"
-	"strings"
 )
 
-type MizuTapOptions struct {
-	GuiPort                uint16
-	Namespace              string
-	AllNamespaces          bool
-	Analysis               bool
-	AnalysisDestination    string
-	KubeConfigPath         string
-	MizuImage              string
-	PlainTextFilterRegexes []string
-	TapOutgoing            bool
-	HideHealthChecks       bool
-	MaxEntriesDBSizeBytes  int64
-	SleepIntervalSec       uint16
-}
-
-var mizuTapOptions = &MizuTapOptions{}
-var direction string
-var humanMaxEntriesDBSize string
-var regex *regexp.Regexp
-const maxEntriesDBSizeFlagName = "max-entries-db-size"
-
-
-const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic
-to UP9 cloud for further analysis and enriched presentation options.
-`
+const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic for further analysis and enriched presentation options.`
 
 var tapCmd = &cobra.Command{
 	Use:   "tap [POD REGEX]",
@@ -44,47 +18,31 @@ var tapCmd = &cobra.Command{
 	Long: `Record the ingoing traffic of a kubernetes pod.
 Supported protocols are HTTP and gRPC.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		RunMizuTap(regex, mizuTapOptions)
+		go mizu.ReportRun("tap", mizu.Config.Tap)
+		RunMizuTap()
 		return nil
 	},
 	PreRunE: func(cmd *cobra.Command, args []string) error {
-		if len(args) == 0 {
-			return errors.New("POD REGEX argument is required")
+		if len(args) == 1 {
+			mizu.Config.Tap.PodRegexStr = args[0]
 		} else if len(args) > 1 {
 			return errors.New("unexpected number of arguments")
 		}
 
-		var compileErr error
-		regex, compileErr = regexp.Compile(args[0])
-		if compileErr != nil {
-			return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], compileErr))
+		if err := mizu.Config.Tap.Validate(); err != nil {
+			return err
 		}
 
-		var parseHumanDataSizeErr error
-		mizuTapOptions.MaxEntriesDBSizeBytes, parseHumanDataSizeErr = units.HumanReadableToBytes(humanMaxEntriesDBSize)
-		if parseHumanDataSizeErr != nil {
-			return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize))
-		} else if cmd.Flags().Changed(maxEntriesDBSizeFlagName) {
-			// We're parsing human readable file sizes here so its best to be unambiguous
-			fmt.Printf("Setting max entries db size to %s\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
-		}
+		mizu.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", mizu.Config.Tap.HumanMaxEntriesDBSize)
 
-		directionLowerCase := strings.ToLower(direction)
-		if directionLowerCase == "any" {
-			mizuTapOptions.TapOutgoing = true
-		} else if directionLowerCase == "in" {
-			mizuTapOptions.TapOutgoing = false
-		} else {
-			return errors.New(fmt.Sprintf("%s is not a valid value for flag --direction. Acceptable values are in/any.", direction))
-		}
-
-		if mizuTapOptions.Analysis {
-			fmt.Printf(analysisMessageToConfirm)
-			if !uiUtils.AskForConfirmation("Would you like to proceed [y/n]: ") {
-				fmt.Println("You can always run mizu without analysis, aborting")
+		if mizu.Config.Tap.Analysis {
+			mizu.Log.Infof(analysisMessageToConfirm)
+			if !uiUtils.AskForConfirmation("Would you like to proceed [Y/n]: ") {
+				mizu.Log.Infof("You can always run mizu without analysis, aborting")
 				os.Exit(0)
 			}
 		}
+
 		return nil
 	},
 }
@@ -92,16 +50,18 @@ Supported protocols are HTTP and gRPC.`,
 func init() {
 	rootCmd.AddCommand(tapCmd)
 
-	tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
-	tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector")
-	tapCmd.Flags().BoolVar(&mizuTapOptions.Analysis, "analysis", false, "Uploads traffic to UP9 for further analysis (Beta)")
-	tapCmd.Flags().StringVar(&mizuTapOptions.AnalysisDestination, "dest", "up9.app", "Destination environment")
-	tapCmd.Flags().Uint16VarP(&mizuTapOptions.SleepIntervalSec, "upload-interval", "", 10, "Interval in seconds for uploading data to UP9")
-	tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces")
-	tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
-	tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector")
-	tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "regex-masking", "r", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies")
-	tapCmd.Flags().StringVarP(&direction, "direction", "", "in", "Record traffic that goes in this direction (relative to the tapped pod): in/any")
-	tapCmd.Flags().BoolVar(&mizuTapOptions.HideHealthChecks, "hide-healthchecks", false, "hides requests with kube-probe or prometheus user-agent headers")
-	tapCmd.Flags().StringVarP(&humanMaxEntriesDBSize, maxEntriesDBSizeFlagName, "", "200MB", "override the default max entries db size of 200mb")
+	defaultTapConfig := configStructs.TapConfig{}
+	defaults.Set(&defaultTapConfig)
+
+	tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
+	tapCmd.Flags().StringP(configStructs.NamespaceTapName, "n", defaultTapConfig.Namespace, "Namespace selector")
+	tapCmd.Flags().Bool(configStructs.AnalysisTapName, defaultTapConfig.Analysis, "Uploads traffic to UP9 for further analysis (Beta)")
+	tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
+	tapCmd.Flags().StringP(configStructs.KubeConfigPathTapName, "k", defaultTapConfig.KubeConfigPath, "Path to kube-config file")
+	tapCmd.Flags().StringArrayP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
+	tapCmd.Flags().Bool(configStructs.HideHealthChecksTapName, defaultTapConfig.HideHealthChecks, "hides requests with kube-probe or prometheus user-agent headers")
+	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
+	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "override the default max entries db size of 200mb")
+	tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any")
+	tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
 }

cli/cmd/tapRunner.go

@@ -1,15 +1,19 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
-	"github.com/romana/rlog"
 	"github.com/up9inc/mizu/cli/kubernetes"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
 	core "k8s.io/api/core/v1"
-	"log"
+	errors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/clientcmd"
 	"net/http"
 	"net/url"
 	"os"
@@ -20,47 +24,62 @@ import (
 )
 
 var mizuServiceAccountExists bool
-var aggregatorService *core.Service
+var apiServerService *core.Service
 
 const (
 	updateTappersDelay = 5 * time.Second
+	cleanupTimeout     = time.Minute
 )
 
 var currentlyTappedPods []core.Pod
 
-func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
-	mizuApiFilteringOptions, err := getMizuApiFilteringOptions(tappingOptions)
+func RunMizuTap() {
+	mizuApiFilteringOptions, err := getMizuApiFilteringOptions()
 	if err != nil {
 		return
 	}
 
-	kubernetesProvider := kubernetes.NewProvider(tappingOptions.KubeConfigPath)
+	kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.Tap.KubeConfigPath)
+	if err != nil {
+		if clientcmd.IsEmptyConfig(err) {
+			mizu.Log.Infof(uiUtils.Red, "Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'\n")
+			return
+		}
+		if clientcmd.IsConfigurationInvalid(err) {
+			mizu.Log.Infof(uiUtils.Red, "Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'\n")
+			return
+		}
+	}
 
 	defer cleanUpMizuResources(kubernetesProvider)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel() // cancel will be called when this function exits
 
-	targetNamespace := getNamespace(tappingOptions, kubernetesProvider)
-	if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegexQuery, targetNamespace); err != nil {
-		return
-	} else {
-		currentlyTappedPods = matchingPods
-	}
-
+	targetNamespace := getNamespace(kubernetesProvider)
 	var namespacesStr string
 	if targetNamespace != mizu.K8sAllNamespaces {
 		namespacesStr = fmt.Sprintf("namespace \"%s\"", targetNamespace)
 	} else {
 		namespacesStr = "all namespaces"
 	}
-	fmt.Printf("Tapping pods in %s\n", namespacesStr)
+	mizu.CheckNewerVersion()
+	mizu.Log.Infof("Tapping pods in %s", namespacesStr)
+
+	if err, _ := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespace); err != nil {
+		mizu.Log.Infof("Error listing pods: %v", err)
+		return
+	}
 
 	if len(currentlyTappedPods) == 0 {
 		var suggestionStr string
 		if targetNamespace != mizu.K8sAllNamespaces {
 			suggestionStr = "\nSelect a different namespace with -n or tap all namespaces with -A"
 		}
-		fmt.Printf("Did not find any pods matching the regex argument%s\n", suggestionStr)
+		mizu.Log.Infof("Did not find any pods matching the regex argument%s", suggestionStr)
+	}
+
+	if mizu.Config.Tap.DryRun {
+		return
 	}
 
 	nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
@@ -68,86 +87,116 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
 		return
 	}
 
-	if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions, mizuApiFilteringOptions); err != nil {
+	if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions); err != nil {
 		return
 	}
 
-	go portForwardApiPod(ctx, kubernetesProvider, cancel, tappingOptions) // TODO convert this to job for built in pod ttl or have the running app handle this
-	go watchPodsForTapping(ctx, kubernetesProvider, cancel, podRegexQuery, tappingOptions)
-	go syncApiStatus(ctx, cancel, tappingOptions)
+	go createProxyToApiServerPod(ctx, kubernetesProvider, cancel)
+	go watchPodsForTapping(ctx, kubernetesProvider, cancel)
 
 	//block until exit signal or error
 	waitForFinish(ctx, cancel)
 }
 
-func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
-	if err := createMizuAggregator(ctx, kubernetesProvider, tappingOptions, mizuApiFilteringOptions); err != nil {
+func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
+	if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
 		return err
 	}
 
-	if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
+	if err := createMizuApiServer(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
+		return err
+	}
+
+	if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
+func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider) error {
+	_, err := kubernetesProvider.CreateNamespace(ctx, mizu.ResourcesNamespace)
+	if err != nil {
+		mizu.Log.Infof("Error creating Namespace %s: %v", mizu.ResourcesNamespace, err)
+		return err
+	}
+	mizu.Log.Debugf("Successfully creating Namespace %s", mizu.ResourcesNamespace)
+	return nil
+}
+
+func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
 	var err error
 
 	mizuServiceAccountExists = createRBACIfNecessary(ctx, kubernetesProvider)
-	_, err = kubernetesProvider.CreateMizuAggregatorPod(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.MizuImage, mizuServiceAccountExists, mizuApiFilteringOptions, tappingOptions.MaxEntriesDBSizeBytes)
+	var serviceAccountName string
+	if mizuServiceAccountExists {
+		serviceAccountName = mizu.ServiceAccountName
+	} else {
+		serviceAccountName = ""
+	}
+	_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, mizu.Config.MizuImage, serviceAccountName, mizuApiFilteringOptions, mizu.Config.Tap.MaxEntriesDBSizeBytes())
 	if err != nil {
-		fmt.Printf("Error creating mizu collector pod: %v\n", err)
+		mizu.Log.Infof("Error creating mizu %s pod: %v", mizu.ApiServerPodName, err)
 		return err
 	}
+	mizu.Log.Debugf("Successfully created API server pod: %s", mizu.ApiServerPodName)
 
-	aggregatorService, err = kubernetesProvider.CreateService(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, mizu.AggregatorPodName)
+	apiServerService, err = kubernetesProvider.CreateService(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, mizu.ApiServerPodName)
 	if err != nil {
-		fmt.Printf("Error creating mizu collector service: %v\n", err)
+		mizu.Log.Infof("Error creating mizu %s service: %v", mizu.ApiServerPodName, err)
 		return err
 	}
+	mizu.Log.Debugf("Successfully created service: %s", mizu.ApiServerPodName)
 
 	return nil
 }
 
-func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.TrafficFilteringOptions, error) {
+func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
+
 	var compiledRegexSlice []*shared.SerializableRegexp
 
-	if tappingOptions.PlainTextFilterRegexes != nil && len(tappingOptions.PlainTextFilterRegexes) > 0 {
+	if mizu.Config.Tap.PlainTextFilterRegexes != nil && len(mizu.Config.Tap.PlainTextFilterRegexes) > 0 {
 		compiledRegexSlice = make([]*shared.SerializableRegexp, 0)
-		for _, regexStr := range tappingOptions.PlainTextFilterRegexes {
+		for _, regexStr := range mizu.Config.Tap.PlainTextFilterRegexes {
 			compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr)
 			if err != nil {
-				fmt.Printf("Regex %s is invalid: %v", regexStr, err)
+				mizu.Log.Infof("Regex %s is invalid: %v", regexStr, err)
 				return nil, err
 			}
 			compiledRegexSlice = append(compiledRegexSlice, compiledRegex)
 		}
 	}
 
-	return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: tappingOptions.HideHealthChecks}, nil
+	return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: mizu.Config.Tap.HideHealthChecks, DisableRedaction: mizu.Config.Tap.DisableRedaction}, nil
 }
 
-func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error {
+func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string) error {
 	if len(nodeToTappedPodIPMap) > 0 {
+		var serviceAccountName string
+		if mizuServiceAccountExists {
+			serviceAccountName = mizu.ServiceAccountName
+		} else {
+			serviceAccountName = ""
+		}
+
 		if err := kubernetesProvider.ApplyMizuTapperDaemonSet(
 			ctx,
 			mizu.ResourcesNamespace,
 			mizu.TapperDaemonSetName,
-			tappingOptions.MizuImage,
+			mizu.Config.MizuImage,
 			mizu.TapperPodName,
-			fmt.Sprintf("%s.%s.svc.cluster.local", aggregatorService.Name, aggregatorService.Namespace),
+			fmt.Sprintf("%s.%s.svc.cluster.local", apiServerService.Name, apiServerService.Namespace),
 			nodeToTappedPodIPMap,
-			mizuServiceAccountExists,
-			tappingOptions.TapOutgoing,
+			serviceAccountName,
+			mizu.Config.Tap.TapOutgoing(),
 		); err != nil {
-			fmt.Printf("Error creating mizu tapper daemonset: %v\n", err)
+			mizu.Log.Infof("Error creating mizu tapper daemonset: %v", err)
 			return err
 		}
+		mizu.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap))
 	} else {
 		if err := kubernetesProvider.RemoveDaemonSet(ctx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
-			fmt.Printf("Error deleting mizu tapper daemonset: %v\n", err)
+			mizu.Log.Errorf("Error deleting mizu tapper daemonset: %v", err)
 			return err
 		}
 	}
@@ -156,41 +205,89 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
 }
 
 func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
-	fmt.Printf("\nRemoving mizu resources\n")
+	mizu.Log.Infof("\nRemoving mizu resources\n")
 
-	removalCtx, _ := context.WithTimeout(context.Background(), 5*time.Second)
-	if err := kubernetesProvider.RemovePod(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil {
-		fmt.Printf("Error removing Pod %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err)
+	removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
+	defer cancel()
+
+	if err := kubernetesProvider.RemoveNamespace(removalCtx, mizu.ResourcesNamespace); err != nil {
+		mizu.Log.Infof("Error removing Namespace %s: %s (%v,%+v)", mizu.ResourcesNamespace, err, err, err)
+		return
 	}
-	if err := kubernetesProvider.RemoveService(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil {
-		fmt.Printf("Error removing Service %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err)
+
+	if mizuServiceAccountExists {
+		if err := kubernetesProvider.RemoveNonNamespacedResources(removalCtx, mizu.ClusterRoleName, mizu.ClusterRoleBindingName); err != nil {
+			mizu.Log.Infof("Error removing non-namespaced resources: %s (%v,%+v)", err, err, err)
+			return
+		}
 	}
-	if err := kubernetesProvider.RemoveDaemonSet(removalCtx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
-		fmt.Printf("Error removing DaemonSet %s in namespace %s: %s (%v,%+v)\n", mizu.TapperDaemonSetName, mizu.ResourcesNamespace, err, err, err)
+
+	// Call cancel if a terminating signal was received. Allows user to skip the wait.
+	go func() {
+		waitForFinish(removalCtx, cancel)
+	}()
+
+	if err := kubernetesProvider.WaitUtilNamespaceDeleted(removalCtx, mizu.ResourcesNamespace); err != nil {
+		switch {
+		case removalCtx.Err() == context.Canceled:
+			// Do nothing. User interrupted the wait.
+		case err == wait.ErrWaitTimeout:
+			mizu.Log.Infof("Timeout while removing Namespace %s", mizu.ResourcesNamespace)
+		default:
+			mizu.Log.Infof("Error while waiting for Namespace %s to be deleted: %s (%v,%+v)", mizu.ResourcesNamespace, err, err, err)
+		}
 	}
 }
 
-func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, podRegex *regexp.Regexp, tappingOptions *MizuTapOptions) {
-	targetNamespace := getNamespace(tappingOptions, kubernetesProvider)
+func reportTappedPods() {
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Fetch.MizuPort)
+	tappedPodsUrl := fmt.Sprintf("http://%s/status/tappedPods", mizuProxiedUrl)
 
-	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, targetNamespace), podRegex)
+	podInfos := make([]shared.PodInfo, 0)
+	for _, pod := range currentlyTappedPods {
+		podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
+	}
+	tapStatus := shared.TapStatus{Pods: podInfos}
+
+	if jsonValue, err := json.Marshal(tapStatus); err != nil {
+		mizu.Log.Debugf("[ERROR] failed Marshal the tapped pods %v", err)
+	} else {
+		if response, err := http.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+			mizu.Log.Debugf("[ERROR] failed sending to API server the tapped pods %v", err)
+		} else if response.StatusCode != 200 {
+			mizu.Log.Debugf("[ERROR] failed sending to API server the tapped pods, response status code %v", response.StatusCode)
+		} else {
+			mizu.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos))
+		}
+	}
+}
+
+func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+	targetNamespace := getNamespace(kubernetesProvider)
+	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, targetNamespace), mizu.Config.Tap.PodRegex())
 
 	restartTappers := func() {
-		if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex, targetNamespace); err != nil {
-			fmt.Printf("Error getting pods by regex: %s (%v,%+v)\n", err, err, err)
+		err, changeFound := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespace)
+		if err != nil {
+			mizu.Log.Errorf("Error getting pods by regex: %s (%v,%+v)", err, err, err)
 			cancel()
-		} else {
-			currentlyTappedPods = matchingPods
 		}
 
+		if !changeFound {
+			mizu.Log.Debugf("Nothing changed update tappers not needed")
+			return
+		}
+
+		reportTappedPods()
+
 		nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
 		if err != nil {
-			fmt.Printf("Error building node to ips map: %s (%v,%+v)\n", err, err, err)
+			mizu.Log.Errorf("Error building node to ips map: %s (%v,%+v)", err, err, err)
 			cancel()
 		}
 
-		if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
-			fmt.Printf("Error updating daemonset: %s (%v,%+v)\n", err, err, err)
+		if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap); err != nil {
+			mizu.Log.Errorf("Error updating daemonset: %s (%v,%+v)", err, err, err)
 			cancel()
 		}
 	}
@@ -198,21 +295,21 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 
 	for {
 		select {
-		case newTarget := <-added:
-			fmt.Printf(mizu.Green, fmt.Sprintf("+%s\n", newTarget.Name))
-
-		case removedTarget := <-removed:
-			fmt.Printf(mizu.Red, fmt.Sprintf("-%s\n", removedTarget.Name))
+		case pod := <-added:
+			mizu.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
 			restartTappersDebouncer.SetOn()
-
-		case modifiedTarget := <-modified:
+		case pod := <-removed:
+			mizu.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
+			restartTappersDebouncer.SetOn()
+		case pod := <-modified:
+			mizu.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
 			// Act only if the modified pod has already obtained an IP address.
 			// After filtering for IPs, on a normal pod restart this includes the following events:
 			// - Pod deletion
 			// - Pod reaches start state
 			// - Pod reaches ready state
 			// Ready/unready transitions might also trigger this event.
-			if modifiedTarget.Status.PodIP != "" {
+			if pod.Status.PodIP != "" {
 				restartTappersDebouncer.SetOn()
 			}
 
@@ -226,74 +323,128 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	}
 }
 
-func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
-	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.AggregatorPodName))
+func updateCurrentlyTappedPods(kubernetesProvider *kubernetes.Provider, ctx context.Context, targetNamespace string) (error, bool) {
+	changeFound := false
+	if matchingPods, err := kubernetesProvider.GetAllRunningPodsMatchingRegex(ctx, mizu.Config.Tap.PodRegex(), targetNamespace); err != nil {
+		mizu.Log.Infof("Error getting pods by regex: %s (%v,%+v)", err, err, err)
+		return err, false
+	} else {
+		addedPods, removedPods := getPodArrayDiff(currentlyTappedPods, matchingPods)
+		for _, addedPod := range addedPods {
+			changeFound = true
+			mizu.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", addedPod.Name))
+		}
+		for _, removedPod := range removedPods {
+			changeFound = true
+			mizu.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedPod.Name))
+		}
+		currentlyTappedPods = matchingPods
+	}
+
+	return nil, changeFound
+}
+
+func getPodArrayDiff(oldPods []core.Pod, newPods []core.Pod) (added []core.Pod, removed []core.Pod) {
+	added = getMissingPods(newPods, oldPods)
+	removed = getMissingPods(oldPods, newPods)
+
+	return added, removed
+}
+
+//returns pods present in pods1 array and missing in pods2 array
+func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod {
+	missingPods := make([]core.Pod, 0)
+	for _, pod1 := range pods1 {
+		var found = false
+		for _, pod2 := range pods2 {
+			if pod1.UID == pod2.UID {
+				found = true
+				break
+			}
+		}
+		if !found {
+			missingPods = append(missingPods, pod1)
+		}
+	}
+	return missingPods
+}
+
+func createProxyToApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName))
 	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, mizu.ResourcesNamespace), podExactRegex)
 	isPodReady := false
+	timeAfter := time.After(25 * time.Second)
 	for {
 		select {
+		case <-ctx.Done():
+			return
 		case <-added:
+			mizu.Log.Debugf("Got agent pod added event")
 			continue
 		case <-removed:
-			fmt.Printf("%s removed\n", mizu.AggregatorPodName)
+			mizu.Log.Infof("%s removed", mizu.ApiServerPodName)
 			cancel()
 			return
 		case modifiedPod := <-modified:
-			if modifiedPod.Status.Phase == "Running" && !isPodReady {
+			mizu.Log.Debugf("Got agent pod modified event, status phase: %v", modifiedPod.Status.Phase)
+			if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
 				isPodReady = true
 				go func() {
-					err := kubernetes.StartProxy(kubernetesProvider, tappingOptions.GuiPort, mizu.ResourcesNamespace, mizu.AggregatorPodName)
+					err := kubernetes.StartProxy(kubernetesProvider, mizu.Config.Tap.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
 					if err != nil {
-						fmt.Printf("Error occured while running k8s proxy %v\n", err)
+						mizu.Log.Errorf("Error occurred while running k8s proxy %v", err)
 						cancel()
 					}
 				}()
-				mizuProxiedUrl := kubernetes.GetMizuCollectorProxiedHostAndPath(tappingOptions.GuiPort)
-				fmt.Printf("Mizu is available at http://%s\n", mizuProxiedUrl)
-
+				mizu.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort))
 				time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
-				if tappingOptions.Analysis {
-					urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(tappingOptions.AnalysisDestination), tappingOptions.SleepIntervalSec)
-					u, err := url.ParseRequestURI(urlPath)
-
-					if err != nil {
-						log.Fatal(fmt.Sprintf("Failed parsing the URL %v\n", err))
-					}
-					rlog.Debugf("Sending get request to %v\n", u.String())
-					if response, err := http.Get(u.String()); err != nil || response.StatusCode != 200 {
-						fmt.Printf("error sending upload entries req, status code: %v, err: %v\n", response.StatusCode, err)
-					} else {
-						fmt.Printf(mizu.Purple, "Traffic is uploading to UP9 for further analsys")
-						fmt.Println()
-					}
-				}
+				requestForAnalysis()
+				reportTappedPods()
 			}
-
-		case <-time.After(25 * time.Second):
+		case <-timeAfter:
 			if !isPodReady {
-				fmt.Printf("error: %s pod was not ready in time", mizu.AggregatorPodName)
+				mizu.Log.Errorf("Error: %s pod was not ready in time", mizu.ApiServerPodName)
 				cancel()
 			}
-
 		case <-errorChan:
+			mizu.Log.Debugf("[ERROR] Agent creation, watching %v namespace", mizu.ResourcesNamespace)
 			cancel()
-
-		case <-ctx.Done():
-			return
 		}
 	}
 }
 
+func requestForAnalysis() {
+	if !mizu.Config.Tap.Analysis {
+		return
+	}
+
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.Tap.GuiPort)
+	urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(mizu.Config.Tap.AnalysisDestination), mizu.Config.Tap.SleepIntervalSec)
+	u, parseErr := url.ParseRequestURI(urlPath)
+	if parseErr != nil {
+		mizu.Log.Fatal("Failed parsing the URL (consider changing the analysis dest URL), err: %v", parseErr)
+	}
+
+	mizu.Log.Debugf("Sending get request to %v", u.String())
+	if response, requestErr := http.Get(u.String()); requestErr != nil {
+		mizu.Log.Errorf("Failed to notify agent for analysis, err: %v", requestErr)
+	} else if response.StatusCode != 200 {
+		mizu.Log.Errorf("Failed to notify agent for analysis, status code: %v", response.StatusCode)
+	} else {
+		mizu.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis")
+	}
+}
+
 func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
-	mizuRBACExists, err := kubernetesProvider.DoesMizuRBACExist(ctx, mizu.ResourcesNamespace)
+	mizuRBACExists, err := kubernetesProvider.DoesServiceAccountExist(ctx, mizu.ResourcesNamespace, mizu.ServiceAccountName)
 	if err != nil {
-		fmt.Printf("warning: could not ensure mizu rbac resources exist %v\n", err)
+		mizu.Log.Infof("warning: could not ensure mizu rbac resources exist %v", err)
 		return false
 	}
 	if !mizuRBACExists {
-		err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.ResourcesNamespace, mizu.RBACVersion)
-		if err != nil {
-			fmt.Printf("warning: could not create mizu rbac resources %v\n", err)
+		err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.ResourcesNamespace, mizu.ServiceAccountName, mizu.ClusterRoleName, mizu.ClusterRoleBindingName, mizu.RBACVersion)
+		if err != nil && !errors.IsAlreadyExists(err) {
+			mizu.Log.Infof("warning: could not create mizu rbac resources %v", err)
 			return false
 		}
 	}
@@ -326,34 +477,11 @@ func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
 	}
 }
 
-func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
-	controlSocketStr := fmt.Sprintf("ws://%s/ws", kubernetes.GetMizuCollectorProxiedHostAndPath(tappingOptions.GuiPort))
-	controlSocket, err := mizu.CreateControlSocket(controlSocketStr)
-	if err != nil {
-		fmt.Printf("error establishing control socket connection %s\n", err)
-		cancel()
-	}
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		default:
-			err = controlSocket.SendNewTappedPodsListMessage(currentlyTappedPods)
-			if err != nil {
-				rlog.Debugf("error Sending message via control socket %v, error: %s\n", controlSocketStr, err)
-			}
-			time.Sleep(10 * time.Second)
-		}
-	}
-
-}
-
-func getNamespace(tappingOptions *MizuTapOptions, kubernetesProvider *kubernetes.Provider) string {
-	if tappingOptions.AllNamespaces {
+func getNamespace(kubernetesProvider *kubernetes.Provider) string {
+	if mizu.Config.Tap.AllNamespaces {
 		return mizu.K8sAllNamespaces
-	} else if len(tappingOptions.Namespace) > 0 {
-		return tappingOptions.Namespace
+	} else if len(mizu.Config.Tap.Namespace) > 0 {
+		return mizu.Config.Tap.Namespace
 	} else {
 		return kubernetesProvider.CurrentNamespace()
 	}

cli/cmd/version.go

@@ -1,32 +1,26 @@
 package cmd
 
 import (
-	"fmt"
+	"github.com/creasty/defaults"
+	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/configStructs"
 	"strconv"
 	"time"
-
-	"github.com/spf13/cobra"
 )
 
-type MizuVersionOptions struct {
-	DebugInfo                bool
-}
-
-
-var mizuVersionOptions = &MizuVersionOptions{}
-
 var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print version info",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		if mizuVersionOptions.DebugInfo {
+		go mizu.ReportRun("version", mizu.Config.Version)
+		if mizu.Config.Version.DebugInfo {
 			timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
-			fmt.Printf("Version: %s \nBranch: %s (%s) \n", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
-			fmt.Printf("Build Time: %s (%s)\n", mizu.BuildTimestamp,  time.Unix(timeStampInt, 0))
+			mizu.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
+			mizu.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
 
 		} else {
-			fmt.Printf("Version: %s (%s)\n", mizu.SemVer, mizu.Branch)
+			mizu.Log.Infof("Version: %s (%s)", mizu.SemVer, mizu.Branch)
 		}
 		return nil
 	},
@@ -35,6 +29,9 @@ var versionCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(versionCmd)
 
-	versionCmd.Flags().BoolVarP(&mizuVersionOptions.DebugInfo, "debug", "d", false, "Provide all information about version")
+	defaultVersionConfig := configStructs.VersionConfig{}
+	defaults.Set(&defaultVersionConfig)
+
+	versionCmd.Flags().BoolP(configStructs.DebugInfoVersionName, "d", defaultVersionConfig.DebugInfo, "Provide all information about version")
 
 }

cli/cmd/view.go

@@ -1,20 +1,23 @@
 package cmd
 
 import (
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/configStructs"
 )
 
-type MizuViewOptions struct {
-	GuiPort                uint16
-}
-
-var mizuViewOptions = &MizuViewOptions{}
-
 var viewCmd = &cobra.Command{
 	Use:   "view",
 	Short: "Open GUI in browser",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		runMizuView(mizuViewOptions)
+		go mizu.ReportRun("view", mizu.Config.View)
+		if isCompatible, err := mizu.CheckVersionCompatibility(mizu.Config.View.GuiPort); err != nil {
+			return err
+		} else if !isCompatible {
+			return nil
+		}
+		runMizuView()
 		return nil
 	},
 }
@@ -22,6 +25,9 @@ var viewCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(viewCmd)
 
-	viewCmd.Flags().Uint16VarP(&mizuViewOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
+	defaultViewConfig := configStructs.ViewConfig{}
+	defaults.Set(&defaultViewConfig)
 
+	viewCmd.Flags().Uint16P(configStructs.GuiPortViewName, "p", defaultViewConfig.GuiPort, "Provide a custom port for the web interface webserver")
+	viewCmd.Flags().StringP(configStructs.KubeConfigPathViewName, "k", defaultViewConfig.KubeConfigPath, "Path to kube-config file")
 }

cli/cmd/viewRunner.go

@@ -5,35 +5,47 @@ import (
 	"fmt"
 	"github.com/up9inc/mizu/cli/kubernetes"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"k8s.io/client-go/tools/clientcmd"
 	"net/http"
 )
 
-func runMizuView(mizuViewOptions *MizuViewOptions) {
-	kubernetesProvider := kubernetes.NewProvider("")
+func runMizuView() {
+	kubernetesProvider, err := kubernetes.NewProvider(mizu.Config.View.KubeConfigPath)
+	if err != nil {
+		if clientcmd.IsEmptyConfig(err) {
+			mizu.Log.Infof("Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'")
+			return
+		}
+		if clientcmd.IsConfigurationInvalid(err) {
+			mizu.Log.Infof(uiUtils.Red, "Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'")
+			return
+		}
+	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	exists, err := kubernetesProvider.DoesServicesExist(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName)
+	exists, err := kubernetesProvider.DoesServicesExist(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName)
 	if err != nil {
 		panic(err)
 	}
 	if !exists {
-		fmt.Printf("The %s service not found\n", mizu.AggregatorPodName)
+		mizu.Log.Infof("The %s service not found", mizu.ApiServerPodName)
 		return
 	}
 
-	mizuProxiedUrl := kubernetes.GetMizuCollectorProxiedHostAndPath(mizuViewOptions.GuiPort)
+	mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.View.GuiPort)
 	_, err = http.Get(fmt.Sprintf("http://%s/", mizuProxiedUrl))
 	if err == nil {
-		fmt.Printf("Found a running service %s and open port %d\n", mizu.AggregatorPodName, mizuViewOptions.GuiPort)
+		mizu.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, mizu.Config.View.GuiPort)
 		return
 	}
-	fmt.Printf("Found service %s, creating k8s proxy\n", mizu.AggregatorPodName)
+	mizu.Log.Infof("Found service %s, creating k8s proxy", mizu.ApiServerPodName)
 
-	fmt.Printf("Mizu is available at  http://%s\n", kubernetes.GetMizuCollectorProxiedHostAndPath(mizuViewOptions.GuiPort))
-	err = kubernetes.StartProxy(kubernetesProvider, mizuViewOptions.GuiPort, mizu.ResourcesNamespace, mizu.AggregatorPodName)
+	mizu.Log.Infof("Mizu is available at  http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizu.Config.View.GuiPort))
+	err = kubernetes.StartProxy(kubernetesProvider, mizu.Config.View.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
 	if err != nil {
-		fmt.Printf("Error occured while running k8s proxy %v\n", err)
+		mizu.Log.Infof("Error occured while running k8s proxy %v", err)
 	}
 }

cli/go.mod

@@ -3,10 +3,14 @@ module github.com/up9inc/mizu/cli
 go 1.16
 
 require (
+	github.com/creasty/defaults v1.5.1
+	github.com/google/go-github/v37 v37.0.0
 	github.com/gorilla/websocket v1.4.2
-	github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.1.3
+	github.com/spf13/pflag v1.0.5
 	github.com/up9inc/mizu/shared v0.0.0
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.21.2
 	k8s.io/apimachinery v0.21.2
 	k8s.io/client-go v0.21.2

cli/go.sum

@@ -82,6 +82,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creasty/defaults v1.5.1 h1:j8WexcS3d/t4ZmllX4GEkl4wIB/trOr035ajcLHCISM=
+github.com/creasty/defaults v1.5.1/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -99,6 +101,7 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
@@ -174,6 +177,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -203,8 +207,13 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github/v37 v37.0.0 h1:rCspN8/6kB1BAJWZfuafvHhyfIo5fkAulaP/3bOQ/tM=
+github.com/google/go-github/v37 v37.0.0/go.mod h1:LM7in3NmXDrX58GbEHy7FtNLbI2JijX93RnMKvWG3m4=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -218,6 +227,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
@@ -245,6 +255,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
@@ -324,6 +335,8 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
@@ -331,6 +344,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -355,8 +369,6 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
-github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -679,8 +691,9 @@ gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -705,6 +718,7 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
 k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
+k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
 k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
 k8s.io/kubectl v0.21.2 h1:9XPCetvOMDqrIZZXb1Ei+g8t6KrIp9ENJaysQjUuLiE=
 k8s.io/kubectl v0.21.2/go.mod h1:PgeUclpG8VVmmQIl8zpLar3IQEpFc9mrmvlwY3CK1xo=

cli/kubernetes/provider.go

@@ -6,6 +6,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/up9inc/mizu/cli/mizu"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/homedir"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -14,8 +19,8 @@ import (
 	core "k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	resource "k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/watch"
 	applyconfapp "k8s.io/client-go/applyconfigurations/apps/v1"
@@ -29,7 +34,7 @@ import (
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	_ "k8s.io/client-go/tools/portforward"
-	"k8s.io/client-go/util/homedir"
+	watchtools "k8s.io/client-go/tools/watch"
 )
 
 type Provider struct {
@@ -40,15 +45,14 @@ type Provider struct {
 }
 
 const (
-	serviceAccountName = "mizu-service-account"
-	fieldManagerName   = "mizu-manager"
+	fieldManagerName = "mizu-manager"
 )
 
-func NewProvider(kubeConfigPath string) *Provider {
+func NewProvider(kubeConfigPath string) (*Provider, error) {
 	kubernetesConfig := loadKubernetesConfiguration(kubeConfigPath)
 	restClientConfig, err := kubernetesConfig.ClientConfig()
 	if err != nil {
-		panic(err.Error())
+		return nil, err
 	}
 	clientSet := getClientSet(restClientConfig)
 
@@ -56,7 +60,7 @@ func NewProvider(kubeConfigPath string) *Provider {
 		clientSet:        clientSet,
 		kubernetesConfig: kubernetesConfig,
 		clientConfig:     *restClientConfig,
-	}
+	}, nil
 }
 
 func (provider *Provider) CurrentNamespace() string {
@@ -64,6 +68,46 @@ func (provider *Provider) CurrentNamespace() string {
 	return ns
 }
 
+func (provider *Provider) WaitUtilNamespaceDeleted(ctx context.Context, name string) error {
+	fieldSelector := fmt.Sprintf("metadata.name=%s", name)
+	var limit int64 = 1
+	lw := &cache.ListWatch{
+		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+			options.FieldSelector = fieldSelector
+			options.Limit = limit
+			return provider.clientSet.CoreV1().Namespaces().List(ctx, options)
+		},
+		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+			options.FieldSelector = fieldSelector
+			options.Limit = limit
+			return provider.clientSet.CoreV1().Namespaces().Watch(ctx, options)
+		},
+	}
+
+	var preconditionFunc watchtools.PreconditionFunc = func(store cache.Store) (bool, error) {
+		_, exists, err := store.Get(&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}})
+		if err != nil {
+			return false, err
+		}
+		if exists {
+			return false, nil
+		}
+		return true, nil
+	}
+
+	conditionFunc := func(e watch.Event) (bool, error) {
+		if e.Type == watch.Deleted {
+			return true, nil
+		}
+		return false, nil
+	}
+
+	obj := &core.Namespace{}
+	_, err := watchtools.UntilWithSync(ctx, lw, obj, preconditionFunc, conditionFunc)
+
+	return err
+}
+
 func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) watch.Interface {
 	watcher, err := provider.clientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true})
 	if err != nil {
@@ -72,7 +116,16 @@ func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) w
 	return watcher
 }
 
-func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool, mizuApiFilteringOptions *shared.TrafficFilteringOptions, maxEntriesDBSizeBytes int64) (*core.Pod, error) {
+func (provider *Provider) CreateNamespace(ctx context.Context, name string) (*core.Namespace, error) {
+	namespaceSpec := &core.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+	}
+	return provider.clientSet.CoreV1().Namespaces().Create(ctx, namespaceSpec, metav1.CreateOptions{})
+}
+
+func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, namespace string, podName string, podImage string, serviceAccountName string, mizuApiFilteringOptions *shared.TrafficFilteringOptions, maxEntriesDBSizeBytes int64) (*core.Pod, error) {
 	marshaledFilteringOptions, err := json.Marshal(mizuApiFilteringOptions)
 	if err != nil {
 		return nil, err
@@ -80,19 +133,19 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace
 
 	cpuLimit, err := resource.ParseQuantity("750m")
 	if err != nil {
-		return nil, errors.New("invalid cpu limit for aggregator container")
+		return nil, errors.New(fmt.Sprintf("invalid cpu limit for %s container", podName))
 	}
 	memLimit, err := resource.ParseQuantity("512Mi")
 	if err != nil {
-		return nil, errors.New("invalid memory limit for aggregator container")
+		return nil, errors.New(fmt.Sprintf("invalid memory limit for %s container", podName))
 	}
 	cpuRequests, err := resource.ParseQuantity("50m")
 	if err != nil {
-		return nil, errors.New("invalid cpu request for aggregator container")
+		return nil, errors.New(fmt.Sprintf("invalid cpu request for %s container", podName))
 	}
 	memRequests, err := resource.ParseQuantity("50Mi")
 	if err != nil {
-		return nil, errors.New("invalid memory request for aggregator container")
+		return nil, errors.New(fmt.Sprintf("invalid memory request for %s container", podName))
 	}
 
 	pod := &core.Pod{
@@ -107,7 +160,7 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace
 					Name:            podName,
 					Image:           podImage,
 					ImagePullPolicy: core.PullAlways,
-					Command:         []string{"./mizuagent", "--aggregator"},
+					Command:         []string{"./mizuagent", "--api-server"},
 					Env: []core.EnvVar{
 						{
 							Name:  shared.HostModeEnvVar,
@@ -118,17 +171,17 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace
 							Value: string(marshaledFilteringOptions),
 						},
 						{
-							Name:  shared.MaxEntriesDBSizeByteSEnvVar,
+							Name:  shared.MaxEntriesDBSizeBytesEnvVar,
 							Value: strconv.FormatInt(maxEntriesDBSizeBytes, 10),
 						},
 					},
 					Resources: core.ResourceRequirements{
 						Limits: core.ResourceList{
-							"cpu": cpuLimit,
+							"cpu":    cpuLimit,
 							"memory": memLimit,
 						},
 						Requests: core.ResourceList{
-							"cpu": cpuRequests,
+							"cpu":    cpuRequests,
 							"memory": memRequests,
 						},
 					},
@@ -139,7 +192,7 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace
 		},
 	}
 	//define the service account only when it exists to prevent pod crash
-	if linkServiceAccount {
+	if serviceAccountName != "" {
 		pod.Spec.ServiceAccountName = serviceAccountName
 	}
 	return provider.clientSet.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
@@ -160,7 +213,7 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s
 	return provider.clientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{})
 }
 
-func (provider *Provider) DoesMizuRBACExist(ctx context.Context, namespace string) (bool, error) {
+func (provider *Provider) DoesServiceAccountExist(ctx context.Context, namespace string, serviceAccountName string) (bool, error) {
 	serviceAccount, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{})
 
 	var statusError *k8serrors.StatusError
@@ -191,9 +244,7 @@ func (provider *Provider) DoesServicesExist(ctx context.Context, namespace strin
 	return service != nil, nil
 }
 
-func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, version string) error {
-	clusterRoleName := "mizu-cluster-role"
-
+func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, serviceAccountName string, clusterRoleName string, clusterRoleBindingName string, version string) error {
 	serviceAccount := &core.ServiceAccount{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      serviceAccountName,
@@ -216,7 +267,7 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
 	}
 	clusterRoleBinding := &rbac.ClusterRoleBinding{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:   "mizu-cluster-role-binding",
+			Name:   clusterRoleBindingName,
 			Labels: map[string]string{"mizu-cli-version": version},
 		},
 		RoleRef: rbac.RoleRef{
@@ -247,9 +298,50 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
 	return nil
 }
 
+func (provider *Provider) RemoveNamespace(ctx context.Context, name string) error {
+	if isFound, err := provider.CheckNamespaceExists(ctx, name); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{})
+}
+
+func (provider *Provider) RemoveNonNamespacedResources(ctx context.Context, clusterRoleName string, clusterRoleBindingName string) error {
+	if err := provider.RemoveClusterRole(ctx, clusterRoleName); err != nil {
+		return err
+	}
+
+	if err := provider.RemoveClusterRoleBinding(ctx, clusterRoleBindingName); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (provider *Provider) RemoveClusterRole(ctx context.Context, name string) error {
+	if isFound, err := provider.CheckClusterRoleExists(ctx, name); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.RbacV1().ClusterRoles().Delete(ctx, name, metav1.DeleteOptions{})
+}
+
+func (provider *Provider) RemoveClusterRoleBinding(ctx context.Context, name string) error {
+	if isFound, err := provider.CheckClusterRoleBindingExists(ctx, name); err != nil {
+		return err
+	} else if !isFound {
+		return nil
+	}
+
+	return provider.clientSet.RbacV1().ClusterRoleBindings().Delete(ctx, name, metav1.DeleteOptions{})
+}
+
 func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) error {
-	if isFound, err := provider.CheckPodExists(ctx, namespace, podName);
-	err != nil {
+	if isFound, err := provider.CheckPodExists(ctx, namespace, podName); err != nil {
 		return err
 	} else if !isFound {
 		return nil
@@ -259,8 +351,7 @@ func (provider *Provider) RemovePod(ctx context.Context, namespace string, podNa
 }
 
 func (provider *Provider) RemoveService(ctx context.Context, namespace string, serviceName string) error {
-	if isFound, err := provider.CheckServiceExists(ctx, namespace, serviceName);
-	err != nil {
+	if isFound, err := provider.CheckServiceExists(ctx, namespace, serviceName); err != nil {
 		return err
 	} else if !isFound {
 		return nil
@@ -270,8 +361,7 @@ func (provider *Provider) RemoveService(ctx context.Context, namespace string, s
 }
 
 func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, daemonSetName string) error {
-	if isFound, err := provider.CheckDaemonSetExists(ctx, namespace, daemonSetName);
-	err != nil {
+	if isFound, err := provider.CheckDaemonSetExists(ctx, namespace, daemonSetName); err != nil {
 		return err
 	} else if !isFound {
 		return nil
@@ -280,10 +370,61 @@ func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string,
 	return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{})
 }
 
+func (provider *Provider) CheckNamespaceExists(ctx context.Context, name string) (bool, error) {
+	listOptions := metav1.ListOptions{
+		FieldSelector: fmt.Sprintf("metadata.name=%s", name),
+		Limit:         1,
+	}
+	resourceList, err := provider.clientSet.CoreV1().Namespaces().List(ctx, listOptions)
+	if err != nil {
+		return false, err
+	}
+
+	if len(resourceList.Items) > 0 {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func (provider *Provider) CheckClusterRoleExists(ctx context.Context, name string) (bool, error) {
+	listOptions := metav1.ListOptions{
+		FieldSelector: fmt.Sprintf("metadata.name=%s", name),
+		Limit:         1,
+	}
+	resourceList, err := provider.clientSet.RbacV1().ClusterRoles().List(ctx, listOptions)
+	if err != nil {
+		return false, err
+	}
+
+	if len(resourceList.Items) > 0 {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func (provider *Provider) CheckClusterRoleBindingExists(ctx context.Context, name string) (bool, error) {
+	listOptions := metav1.ListOptions{
+		FieldSelector: fmt.Sprintf("metadata.name=%s", name),
+		Limit:         1,
+	}
+	resourceList, err := provider.clientSet.RbacV1().ClusterRoleBindings().List(ctx, listOptions)
+	if err != nil {
+		return false, err
+	}
+
+	if len(resourceList.Items) > 0 {
+		return true, nil
+	}
+
+	return false, nil
+}
+
 func (provider *Provider) CheckPodExists(ctx context.Context, namespace string, name string) (bool, error) {
 	listOptions := metav1.ListOptions{
 		FieldSelector: fmt.Sprintf("metadata.name=%s", name),
-		Limit: 1,
+		Limit:         1,
 	}
 	resourceList, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, listOptions)
 	if err != nil {
@@ -300,7 +441,7 @@ func (provider *Provider) CheckPodExists(ctx context.Context, namespace string,
 func (provider *Provider) CheckServiceExists(ctx context.Context, namespace string, name string) (bool, error) {
 	listOptions := metav1.ListOptions{
 		FieldSelector: fmt.Sprintf("metadata.name=%s", name),
-		Limit: 1,
+		Limit:         1,
 	}
 	resourceList, err := provider.clientSet.CoreV1().Services(namespace).List(ctx, listOptions)
 	if err != nil {
@@ -317,7 +458,7 @@ func (provider *Provider) CheckServiceExists(ctx context.Context, namespace stri
 func (provider *Provider) CheckDaemonSetExists(ctx context.Context, namespace string, name string) (bool, error) {
 	listOptions := metav1.ListOptions{
 		FieldSelector: fmt.Sprintf("metadata.name=%s", name),
-		Limit: 1,
+		Limit:         1,
 	}
 	resourceList, err := provider.clientSet.AppsV1().DaemonSets(namespace).List(ctx, listOptions)
 	if err != nil {
@@ -331,7 +472,9 @@ func (provider *Provider) CheckDaemonSetExists(ctx context.Context, namespace st
 	return false, nil
 }
 
-func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, aggregatorPodIp string, nodeToTappedPodIPMap map[string][]string, linkServiceAccount bool, tapOutgoing bool) error {
+func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool) error {
+	mizu.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
+
 	if len(nodeToTappedPodIPMap) == 0 {
 		return fmt.Errorf("Daemon set %s must tap at least 1 pod", daemonSetName)
 	}
@@ -346,18 +489,17 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 		"-i", "any",
 		"--tap",
 		"--hardump",
-		"--aggregator-address", fmt.Sprintf("ws://%s/wsTapper", aggregatorPodIp),
+		"--api-server-address", fmt.Sprintf("ws://%s/wsTapper", apiServerPodIp),
 	}
 	if tapOutgoing {
 		mizuCmd = append(mizuCmd, "--anydirection")
 	}
 
-	privileged := true
 	agentContainer := applyconfcore.Container()
 	agentContainer.WithName(tapperPodName)
 	agentContainer.WithImage(podImage)
 	agentContainer.WithImagePullPolicy(core.PullAlways)
-	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(privileged))
+	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true))
 	agentContainer.WithCommand(mizuCmd...)
 	agentContainer.WithEnv(
 		applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"),
@@ -372,26 +514,26 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	)
 	cpuLimit, err := resource.ParseQuantity("500m")
 	if err != nil {
-		return errors.New("invalid cpu limit for tapper container")
+		return errors.New(fmt.Sprintf("invalid cpu limit for %s container", tapperPodName))
 	}
 	memLimit, err := resource.ParseQuantity("1Gi")
 	if err != nil {
-		return errors.New("invalid memory limit for tapper container")
+		return errors.New(fmt.Sprintf("invalid memory limit for %s container", tapperPodName))
 	}
 	cpuRequests, err := resource.ParseQuantity("50m")
 	if err != nil {
-		return errors.New("invalid cpu request for tapper container")
+		return errors.New(fmt.Sprintf("invalid cpu request for %s container", tapperPodName))
 	}
 	memRequests, err := resource.ParseQuantity("50Mi")
 	if err != nil {
-		return errors.New("invalid memory request for tapper container")
+		return errors.New(fmt.Sprintf("invalid memory request for %s container", tapperPodName))
 	}
 	agentResourceLimits := core.ResourceList{
-		"cpu": cpuLimit,
+		"cpu":    cpuLimit,
 		"memory": memLimit,
 	}
 	agentResourceRequests := core.ResourceList{
-		"cpu": cpuRequests,
+		"cpu":    cpuRequests,
 		"memory": memRequests,
 	}
 	agentResources := applyconfcore.ResourceRequirements().WithRequests(agentResourceRequests).WithLimits(agentResourceLimits)
@@ -425,7 +567,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	podSpec.WithHostNetwork(true)
 	podSpec.WithDNSPolicy(core.DNSClusterFirstWithHostNet)
 	podSpec.WithTerminationGracePeriodSeconds(0)
-	if linkServiceAccount {
+	if serviceAccountName != "" {
 		podSpec.WithServiceAccountName(serviceAccountName)
 	}
 	podSpec.WithContainers(agentContainer)
@@ -446,14 +588,14 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	return err
 }
 
-func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespace string) ([]core.Pod, error) {
+func (provider *Provider) GetAllRunningPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespace string) ([]core.Pod, error) {
 	pods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
 	if err != nil {
 		return nil, err
 	}
 	matchingPods := make([]core.Pod, 0)
 	for _, pod := range pods.Items {
-		if regex.MatchString(pod.Name) {
+		if regex.MatchString(pod.Name) && isPodRunning(&pod) {
 			matchingPods = append(matchingPods, pod)
 		}
 	}
@@ -469,11 +611,16 @@ func getClientSet(config *restclient.Config) *kubernetes.Clientset {
 }
 
 func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
+	if kubeConfigPath == "" {
+		kubeConfigPath = os.Getenv("KUBECONFIG")
+	}
+
 	if kubeConfigPath == "" {
 		home := homedir.HomeDir()
 		kubeConfigPath = filepath.Join(home, ".kube", "config")
 	}
 
+	mizu.Log.Debugf("Using kube config %s", kubeConfigPath)
 	configPathList := filepath.SplitList(kubeConfigPath)
 	configLoadingRules := &clientcmd.ClientConfigLoadingRules{}
 	if len(configPathList) <= 1 {
@@ -489,3 +636,7 @@ func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
 		},
 	)
 }
+
+func isPodRunning(pod *core.Pod) bool {
+	return pod.Status.Phase == core.PodRunning
+}

cli/kubernetes/proxy.go

@@ -2,6 +2,7 @@ package kubernetes
 
 import (
 	"fmt"
+	"github.com/up9inc/mizu/cli/mizu"
 	"k8s.io/kubectl/pkg/proxy"
 	"net"
 	"net/http"
@@ -13,6 +14,7 @@ const k8sProxyApiPrefix = "/"
 const mizuServicePort = 80
 
 func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace string, mizuServiceName string) error {
+	mizu.Log.Debugf("Starting proxy. namespace: [%v], service name: [%s], port: [%v]", mizuNamespace, mizuServiceName, mizuPort)
 	filter := &proxy.FilterServer{
 		AcceptPaths:   proxy.MakeRegexpArrayOrDie(proxy.DefaultPathAcceptRE),
 		RejectPaths:   proxy.MakeRegexpArrayOrDie(proxy.DefaultPathRejectRE),
@@ -40,24 +42,24 @@ func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace str
 	return server.Serve(l)
 }
 
-func getMizuCollectorProxiedHostAndPath(mizuNamespace string, mizuServiceName string) string {
+func getMizuApiServerProxiedHostAndPath(mizuNamespace string, mizuServiceName string) string {
 	return fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy/", mizuNamespace, mizuServiceName, mizuServicePort)
 }
 
-func GetMizuCollectorProxiedHostAndPath(mizuPort uint16) string {
+func GetMizuApiServerProxiedHostAndPath(mizuPort uint16) string {
 	return fmt.Sprintf("localhost:%d/mizu", mizuPort)
 }
 
 func getRerouteHttpHandlerMizuAPI(proxyHandler http.Handler, mizuNamespace string, mizuServiceName string) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		r.URL.Path = strings.Replace(r.URL.Path, "/mizu/", getMizuCollectorProxiedHostAndPath(mizuNamespace, mizuServiceName), 1)
+		r.URL.Path = strings.Replace(r.URL.Path, "/mizu/", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName), 1)
 		proxyHandler.ServeHTTP(w, r)
 	})
 }
 
 func getRerouteHttpHandlerMizuStatic(proxyHandler http.Handler, mizuNamespace string, mizuServiceName string) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		r.URL.Path = strings.Replace(r.URL.Path, "/static/", fmt.Sprintf("%s/static/", getMizuCollectorProxiedHostAndPath(mizuNamespace, mizuServiceName)), 1)
+		r.URL.Path = strings.Replace(r.URL.Path, "/static/", fmt.Sprintf("%s/static/", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName)), 1)
 		proxyHandler.ServeHTTP(w, r)
 	})
 }

cli/mizu.go

@@ -1,7 +1,11 @@
 package main
 
-import "github.com/up9inc/mizu/cli/cmd"
+import (
+	"github.com/up9inc/mizu/cli/cmd"
+	"github.com/up9inc/mizu/cli/mizu"
+)
 
 func main() {
+	mizu.InitLogger()
 	cmd.Execute()
 }

cli/mizu/config.go

@@ -0,0 +1,199 @@
+package mizu
+
+import (
+	"errors"
+	"fmt"
+	"github.com/creasty/defaults"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"gopkg.in/yaml.v3"
+	"io/ioutil"
+	"os"
+	"path"
+	"reflect"
+	"strconv"
+	"strings"
+)
+
+const (
+	Separator      = "="
+	SetCommandName = "set"
+)
+
+var Config = ConfigStruct{}
+
+func InitConfig(cmd *cobra.Command) error {
+	if err := defaults.Set(&Config); err != nil {
+		return err
+	}
+
+	if err := mergeConfigFile(); err != nil {
+		Log.Errorf("Could not load config file, error %v", err)
+		Log.Fatalf("You can regenerate the file using `mizu config -r` or just remove it %v", GetConfigFilePath())
+	}
+
+	cmd.Flags().Visit(initFlag)
+
+	finalConfigPrettified, _ := uiUtils.PrettyJson(Config)
+	Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified)
+
+	return nil
+}
+
+func GetConfigWithDefaults() (string, error) {
+	defaultConf := ConfigStruct{}
+	if err := defaults.Set(&defaultConf); err != nil {
+		return "", err
+	}
+	return uiUtils.PrettyYaml(defaultConf)
+}
+
+func GetConfigFilePath() string {
+	return path.Join(getMizuFolderPath(), "config.yaml")
+}
+
+func mergeConfigFile() error {
+	reader, openErr := os.Open(GetConfigFilePath())
+	if openErr != nil {
+		return nil
+	}
+
+	buf, readErr := ioutil.ReadAll(reader)
+	if readErr != nil {
+		return readErr
+	}
+
+	if err := yaml.Unmarshal(buf, &Config); err != nil {
+		return err
+	}
+	Log.Debugf("Found config file, merged to default options")
+
+	return nil
+}
+
+func initFlag(f *pflag.Flag) {
+	configElem := reflect.ValueOf(&Config).Elem()
+
+	sliceValue, isSliceValue := f.Value.(pflag.SliceValue)
+	if !isSliceValue {
+		mergeFlagValue(configElem, f.Name, f.Value.String())
+		return
+	}
+
+	if f.Name == SetCommandName {
+		if setError := mergeSetFlag(sliceValue.GetSlice()); setError != nil {
+			Log.Infof(uiUtils.Red, "Invalid set argument")
+		}
+		return
+	}
+
+	mergeFlagValues(configElem, f.Name, sliceValue.GetSlice())
+}
+
+func mergeSetFlag(setValues []string) error {
+	configElem := reflect.ValueOf(&Config).Elem()
+
+	for _, setValue := range setValues {
+		if !strings.Contains(setValue, Separator) {
+			return errors.New(fmt.Sprintf("invalid set argument %s", setValue))
+		}
+
+		split := strings.SplitN(setValue, Separator, 2)
+		if len(split) != 2 {
+			return errors.New(fmt.Sprintf("invalid set argument %s", setValue))
+		}
+
+		argumentKey, argumentValue := split[0], split[1]
+		mergeFlagValue(configElem, argumentKey, argumentValue)
+	}
+
+	return nil
+}
+
+func mergeFlagValue(currentElem reflect.Value, flagKey string, flagValue string) {
+	for i := 0; i < currentElem.NumField(); i++ {
+		currentField := currentElem.Type().Field(i)
+		currentFieldByName := currentElem.FieldByName(currentField.Name)
+
+		if currentField.Type.Kind() == reflect.Struct {
+			mergeFlagValue(currentFieldByName, flagKey, flagValue)
+			continue
+		}
+
+		if currentField.Tag.Get("yaml") != flagKey {
+			continue
+		}
+
+		flagValueKind := currentField.Type.Kind()
+
+		parsedValue, err := getParsedValue(flagValueKind, flagValue)
+		if err != nil {
+			Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for key %s, expected %s", flagValue, flagKey, flagValueKind))
+			return
+		}
+
+		currentFieldByName.Set(parsedValue)
+	}
+}
+
+func mergeFlagValues(currentElem reflect.Value, flagKey string, flagValues []string) {
+	for i := 0; i < currentElem.NumField(); i++ {
+		currentField := currentElem.Type().Field(i)
+		currentFieldByName := currentElem.FieldByName(currentField.Name)
+
+		if currentField.Type.Kind() == reflect.Struct {
+			mergeFlagValues(currentFieldByName, flagKey, flagValues)
+			continue
+		}
+
+		if currentField.Tag.Get("yaml") != flagKey {
+			continue
+		}
+
+		flagValueKind := currentField.Type.Elem().Kind()
+
+		parsedValues := reflect.MakeSlice(reflect.SliceOf(currentField.Type.Elem()), 0, 0)
+		for _, flagValue := range flagValues {
+			parsedValue, err := getParsedValue(flagValueKind, flagValue)
+			if err != nil {
+				Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for key %s, expected %s", flagValue, flagKey, flagValueKind))
+				return
+			}
+
+			parsedValues = reflect.Append(parsedValues, parsedValue)
+		}
+
+		currentFieldByName.Set(parsedValues)
+	}
+}
+
+func getParsedValue(kind reflect.Kind, value string) (reflect.Value, error) {
+	switch kind {
+	case reflect.String:
+		return reflect.ValueOf(value), nil
+	case reflect.Bool:
+		boolArgumentValue, err := strconv.ParseBool(value)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(boolArgumentValue), nil
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		intArgumentValue, err := strconv.ParseInt(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(intArgumentValue), nil
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		uintArgumentValue, err := strconv.ParseUint(value, 10, 64)
+		if err != nil {
+			break
+		}
+
+		return reflect.ValueOf(uintArgumentValue), nil
+	}
+
+	return reflect.ValueOf(nil), errors.New("value to parse does not match type")
+}

cli/mizu/configStruct.go

@@ -0,0 +1,19 @@
+package mizu
+
+import (
+	"fmt"
+	"github.com/up9inc/mizu/cli/mizu/configStructs"
+)
+
+type ConfigStruct struct {
+	Tap       configStructs.TapConfig     `yaml:"tap"`
+	Fetch     configStructs.FetchConfig   `yaml:"fetch"`
+	Version   configStructs.VersionConfig `yaml:"version"`
+	View      configStructs.ViewConfig    `yaml:"view"`
+	MizuImage string                      `yaml:"mizu-image"`
+	Telemetry bool                        `yaml:"telemetry" default:"true"`
+}
+
+func (config *ConfigStruct) SetDefaults() {
+	config.MizuImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", Branch, SemVer)
+}

cli/mizu/configStructs/fetchConfig.go

@@ -0,0 +1,15 @@
+package configStructs
+
+const (
+	DirectoryFetchName     = "directory"
+	FromTimestampFetchName = "from"
+	ToTimestampFetchName   = "to"
+	MizuPortFetchName      = "port"
+)
+
+type FetchConfig struct {
+	Directory     string `yaml:"directory" default:"."`
+	FromTimestamp int    `yaml:"from" default:"0"`
+	ToTimestamp   int    `yaml:"to" default:"0"`
+	MizuPort      uint16 `yaml:"port" default:"8899"`
+}

cli/mizu/configStructs/tapConfig.go

@@ -0,0 +1,78 @@
+package configStructs
+
+import (
+	"errors"
+	"fmt"
+	"github.com/up9inc/mizu/shared/units"
+	"regexp"
+	"strings"
+)
+
+const (
+	GuiPortTapName                = "gui-port"
+	NamespaceTapName              = "namespace"
+	AnalysisTapName               = "analysis"
+	AllNamespacesTapName          = "all-namespaces"
+	KubeConfigPathTapName         = "kube-config"
+	PlainTextFilterRegexesTapName = "regex-masking"
+	HideHealthChecksTapName       = "hide-healthchecks"
+	DisableRedactionTapName       = "no-redact"
+	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
+	DirectionTapName              = "direction"
+	DryRunTapName                 = "dry-run"
+)
+
+type TapConfig struct {
+	AnalysisDestination    string   `yaml:"dest" default:"up9.app"`
+	SleepIntervalSec       int      `yaml:"upload-interval" default:"10"`
+	PodRegexStr            string   `yaml:"regex" default:".*"`
+	GuiPort                uint16   `yaml:"gui-port" default:"8899"`
+	Namespace              string   `yaml:"namespace"`
+	Analysis               bool     `yaml:"analysis" default:"false"`
+	AllNamespaces          bool     `yaml:"all-namespaces" default:"false"`
+	KubeConfigPath         string   `yaml:"kube-config"`
+	PlainTextFilterRegexes []string `yaml:"regex-masking"`
+	HideHealthChecks       bool     `yaml:"hide-healthchecks" default:"false"`
+	DisableRedaction       bool     `yaml:"no-redact" default:"false"`
+	HumanMaxEntriesDBSize  string   `yaml:"max-entries-db-size" default:"200MB"`
+	Direction              string   `yaml:"direction" default:"in"`
+	DryRun                 bool     `yaml:"dry-run" default:"false"`
+}
+
+func (config *TapConfig) PodRegex() *regexp.Regexp {
+	podRegex, _ := regexp.Compile(config.PodRegexStr)
+	return podRegex
+}
+
+func (config *TapConfig) TapOutgoing() bool {
+	directionLowerCase := strings.ToLower(config.Direction)
+	if directionLowerCase == "any" {
+		return true
+	}
+
+	return false
+}
+
+func (config *TapConfig) MaxEntriesDBSizeBytes() int64 {
+	maxEntriesDBSizeBytes, _ := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
+	return maxEntriesDBSizeBytes
+}
+
+func (config *TapConfig) Validate() error {
+	_, compileErr := regexp.Compile(config.PodRegexStr)
+	if compileErr != nil {
+		return errors.New(fmt.Sprintf("%s is not a valid regex %s", config.PodRegexStr, compileErr))
+	}
+
+	_, parseHumanDataSizeErr := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
+	if parseHumanDataSizeErr != nil {
+		return errors.New(fmt.Sprintf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize))
+	}
+
+	directionLowerCase := strings.ToLower(config.Direction)
+	if directionLowerCase != "any" && directionLowerCase != "in" {
+		return errors.New(fmt.Sprintf("%s is not a valid value for flag --%s. Acceptable values are in/any.", config.Direction, DirectionTapName))
+	}
+
+	return nil
+}

cli/mizu/configStructs/versionConfig.go

@@ -0,0 +1,9 @@
+package configStructs
+
+const (
+	DebugInfoVersionName = "debug"
+)
+
+type VersionConfig struct {
+	DebugInfo bool `yaml:"debug" default:"false"`
+}

cli/mizu/configStructs/viewConfig.go

@@ -0,0 +1,11 @@
+package configStructs
+
+const (
+	GuiPortViewName        = "gui-port"
+	KubeConfigPathViewName = "kube-config"
+)
+
+type ViewConfig struct {
+	GuiPort        uint16 `yaml:"gui-port" default:"8899"`
+	KubeConfigPath string `yaml:"kube-config"`
+}

cli/mizu/consts.go

@@ -1,5 +1,10 @@
 package mizu
 
+import (
+	"os"
+	"path"
+)
+
 var (
 	SemVer         = "0.0.1"
 	Branch         = "develop"
@@ -9,20 +14,20 @@ var (
 )
 
 const (
-	ResourcesNamespace  = "default"
-	TapperDaemonSetName = "mizu-tapper-daemon-set"
-	AggregatorPodName   = "mizu-collector"
-	TapperPodName       = "mizu-tapper"
-	K8sAllNamespaces    = ""
+	ApiServerPodName       = "mizu-api-server"
+	ClusterRoleBindingName = "mizu-cluster-role-binding"
+	ClusterRoleName        = "mizu-cluster-role"
+	K8sAllNamespaces       = ""
+	ResourcesNamespace     = "mizu"
+	ServiceAccountName     = "mizu-service-account"
+	TapperDaemonSetName    = "mizu-tapper-daemon-set"
+	TapperPodName          = "mizu-tapper"
 )
 
-const (
-	Black        = "\033[1;30m%s\033[0m"
-	Red          = "\033[1;31m%s\033[0m"
-	Green        = "\033[1;32m%s\033[0m"
-	Yellow       = "\033[1;33m%s\033[0m"
-	Purple       = "\033[1;34m%s\033[0m"
-	Magenta      = "\033[1;35m%s\033[0m"
-	Teal         = "\033[1;36m%s\033[0m"
-	White        = "\033[1;37m%s\033[0m"
-)
+func getMizuFolderPath() string {
+	home, homeDirErr := os.UserHomeDir()
+	if homeDirErr != nil {
+		return ""
+	}
+	return path.Join(home, ".mizu")
+}

cli/mizu/logger.go

@@ -0,0 +1,39 @@
+package mizu
+
+import (
+	"fmt"
+	"github.com/op/go-logging"
+	"os"
+	"path"
+)
+
+var Log = logging.MustGetLogger("mizu_cli")
+
+var format = logging.MustStringFormatter(
+	`%{time} %{level:.5s} ▶ %{pid} %{shortfile} %{shortfunc} ▶ %{message}`,
+)
+
+func InitLogger() {
+	mizuDirPath := getMizuFolderPath()
+	if err := os.MkdirAll(mizuDirPath, os.ModePerm); err != nil {
+		panic(fmt.Sprintf("Failed creating mizu dir: %v, err %v", mizuDirPath, err))
+	}
+	logPath := path.Join(mizuDirPath, "log.log")
+	f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		panic(fmt.Sprintf("Failed mizu log file: %v, err %v", logPath, err))
+	}
+
+	fileLog := logging.NewLogBackend(f, "", 0)
+	consoleLog := logging.NewLogBackend(os.Stderr, "", 0)
+
+	backend2Formatter := logging.NewBackendFormatter(fileLog, format)
+
+	backend1Leveled := logging.AddModuleLevel(consoleLog)
+	backend1Leveled.SetLevel(logging.INFO, "")
+
+	logging.SetBackend(backend1Leveled, backend2Formatter)
+
+	Log.Debugf("\n\n\n")
+	Log.Debugf("Running mizu version %v", SemVer)
+}

cli/mizu/telemetry.go

@@ -0,0 +1,36 @@
+package mizu
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
+
+func ReportRun(cmd string, args interface{}) {
+	if !Config.Telemetry {
+		Log.Debugf("not reporting due to config value")
+		return
+	}
+
+	argsBytes, _ := json.Marshal(args)
+	argsMap := map[string]string{
+		"telemetry_type": "execution",
+		"cmd":            cmd,
+		"args":           string(argsBytes),
+		"component":      "mizu_cli",
+		"BuildTimestamp": BuildTimestamp,
+		"Branch":         Branch,
+		"version":        SemVer}
+	argsMap["message"] = fmt.Sprintf("mizu %v - %v", argsMap["cmd"], string(argsBytes))
+
+	jsonValue, _ := json.Marshal(argsMap)
+
+	if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+		Log.Debugf("error sending telemetry err: %v, response %v", err, resp)
+	} else {
+		Log.Debugf("Successfully reported telemetry")
+	}
+}

cli/mizu/versionCheck.go

@@ -0,0 +1,92 @@
+package mizu
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/google/go-github/v37/github"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/semver"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+func getApiVersion(port uint16) (string, error) {
+	versionUrl, _ := url.Parse(fmt.Sprintf("http://localhost:%d/mizu/metadata/version", port))
+	req := &http.Request{
+		Method: http.MethodGet,
+		URL:    versionUrl,
+	}
+	statusResp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer statusResp.Body.Close()
+
+	versionResponse := &shared.VersionResponse{}
+	if err := json.NewDecoder(statusResp.Body).Decode(&versionResponse); err != nil {
+		return "", err
+	}
+
+	return versionResponse.SemVer, nil
+}
+
+func CheckVersionCompatibility(port uint16) (bool, error) {
+	apiSemVer, err := getApiVersion(port)
+	if err != nil {
+		return false, err
+	}
+
+	if semver.SemVersion(apiSemVer).Major() == semver.SemVersion(SemVer).Major() &&
+		semver.SemVersion(apiSemVer).Minor() == semver.SemVersion(SemVer).Minor() {
+		return true, nil
+	}
+
+	Log.Infof(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", SemVer, apiSemVer))
+	return false, nil
+}
+
+func CheckNewerVersion() {
+	Log.Debugf("Checking for newer version...")
+	start := time.Now()
+	client := github.NewClient(nil)
+	latestRelease, _, err := client.Repositories.GetLatestRelease(context.Background(), "up9inc", "mizu")
+	if err != nil {
+		Log.Debugf("[ERROR] Failed to get latest release")
+		return
+	}
+
+	versionFileUrl := ""
+	for _, asset := range latestRelease.Assets {
+		if *asset.Name == "version.txt" {
+			versionFileUrl = *asset.BrowserDownloadURL
+			break
+		}
+	}
+	if versionFileUrl == "" {
+		Log.Debugf("[ERROR] Version file not found in the latest release")
+		return
+	}
+
+	res, err := http.Get(versionFileUrl)
+	if err != nil {
+		Log.Debugf("[ERROR] Failed to get the version file %v", err)
+		return
+	}
+
+	data, err := ioutil.ReadAll(res.Body)
+	res.Body.Close()
+	if err != nil {
+		Log.Debugf("[ERROR] Failed to read the version file -> %v", err)
+		return
+	}
+	gitHubVersion := string(data)
+	gitHubVersion = gitHubVersion[:len(gitHubVersion)-1]
+	Log.Debugf("Finished version validation, took %v", time.Since(start))
+	if SemVer < gitHubVersion {
+		Log.Infof(uiUtils.Yellow, fmt.Sprintf("Update available! %v -> %v (%v)", SemVer, gitHubVersion, *latestRelease.HTMLURL))
+	}
+}

cli/uiUtils/colors.go

@@ -0,0 +1,13 @@
+package uiUtils
+
+
+const (
+	Black        = "\033[1;30m%s\033[0m"
+	Red          = "\033[1;31m%s\033[0m"
+	Green        = "\033[1;32m%s\033[0m"
+	Yellow       = "\033[1;33m%s\033[0m"
+	Purple       = "\033[1;34m%s\033[0m"
+	Magenta      = "\033[1;35m%s\033[0m"
+	Teal         = "\033[1;36m%s\033[0m"
+	White        = "\033[1;37m%s\033[0m"
+)

cli/uiUtils/confirmation.go

@@ -3,7 +3,6 @@ package uiUtils
 import (
 	"bufio"
 	"fmt"
-	"github.com/up9inc/mizu/cli/mizu"
 	"log"
 	"os"
 	"strings"
@@ -12,7 +11,7 @@ import (
 func AskForConfirmation(s string) bool {
 	reader := bufio.NewReader(os.Stdin)
 
-	fmt.Printf(mizu.Magenta, s)
+	fmt.Printf(Magenta, s)
 
 	response, err := reader.ReadString('\n')
 	if err != nil {

cli/uiUtils/prettyString.go

@@ -0,0 +1,36 @@
+package uiUtils
+
+import (
+	"bytes"
+	"encoding/json"
+	"gopkg.in/yaml.v3"
+)
+
+const (
+	empty = ""
+	tab   = "\t"
+)
+
+func PrettyJson(data interface{}) (string, error) {
+	buffer := new(bytes.Buffer)
+	encoder := json.NewEncoder(buffer)
+	encoder.SetIndent(empty, tab)
+
+	err := encoder.Encode(data)
+	if err != nil {
+		return empty, err
+	}
+	return buffer.String(), nil
+}
+
+func PrettyYaml(data interface{}) (string, error) {
+	buffer := new(bytes.Buffer)
+	encoder := yaml.NewEncoder(buffer)
+	encoder.SetIndent(0)
+
+	err := encoder.Encode(data)
+	if err != nil {
+		return empty, err
+	}
+	return buffer.String(), nil
+}

debug.Dockerfile

@@ -1,4 +1,4 @@
-# creates image in which mizu api is remotely debuggable using delve
+# creates image in which mizu agent is remotely debuggable using delve
 FROM node:14-slim AS site-build
 
 WORKDIR /app/ui-build
@@ -14,18 +14,21 @@ ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 
 RUN apk add libpcap-dev gcc g++ make
 
-# Move to api working directory (/api-build).
-WORKDIR /app/api-build
+# Move to agent working directory (/agent-build).
+WORKDIR /app/agent-build
 
-COPY api/go.mod api/go.sum ./
+COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
+COPY tap/go.mod tap/go.mod ../tap/
+
 RUN go mod download
 # cheap trick to make the build faster (As long as go.mod wasn't changes)
 RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
 
-# Copy and build api code
+# Copy and build agent code
 COPY shared ../shared
-COPY api .
+COPY tap ../tap
+COPY agent .
 RUN go build -gcflags="all=-N -l" -o mizuagent .
 
 
@@ -35,10 +38,11 @@ RUN apk add bash libpcap-dev tcpdump
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
-COPY --from=builder ["/app/api-build/mizuagent", "."]
+COPY --from=builder ["/app/agent-build/mizuagent", "."]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 
 # install remote debugging tool
 RUN go get github.com/go-delve/delve/cmd/dlv
 
-CMD ["sh", "-c", "dlv --headless=true --listen=:2345 --log --api-version=2 --accept-multiclient exec ./mizuagent -- --aggregator"]
+ENTRYPOINT "/app/mizuagent"
+#CMD ["sh", "-c", "dlv --headless=true --listen=:2345 --log --api-version=2 --accept-multiclient exec ./mizuagent -- --api-server"]

examples/roles/permissions-all-namespaces-without-ip-resolution.yaml

@@ -0,0 +1,35 @@
+# This example shows the roles required for a user to be able to use Mizu in all namespaces with IP resolution disabled.
+# (Traffic will be recorded, but Mizu will not translate IP addresses to names)
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-clusterrole
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["list", "watch", "create"]
+- apiGroups: [""]
+  resources: ["services"]
+  verbs: ["create"]
+- apiGroups: ["apps"]
+  resources: ["daemonsets"]
+  verbs: ["create", "patch"]
+- apiGroups: [""]
+  resources: ["namespaces"]
+  verbs: ["list", "watch", "create", "delete"]
+- apiGroups: [""]
+  resources: ["services/proxy"]
+  verbs: ["get"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-clusterrolebindings
+subjects:
+- kind: User
+  name: user1
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: mizu-runner-clusterrole
+  apiGroup: rbac.authorization.k8s.io

examples/roles/permissions-all-namespaces.yaml

@@ -0,0 +1,52 @@
+# This example shows the roles required for a user to be able to use Mizu in all namespaces.
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-clusterrole
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch", "create"]
+- apiGroups: [""]
+  resources: ["services"]
+  verbs: ["get", "list", "watch", "create"]
+- apiGroups: ["apps"]
+  resources: ["daemonsets"]
+  verbs: ["create", "patch"]
+- apiGroups: [""]
+  resources: ["namespaces"]
+  verbs: ["list", "watch", "create", "delete"]
+- apiGroups: [""]
+  resources: ["services/proxy"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["serviceaccounts"]
+  verbs: ["get", "create"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["clusterroles"]
+  verbs: ["list", "create", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["clusterrolebindings"]
+  verbs: ["list", "create", "delete"]
+- apiGroups: ["apps", "extensions"]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["apps", "extensions"]
+  resources: ["services"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["", "apps", "extensions"]
+  resources: ["endpoints"]
+  verbs: ["get", "list", "watch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-clusterrolebindings
+subjects:
+- kind: User
+  name: user1
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: mizu-runner-clusterrole
+  apiGroup: rbac.authorization.k8s.io

shared/consts.go

@@ -5,5 +5,5 @@ const (
 	HostModeEnvVar                   = "HOST_MODE"
 	NodeNameEnvVar                   = "NODE_NAME"
 	TappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST"
-	MaxEntriesDBSizeByteSEnvVar      = "MAX_ENTRIES_DB_BYTES"
+	MaxEntriesDBSizeBytesEnvVar      = "MAX_ENTRIES_DB_BYTES"
 )

shared/models.go

@@ -7,6 +7,7 @@ const (
 	WebSocketMessageTypeTappedEntry   WebSocketMessageType = "tappedEntry"
 	WebSocketMessageTypeUpdateStatus  WebSocketMessageType = "status"
 	WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus"
+	WebsocketMessageTypeOutboundLink  WebSocketMessageType = "outboundLink"
 )
 
 type WebSocketMessageMetadata struct {
@@ -31,7 +32,8 @@ type WebSocketStatusMessage struct {
 }
 
 type TapStatus struct {
-	Pods []PodInfo `json:"pods"`
+	Pods     []PodInfo `json:"pods"`
+	TLSLinks []TLSLinkInfo `json:"tlsLinks"`
 }
 
 type PodInfo struct {
@@ -39,6 +41,13 @@ type PodInfo struct {
 	Name      string `json:"name"`
 }
 
+type TLSLinkInfo struct {
+	SourceIP                string `json:"sourceIP"`
+	DestinationAddress      string `json:"destinationAddress"`
+	ResolvedDestinationName string `json:"resolvedDestinationName"`
+	ResolvedSourceName      string `json:"resolvedSourceName"`
+}
+
 func CreateWebSocketStatusMessage(tappingStatus TapStatus) WebSocketStatusMessage {
 	return WebSocketStatusMessage{
 		WebSocketMessageMetadata: &WebSocketMessageMetadata{
@@ -60,4 +69,10 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc
 type TrafficFilteringOptions struct {
 	PlainTextMaskingRegexes []*SerializableRegexp
 	HideHealthChecks        bool
+	DisableRedaction        bool
 }
+
+type VersionResponse struct {
+	SemVer string `json:"semver"`
+}
+

shared/semver/semver.go

@@ -0,0 +1,28 @@
+package semver
+
+import (
+	"regexp"
+)
+
+type SemVersion string
+
+func (v SemVersion) Breakdown() (string, string, string) {
+	re := regexp.MustCompile(`\d+`)
+	breakdown := re.FindAllString(string(v), 3)
+	return breakdown[0], breakdown[1], breakdown[2]
+}
+
+func (v SemVersion) Major() string {
+	major, _, _ := v.Breakdown()
+	return major
+}
+
+func (v SemVersion) Minor() string {
+	_, minor, _ := v.Breakdown()
+	return minor
+}
+
+func (v SemVersion) Patch() string {
+	_, _, patch := v.Breakdown()
+	return patch
+}

tap/cleaner.go

@@ -1,6 +1,7 @@
 package tap
 
 import (
+	"github.com/romana/rlog"
 	"sync"
 	"time"
 
@@ -20,19 +21,21 @@ type Cleaner struct {
 	cleanPeriod       time.Duration
 	connectionTimeout time.Duration
 	stats             CleanerStats
-	statsMutex	  sync.Mutex
+	statsMutex        sync.Mutex
 }
 
 func (cl *Cleaner) clean() {
 	startCleanTime := time.Now()
 
 	cl.assemblerMutex.Lock()
+	rlog.Debugf("Assembler Stats before cleaning %s", cl.assembler.Dump())
 	flushed, closed := cl.assembler.FlushCloseOlderThan(startCleanTime.Add(-cl.connectionTimeout))
 	cl.assemblerMutex.Unlock()
 
 	deleted := cl.matcher.deleteOlderThan(startCleanTime.Add(-cl.connectionTimeout))
 
 	cl.statsMutex.Lock()
+	rlog.Debugf("Assembler Stats after cleaning %s", cl.assembler.Dump())
 	cl.stats.flushed += flushed
 	cl.stats.closed += closed
 	cl.stats.deleted += deleted
@@ -55,7 +58,7 @@ func (cl *Cleaner) dumpStats() CleanerStats {
 
 	stats := CleanerStats{
 		flushed: cl.stats.flushed,
-		closed : cl.stats.closed,
+		closed:  cl.stats.closed,
 		deleted: cl.stats.deleted,
 	}
 

tap/go.mod

@@ -8,4 +8,5 @@ require (
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
 	golang.org/x/net v0.0.0-20210421230115-4e50805a0758
+	github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4
 )

tap/go.sum

@@ -1,3 +1,5 @@
+github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M=
+github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=

tap/har_writer.go

@@ -1,9 +1,11 @@
 package tap
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -46,12 +48,27 @@ type HarFile struct {
 }
 
 func NewEntry(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time) (*har.Entry, error) {
-	harRequest, err := har.NewRequest(request, true)
+	harRequest, err := har.NewRequest(request, false)
 	if err != nil {
 		SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err)
 		return nil, errors.New("Failed converting request to HAR")
 	}
 
+	// For requests with multipart/form-data or application/x-www-form-urlencoded Content-Type,
+	// martian/har will parse the request body and place the parameters in harRequest.PostData.Params
+	// instead of harRequest.PostData.Text (as the HAR spec requires it).
+	// Mizu currently only looks at PostData.Text. Therefore, instead of letting martian/har set the content of
+	// PostData, always copy the request body to PostData.Text.
+	if (request.ContentLength > 0) {
+		reqBody, err := ioutil.ReadAll(request.Body)
+		if err != nil {
+			SilentError("read-request-body", "Failed converting request to HAR %s (%v,%+v)", err, err, err)
+			return nil, errors.New("Failed reading request body")
+		}
+		request.Body = ioutil.NopCloser(bytes.NewReader(reqBody))
+		harRequest.PostData.Text = string(reqBody)
+	}
+
 	harResponse, err := har.NewResponse(response, true)
 	if err != nil {
 		SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err)

tap/http_reader.go

@@ -5,21 +5,25 @@ import (
 	"bytes"
 	"encoding/hex"
 	"fmt"
+	"github.com/bradleyfalzon/tlsx"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strconv"
 	"sync"
 	"time"
 )
 
+const checkTLSPacketAmount = 100
+
 type httpReaderDataMsg struct {
 	bytes     []byte
 	timestamp time.Time
 }
 
 type tcpID struct {
-	srcIP string
-	dstIP string
+	srcIP   string
+	dstIP   string
 	srcPort string
 	dstPort string
 }
@@ -42,28 +46,44 @@ func (tid *tcpID) String() string {
  * Implements io.Reader interface (Read)
  */
 type httpReader struct {
-	ident         string
-	tcpID         tcpID
-	isClient      bool
-	isHTTP2       bool
-	isOutgoing    bool
-	msgQueue      chan httpReaderDataMsg // Channel of captured reassembled tcp payload
-	data          []byte
-	captureTime   time.Time
-	hexdump       bool
-	parent        *tcpStream
-	grpcAssembler GrpcAssembler
-	messageCount  uint
-	harWriter     *HarWriter
+	ident              string
+	tcpID              tcpID
+	isClient           bool
+	isHTTP2            bool
+	isOutgoing         bool
+	msgQueue           chan httpReaderDataMsg // Channel of captured reassembled tcp payload
+	data               []byte
+	captureTime        time.Time
+	hexdump            bool
+	parent             *tcpStream
+	grpcAssembler      GrpcAssembler
+	messageCount       uint
+	harWriter          *HarWriter
+	packetsSeen        uint
+	outboundLinkWriter *OutboundLinkWriter
 }
 
 func (h *httpReader) Read(p []byte) (int, error) {
 	var msg httpReaderDataMsg
+
 	ok := true
 	for ok && len(h.data) == 0 {
 		msg, ok = <-h.msgQueue
 		h.data = msg.bytes
+
 		h.captureTime = msg.timestamp
+		if len(h.data) > 0 {
+			h.packetsSeen += 1
+		}
+		if h.packetsSeen < checkTLSPacketAmount && len(msg.bytes) > 5 { // packets with less than 5 bytes cause tlsx to panic
+			clientHello := tlsx.ClientHello{}
+			err := clientHello.Unmarshall(msg.bytes)
+			if err == nil {
+				fmt.Printf("Detected TLS client hello with SNI %s\n", clientHello.SNI)
+				numericPort, _ := strconv.Atoi(h.tcpID.dstPort)
+				h.outboundLinkWriter.WriteOutboundLink(h.tcpID.srcIP, h.tcpID.dstIP, numericPort, clientHello.SNI, TLSProtocol)
+			}
+		}
 	}
 	if !ok || len(h.data) == 0 {
 		return 0, io.EOF

tap/outboundlinks.go

@@ -1,9 +1,17 @@
 package tap
 
+type OutboundLinkProtocol string
+
+const (
+	TLSProtocol OutboundLinkProtocol = "tls"
+)
+
 type OutboundLink struct {
 	Src     string
 	DstIP   string
 	DstPort int
+	SuggestedResolvedName string
+	SuggestedProtocol OutboundLinkProtocol
 }
 
 func NewOutboundLinkWriter() *OutboundLinkWriter {
@@ -16,11 +24,13 @@ type OutboundLinkWriter struct {
 	OutChan chan *OutboundLink
 }
 
-func (olw *OutboundLinkWriter) WriteOutboundLink(src string, DstIP string, DstPort int) {
+func (olw *OutboundLinkWriter) WriteOutboundLink(src string, DstIP string, DstPort int, SuggestedResolvedName string, SuggestedProtocol OutboundLinkProtocol) {
 	olw.OutChan <- &OutboundLink{
 		Src: src,
 		DstIP: DstIP,
 		DstPort: DstPort,
+		SuggestedResolvedName: SuggestedResolvedName,
+		SuggestedProtocol: SuggestedProtocol,
 	}
 }
 

tap/passive_tapper.go

@@ -33,10 +33,10 @@ import (
 
 const AppPortsEnvVar = "APP_PORTS"
 const maxHTTP2DataLenEnvVar = "HTTP2_DATA_SIZE_LIMIT"
-// default is 1MB, more than the max size accepted by collector and traffic-dumper
-const maxHTTP2DataLenDefault = 1 * 1024 * 1024
+const maxHTTP2DataLenDefault = 1 * 1024 * 1024 // 1MB
 const cleanPeriod = time.Second * 10
-var remoteOnlyOutboundPorts = []int { 80, 443 }
+
+var remoteOnlyOutboundPorts = []int{80, 443}
 
 func parseAppPorts(appPortsList string) []int {
 	ports := make([]int, 0)
@@ -51,15 +51,15 @@ func parseAppPorts(appPortsList string) []int {
 	return ports
 }
 
-var maxcount = flag.Int("c", -1, "Only grab this many packets, then exit")
+var maxcount = flag.Int64("c", -1, "Only grab this many packets, then exit")
 var decoder = flag.String("decoder", "", "Name of the decoder to use (default: guess from capture)")
 var statsevery = flag.Int("stats", 60, "Output statistics every N seconds")
 var lazy = flag.Bool("lazy", false, "If true, do lazy decoding")
 var nodefrag = flag.Bool("nodefrag", false, "If true, do not do IPv4 defrag")
-var checksum = flag.Bool("checksum", false, "Check TCP checksum")  // global
-var nooptcheck = flag.Bool("nooptcheck", true, "Do not check TCP options (useful to ignore MSS on captures with TSO)")  // global
-var ignorefsmerr = flag.Bool("ignorefsmerr", true, "Ignore TCP FSM errors")  // global
-var allowmissinginit = flag.Bool("allowmissinginit", true, "Support streams without SYN/SYN+ACK/ACK sequence")  // global
+var checksum = flag.Bool("checksum", false, "Check TCP checksum")                                                      // global
+var nooptcheck = flag.Bool("nooptcheck", true, "Do not check TCP options (useful to ignore MSS on captures with TSO)") // global
+var ignorefsmerr = flag.Bool("ignorefsmerr", true, "Ignore TCP FSM errors")                                            // global
+var allowmissinginit = flag.Bool("allowmissinginit", true, "Support streams without SYN/SYN+ACK/ACK sequence")         // global
 var verbose = flag.Bool("verbose", false, "Be verbose")
 var debug = flag.Bool("debug", false, "Display debug information")
 var quiet = flag.Bool("quiet", false, "Be quiet regarding errors")
@@ -69,7 +69,7 @@ var nohttp = flag.Bool("nohttp", false, "Disable HTTP parsing")
 var output = flag.String("output", "", "Path to create file for HTTP 200 OK responses")
 var writeincomplete = flag.Bool("writeincomplete", false, "Write incomplete response")
 
-var hexdump = flag.Bool("dump", false, "Dump HTTP request/response as hex")  // global
+var hexdump = flag.Bool("dump", false, "Dump HTTP request/response as hex") // global
 var hexdumppkt = flag.Bool("dumppkt", false, "Dump packet as hex")
 
 // capture
@@ -88,7 +88,7 @@ var dumpToHar = flag.Bool("hardump", false, "Dump traffic to har files")
 var HarOutputDir = flag.String("hardir", "", "Directory in which to store output har files")
 var harEntriesPerFile = flag.Int("harentriesperfile", 200, "Number of max number of har entries to store in each file")
 
-var reqResMatcher = createResponseRequestMatcher()  // global
+var reqResMatcher = createResponseRequestMatcher() // global
 var statsTracker = StatsTracker{}
 
 // global
@@ -118,8 +118,8 @@ var outputLevel int
 var errorsMap map[string]uint
 var errorsMapMutex sync.Mutex
 var nErrors uint
-var ownIps []string           // global
-var hostMode bool             // global
+var ownIps []string // global
+var hostMode bool   // global
 
 /* minOutputLevel: Error will be printed only if outputLevel is above this value
  * t:              key for errorsMap (counting errors)
@@ -175,6 +175,10 @@ type Context struct {
 	CaptureInfo gopacket.CaptureInfo
 }
 
+func GetStats() AppStats {
+	return statsTracker.appStats
+}
+
 func (c *Context) GetCaptureInfo() gopacket.CaptureInfo {
 	return c.CaptureInfo
 }
@@ -197,6 +201,37 @@ func StartPassiveTapper(opts *TapOpts) (<-chan *OutputChannelItem, <-chan *Outbo
 	return nil, outboundLinkWriter.OutChan
 }
 
+func startMemoryProfiler() {
+	dirname := "/app/pprof"
+	rlog.Info("Profiling is on, results will be written to %s", dirname)
+	go func() {
+		if _, err := os.Stat(dirname); os.IsNotExist(err) {
+			if err := os.Mkdir(dirname, 0777); err != nil {
+				log.Fatal("could not create directory for profile: ", err)
+			}
+		}
+
+		for true {
+			t := time.Now()
+
+			filename := fmt.Sprintf("%s/%s__mem.prof", dirname, t.Format("15_04_05"))
+
+			rlog.Info("Writing memory profile to %s\n", filename)
+
+			f, err := os.Create(filename)
+			if err != nil {
+				log.Fatal("could not create memory profile: ", err)
+			}
+			runtime.GC() // get up-to-date statistics
+			if err := pprof.WriteHeapProfile(f); err != nil {
+				log.Fatal("could not write memory profile: ", err)
+			}
+			_ = f.Close()
+			time.Sleep(time.Minute)
+		}
+	}()
+}
+
 func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWriter) {
 	log.SetFlags(log.LstdFlags | log.LUTC | log.Lshortfile)
 
@@ -305,19 +340,23 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 	source.Lazy = *lazy
 	source.NoCopy = true
 	rlog.Info("Starting to read packets")
-	count := 0
-	bytes := int64(0)
-	start := time.Now()
+	statsTracker.setStartTime(time.Now())
 	defragger := ip4defrag.NewIPv4Defragmenter()
 
 	streamFactory := &tcpStreamFactory{
-		doHTTP: !*nohttp,
-		harWriter: harWriter,
+		doHTTP:             !*nohttp,
+		harWriter:          harWriter,
 		outbountLinkWriter: outboundLinkWriter,
-
 	}
 	streamPool := reassembly.NewStreamPool(streamFactory)
 	assembler := reassembly.NewAssembler(streamPool)
+
+	maxBufferedPagesTotal := GetMaxBufferedPagesPerConnection()
+	maxBufferedPagesPerConnection := GetMaxBufferedPagesTotal()
+	rlog.Infof("Assembler options: maxBufferedPagesTotal=%d, maxBufferedPagesPerConnection=%d", maxBufferedPagesTotal, maxBufferedPagesPerConnection)
+	assembler.AssemblerOptions.MaxBufferedPagesTotal = maxBufferedPagesTotal
+	assembler.AssemblerOptions.MaxBufferedPagesPerConnection = maxBufferedPagesPerConnection
+
 	var assemblerMutex sync.Mutex
 
 	signalChan := make(chan os.Signal, 1)
@@ -325,10 +364,10 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 
 	staleConnectionTimeout := time.Second * time.Duration(*staleTimeoutSeconds)
 	cleaner := Cleaner{
-		assembler: assembler,
-		assemblerMutex: &assemblerMutex,
-		matcher: &reqResMatcher,
-		cleanPeriod: cleanPeriod,
+		assembler:         assembler,
+		assemblerMutex:    &assemblerMutex,
+		matcher:           &reqResMatcher,
+		cleanPeriod:       cleanPeriod,
 		connectionTimeout: staleConnectionTimeout,
 	}
 	cleaner.start()
@@ -346,9 +385,9 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 			errorsSummery := fmt.Sprintf("%v", errorsMap)
 			errorsMapMutex.Unlock()
 			log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v) - Errors Summary: %s",
-				count,
-				bytes,
-				time.Since(start),
+				statsTracker.appStats.TotalPacketsCount,
+				statsTracker.appStats.TotalProcessedBytes,
+				time.Since(statsTracker.appStats.StartTime),
 				nErrors,
 				errorMapLen,
 				errorsSummery,
@@ -366,22 +405,26 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 
 			// Since the last print
 			cleanStats := cleaner.dumpStats()
-			appStats := statsTracker.dumpStats()
+			matchedMessages := statsTracker.dumpStats()
 			log.Printf(
 				"flushed connections %d, closed connections: %d, deleted messages: %d, matched messages: %d",
 				cleanStats.flushed,
 				cleanStats.closed,
 				cleanStats.deleted,
-				appStats.matchedMessages,
+				matchedMessages,
 			)
 		}
 	}()
 
+	if GetMemoryProfilingEnabled() {
+		startMemoryProfiler()
+	}
+
 	for packet := range source.Packets() {
-		count++
-		rlog.Debugf("PACKET #%d", count)
+		packetsCount := statsTracker.incPacketsCount()
+		rlog.Debugf("PACKET #%d", packetsCount)
 		data := packet.Data()
-		bytes += int64(len(data))
+		statsTracker.updateProcessedSize(int64(len(data)))
 		if *hexdumppkt {
 			rlog.Debugf("Packet content (%d/0x%x) - %s", len(data), len(data), hex.Dump(data))
 		}
@@ -432,12 +475,17 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 			assemblerMutex.Unlock()
 		}
 
-		done := *maxcount > 0 && count >= *maxcount
+		done := *maxcount > 0 && statsTracker.appStats.TotalPacketsCount >= *maxcount
 		if done {
 			errorsMapMutex.Lock()
 			errorMapLen := len(errorsMap)
 			errorsMapMutex.Unlock()
-			log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)", count, bytes, time.Since(start), nErrors, errorMapLen)
+			log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)",
+				statsTracker.appStats.TotalPacketsCount,
+				statsTracker.appStats.TotalProcessedBytes,
+				time.Since(statsTracker.appStats.StartTime),
+				nErrors,
+				errorMapLen)
 		}
 		select {
 		case <-signalChan:
@@ -494,4 +542,5 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
 	for e := range errorsMap {
 		log.Printf(" %s:\t\t%d", e, errorsMap[e])
 	}
+	log.Printf("AppStats: %v", GetStats())
 }

tap/settings.go

@@ -1,5 +1,18 @@
 package tap
 
+import (
+	"os"
+	"strconv"
+)
+
+const (
+	MemoryProfilingEnabledEnvVarName          = "MEMORY_PROFILING_ENABLED"
+	MaxBufferedPagesTotalEnvVarName           = "MAX_BUFFERED_PAGES_TOTAL"
+	MaxBufferedPagesPerConnectionEnvVarName   = "MAX_BUFFERED_PAGES_PER_CONNECTION"
+	MaxBufferedPagesTotalDefaultValue         = 5000
+	MaxBufferedPagesPerConnectionDefaultValue = 5000
+)
+
 type globalSettings struct {
 	filterPorts       []int
 	filterAuthorities []string
@@ -29,3 +42,23 @@ func GetFilterIPs() []string {
 	copy(addresses, gSettings.filterAuthorities)
 	return addresses
 }
+
+func GetMaxBufferedPagesTotal() int {
+	valueFromEnv, err := strconv.Atoi(os.Getenv(MaxBufferedPagesTotalEnvVarName))
+	if err != nil {
+		return MaxBufferedPagesTotalDefaultValue
+	}
+	return valueFromEnv
+}
+
+func GetMaxBufferedPagesPerConnection() int {
+	valueFromEnv, err := strconv.Atoi(os.Getenv(MaxBufferedPagesPerConnectionEnvVarName))
+	if err != nil {
+		return MaxBufferedPagesPerConnectionDefaultValue
+	}
+	return valueFromEnv
+}
+
+func GetMemoryProfilingEnabled() bool {
+	return os.Getenv(MemoryProfilingEnabledEnvVarName) == "1"
+}

tap/stats_tracker.go

@@ -2,34 +2,54 @@ package tap
 
 import (
 	"sync"
+	"time"
 )
 
 type AppStats struct {
-	matchedMessages int
+	StartTime            time.Time `json:"startTime"`
+	MatchedMessages      int       `json:"matchedMessages"`
+	TotalPacketsCount    int64     `json:"totalPacketsCount"`
+	TotalProcessedBytes  int64     `json:"totalProcessedBytes"`
+	TotalMatchedMessages int64     `json:"totalMatchedMessages"`
 }
 
 type StatsTracker struct {
-	stats             AppStats
-	statsMutex	  sync.Mutex
+	appStats                AppStats
+	matchedMessagesMutex    sync.Mutex
+	totalPacketsCountMutex  sync.Mutex
+	totalProcessedSizeMutex sync.Mutex
 }
 
 func (st *StatsTracker) incMatchedMessages() {
-	st.statsMutex.Lock()
-	st.stats.matchedMessages++
-	st.statsMutex.Unlock()
+	st.matchedMessagesMutex.Lock()
+	st.appStats.MatchedMessages++
+	st.appStats.TotalMatchedMessages++
+	st.matchedMessagesMutex.Unlock()
 }
 
-func (st *StatsTracker) dumpStats() AppStats {
-	st.statsMutex.Lock()
-
-	stats := AppStats{
-		matchedMessages: st.stats.matchedMessages,
-	}
-
-	st.stats.matchedMessages = 0
-
-	st.statsMutex.Unlock()
-
-	return stats
+func (st *StatsTracker) incPacketsCount() int64 {
+	st.totalPacketsCountMutex.Lock()
+	st.appStats.TotalPacketsCount++
+	currentPacketsCount := st.appStats.TotalPacketsCount
+	st.totalPacketsCountMutex.Unlock()
+	return currentPacketsCount
 }
 
+func (st *StatsTracker) updateProcessedSize(size int64) {
+	st.totalProcessedSizeMutex.Lock()
+	st.appStats.TotalProcessedBytes += size
+	st.totalProcessedSizeMutex.Unlock()
+}
+
+func (st *StatsTracker) setStartTime(startTime time.Time) {
+	st.appStats.StartTime = startTime
+}
+
+func (st *StatsTracker) dumpStats() int {
+	st.matchedMessagesMutex.Lock()
+	matchedMessages := st.appStats.MatchedMessages
+	st.appStats.MatchedMessages = 0
+	st.matchedMessagesMutex.Unlock()
+
+	return matchedMessages
+}

tap/tcp_stream_factory.go

@@ -33,7 +33,7 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
 	dstPort := int(tcp.DstPort)
 
 	if factory.shouldNotifyOnOutboundLink(dstIp, dstPort) {
-		factory.outbountLinkWriter.WriteOutboundLink(net.Src().String(), dstIp, dstPort)
+		factory.outbountLinkWriter.WriteOutboundLink(net.Src().String(), dstIp, dstPort, "", "")
 	}
 	props := factory.getStreamProps(srcIp, dstIp, dstPort)
 	isHTTP := props.isTapTarget
@@ -57,11 +57,12 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
 				srcPort: transport.Src().String(),
 				dstPort: transport.Dst().String(),
 			},
-			hexdump:  *hexdump,
-			parent:   stream,
-			isClient: true,
-			isOutgoing: props.isOutgoing,
-			harWriter: factory.harWriter,
+			hexdump:            *hexdump,
+			parent:             stream,
+			isClient:           true,
+			isOutgoing:         props.isOutgoing,
+			harWriter:          factory.harWriter,
+			outboundLinkWriter: factory.outbountLinkWriter,
 		}
 		stream.server = httpReader{
 			msgQueue: make(chan httpReaderDataMsg),
@@ -72,10 +73,11 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
 				srcPort: transport.Dst().String(),
 				dstPort: transport.Src().String(),
 			},
-			hexdump: *hexdump,
-			parent:  stream,
-			isOutgoing: props.isOutgoing,
-			harWriter: factory.harWriter,
+			hexdump:            *hexdump,
+			parent:             stream,
+			isOutgoing:         props.isOutgoing,
+			harWriter:          factory.harWriter,
+			outboundLinkWriter: factory.outbountLinkWriter,
 		}
 		factory.wg.Add(2)
 		// Start reading from channels stream.client.bytes and stream.server.bytes
@@ -131,6 +133,5 @@ func (factory *tcpStreamFactory) shouldNotifyOnOutboundLink(dstIP string, dstPor
 
 type streamProps struct {
 	isTapTarget bool
-	isOutgoing bool
+	isOutgoing  bool
 }
-

ui/package.json

@@ -4,6 +4,7 @@
   "private": true,
   "dependencies": {
     "@material-ui/core": "^4.11.3",
+    "@material-ui/lab": "^4.0.0-alpha.60",
     "@testing-library/jest-dom": "^5.11.10",
     "@testing-library/react": "^11.2.6",
     "@testing-library/user-event": "^12.8.3",
@@ -11,6 +12,7 @@
     "@types/node": "^12.20.10",
     "@types/react": "^17.0.3",
     "@types/react-dom": "^17.0.3",
+    "axios": "^0.21.1",
     "node-sass": "^5.0.0",
     "numeral": "^2.0.6",
     "protobuf-decoder": "^0.1.0",

ui/src/App.tsx

@@ -1,10 +1,12 @@
-import React, {useState} from 'react';
+import React, {useEffect, useState} from 'react';
 import './App.sass';
 import logo from './components/assets/Mizu-logo.svg';
-import {Button} from "@material-ui/core";
+import {Button, Snackbar} from "@material-ui/core";
 import {HarPage} from "./components/HarPage";
 import Tooltip from "./components/Tooltip";
 import {makeStyles} from "@material-ui/core/styles";
+import MuiAlert from '@material-ui/lab/Alert';
+import Api from "./helpers/api";
 
 
 const useStyles = makeStyles(() => ({
@@ -15,11 +17,37 @@ const useStyles = makeStyles(() => ({
     },
 }));
 
+const api = new Api();
+
 const App = () => {
 
     const classes = useStyles();
 
     const [analyzeStatus, setAnalyzeStatus] = useState(null);
+    const [showTLSWarning, setShowTLSWarning] = useState(false);
+    const [userDismissedTLSWarning, setUserDismissedTLSWarning] = useState(false);
+    const [addressesWithTLS, setAddressesWithTLS] = useState(new Set());
+
+    useEffect(() => {
+        (async () => {
+            const recentTLSLinks = await api.getRecentTLSLinks();
+
+            if (recentTLSLinks?.length > 0) {
+                setAddressesWithTLS(new Set([...addressesWithTLS, ...recentTLSLinks]));
+                setShowTLSWarning(true);
+            }
+
+        })();
+    }, []);
+
+    const onTLSDetected = (destAddress: string) => {
+        addressesWithTLS.add(destAddress);
+        setAddressesWithTLS(new Set(addressesWithTLS));
+
+        if (!userDismissedTLSWarning) {
+          setShowTLSWarning(true);
+        }
+    };
 
     const analysisMessage = analyzeStatus?.isRemoteReady ?
         <span>
@@ -88,7 +116,12 @@ const App = () => {
                 </Tooltip>
                 }
             </div>
-            <HarPage setAnalyzeStatus={setAnalyzeStatus}/>
+            <HarPage setAnalyzeStatus={setAnalyzeStatus} onTLSDetected={onTLSDetected}/>
+            <Snackbar open={showTLSWarning && !userDismissedTLSWarning}>
+                <MuiAlert elevation={6} variant="filled" onClose={() => setUserDismissedTLSWarning(true)} severity="warning">
+                    Mizu is detecting TLS traffic{addressesWithTLS.size ? ` (directed to ${Array.from(addressesWithTLS).join(", ")})` : ''}, this type of traffic will not be displayed.
+                </MuiAlert>
+            </Snackbar>
         </div>
     );
 }