Added build ui to pr validation flow (#655)

* initiate recoil state management with entPage and tappingStatus

* first recoil selector

* insert entries and focusedEntryId into recoil

* ws connection, entry data

* manage query by recoil

* identifier for cypress

* conflicts fix

* css fix

* cr fixes

Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>

.github/workflows/acceptance_tests.yml

@@ -30,3 +30,15 @@ jobs:
 
       - name: Test
         run: make acceptance-test
+
+      - name: Slack notification on failure
+        uses: ravsamhq/notify-slack-action@v1
+        if: always()
+        with:
+          status: ${{ job.status }}
+          notification_title: 'Mizu {workflow} has {status_message}'
+          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit: <{commit_url}|{commit_sha}>'
+          footer: 'Linked Repo <{repo_url}|{repo}>'
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/inactive-issues-close.yaml

@@ -0,0 +1,22 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v3
+        with:
+          days-before-issue-stale: 30
+          days-before-issue-close: 14
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
+          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pr_validation.yml

@@ -44,3 +44,18 @@ jobs:
 
       - name: Build Agent
         run: make agent
+
+  build-ui:
+    name: Build UI
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Node 14
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Build UI
+        run: make ui

.github/workflows/security_validation.yml

@@ -1,25 +0,0 @@
-name: Security validation
-
-on:
-  pull_request:
-    branches:
-      - 'develop'
-      - 'main'
-
-jobs:
-  security:
-    name: Check for vulnerabilities
-    runs-on: ubuntu-latest
-    env:
-      SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: snyk/actions/setup@master
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.16'
-
-      - name: Run snyl on all projects
-        run: snyk test --all-projects

.gitignore

@@ -15,7 +15,6 @@
 # vendor/
 .idea/
 build
-*.db
 
 # Mac OS
 .DS_Store
@@ -29,3 +28,19 @@ build
 
 # pprof
 pprof/*
+
+# Database Files
+*.bin
+*.gob
+
+# Nohup Files - https://man7.org/linux/man-pages/man1/nohup.1p.html
+nohup.*
+
+# Cypress tests
+cypress.env.json
+*/cypress/downloads
+*/cypress/fixtures
+*/cypress/plugins
+*/cypress/screenshots
+*/cypress/videos
+*/cypress/support

Dockerfile

@@ -13,7 +13,7 @@ FROM golang:1.16-alpine AS builder
 # Set necessary environment variables needed for our image.
 ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 
-RUN apk add libpcap-dev gcc g++ make bash
+RUN apk add libpcap-dev gcc g++ make bash perl-utils
 
 # Move to agent working directory (/agent-build).
 WORKDIR /app/agent-build
@@ -24,7 +24,7 @@ COPY tap/go.mod tap/go.mod ../tap/
 COPY tap/api/go.* ../tap/api/
 RUN go mod download
 # cheap trick to make the build faster (As long as go.mod wasn't changes)
-RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
+RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
 
 ARG COMMIT_HASH
 ARG GIT_BRANCH
@@ -44,9 +44,10 @@ RUN go build -ldflags="-s -w \
 COPY devops/build_extensions.sh ..
 RUN cd .. && /bin/bash build_extensions.sh
 
-FROM alpine:3.14
+FROM alpine:3.15
+
+RUN apk add bash libpcap-dev
 
-RUN apk add bash libpcap-dev tcpdump
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui agent cli tap docker
+.PHONY: help ui extensions extensions-debug agent agent-debug cli tap docker
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -28,6 +28,9 @@ ui: ## Build UI.
 cli: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build
 
+cli-debug: ## Build CLI.
+	@echo "building cli"; cd cli && $(MAKE) build-debug
+
 build-cli-ci: ## Build CLI for CI.
 	@echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci
 
@@ -37,6 +40,12 @@ agent: ## Build agent.
 	${MAKE} extensions
 	@ls -l agent/build
 
+agent-debug: ## Build agent for debug.
+	@(echo "building mizu agent for debug.." )
+	@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
+	${MAKE} extensions-debug
+	@ls -l agent/build
+
 docker: ## Build and publish agent docker image.
 	$(MAKE) push-docker
 
@@ -62,7 +71,7 @@ push-cli: ## Build and publish CLI.
 	gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
-clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
+clean: clean-ui clean-agent clean-cli clean-docker clean-extensions ## Clean all build artifacts.
 
 clean-ui: ## Clean UI.
 	@(rm -rf ui/build ; echo "UI cleanup done" )
@@ -73,9 +82,15 @@ clean-agent: ## Clean agent.
 clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
+clean-extensions:  ## Clean extensions
+	@(rm -rf tap/extensions/*.so ; echo "Extensions cleanup done" )
+
 clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
+extensions-debug:
+	devops/build_extensions_debug.sh
+
 extensions:
 	devops/build_extensions.sh
 

README.md

@@ -1,19 +1,36 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
 
+<p align="center">
+    <a href="https://github.com/up9inc/mizu/releases/latest">
+        <img alt="GitHub Latest Release" src="https://img.shields.io/github/v/release/up9inc/mizu?logo=GitHub&style=flat-square">
+    </a>
+    <a href="https://github.com/up9inc/mizu/blob/main/LICENSE">
+        <img alt="GitHub License" src="https://img.shields.io/github/license/up9inc/mizu?logo=GitHub&style=flat-square">
+    </a>
+    <a href="https://join.slack.com/t/up9/shared_invite/zt-tfjnduli-QzlR8VV4Z1w3YnPIAJfhlQ">
+      <img alt="Slack" src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social">
+    </a>
+</p>
+
 # The API Traffic Viewer for Kubernetes
 
 A simple-yet-powerful API traffic viewer for Kubernetes enabling you to view all API communication between microservices to help your debug and troubleshoot regressions.
 
-Think TCPDump and Chrome Dev Tools combined.
+Think TCPDump and Wireshark re-invented for Kubernetes.
 
 ![Simple UI](assets/mizu-ui.png)
 
 ## Features
 
 - Simple and powerful CLI
-- Real-time view of all HTTP requests, REST and gRPC API calls
-- No installation or code instrumentation
-- Works completely on premises
+- Monitoring network traffic in real-time. Supported protocols:
+  - [HTTP/1.1](https://datatracker.ietf.org/doc/html/rfc2616) (REST, etc.)
+  - [HTTP/2](https://datatracker.ietf.org/doc/html/rfc7540) (gRPC)
+  - [AMQP](https://www.rabbitmq.com/amqp-0-9-1-reference.html) (RabbitMQ, Apache Qpid, etc.)
+  - [Apache Kafka](https://kafka.apache.org/protocol)
+  - [Redis](https://redis.io/topics/protocol)
+- Works with Kubernetes APIs. No installation or code instrumentation
+- Rich filtering
 
 ## Requirements
 
@@ -44,20 +61,11 @@ SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/
 ### Development (unstable) Build
 Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page
 
-## Kubeconfig & Permissions
-While `mizu`most often works out of the box, you can influence its behavior:
-
-1. [OPTIONAL] Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
-2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination)
-
-For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document
-
-
 ## How to Run
 
 1. Find pods you'd like to tap to in your Kubernetes cluster
-2. Run `mizu tap` or `mizu tap PODNAME`  
-3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI
+2. Run `mizu tap` or `mizu tap PODNAME`
+3. Open browser on `http://localhost:8899` **or** as instructed in the CLI
 4. Watch the API traffic flowing
 5. Type ^C to stop
 
@@ -83,7 +91,7 @@ To tap all pods in current namespace -
 ```
 
 
-To tap specific pod - 
+### To tap specific pod
 ```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
@@ -96,7 +104,7 @@ To tap specific pod -
  ^C
 ```
 
-To tap multiple pods using regex - 
+### To tap multiple pods using regex
 ```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
@@ -114,21 +122,22 @@ To tap multiple pods using regex -
 
 ## Configuration
 
-Mizu can work with config file which should be stored in ${HOME}/.mizu/config.yaml (macOS: ~/.mizu/config.yaml) <br />
-In case no config file found, defaults will be used <br />
+Mizu can optionally work with a config file that can be provided as a CLI argument (using `--set config-path=<PATH>`) or if not provided, will be stored at ${HOME}/.mizu/config.yaml 
 In case of partial configuration defined, all other fields will be used with defaults <br />
 You can always override the defaults or config file with CLI flags
 
 To get the default config params run `mizu config` <br />
 To generate a new config file with default values use `mizu config -r`
 
-### Telemetry
-
-By default, mizu reports usage telemetry. It can be disabled by adding a line of `telemetry: false` in the `${HOME}/.mizu/config.yaml` file
-
 
 ## Advanced Usage
 
+### Kubeconfig
+
+It is possible to change the kubeconfig path using `KUBECONFIG` environment variable or the command like flag
+with `--set kube-config-path=<PATH>`. </br >
+If both are not set - Mizu assumes that configuration is at `${HOME}/.kube/config`
+
 ### Namespace-Restricted Mode
 
 Some users have permission to only manage resources in one particular namespace assigned to them
@@ -142,6 +151,8 @@ using the `--namespace` flag or by setting `tap.namespaces` in the config file
 
 Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
 
+For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document
+
 ### User agent filtering
 
 User-agent filtering (like health checks) - can be configured using command-line options:
@@ -173,30 +184,14 @@ Please see [CONTRACT MONITORING](docs/CONTRACT_MONITORING.md) page for more deta
 
 ### Configure proxy host 
 
-By default, mizu will be accessible via local host: 'http://localhost:8899/mizu/', it is possible to change the host,
-for instance, to '0.0.0.0' which can grant access via machine IP address.
-This setting can be changed via command line flag `--set tap.proxy-host=<value>` or via config file:
-tap
-    proxy-host: 0.0.0.0
-and when changed it will support accessing by IP
+By default, mizu will be accessible via local host: 'http://localhost:8899', it is possible to change the host, for
+instance, to '0.0.0.0' which can grant access via machine IP address. This setting can be changed via command line
+flag `--set tap.proxy-host=<value>` or via config file:
+tap proxy-host: 0.0.0.0 and when changed it will support accessing by IP
 
-### Run in daemon mode
+### Install Mizu standalone
 
-Mizu can be ran detached from the cli using the daemon flag: `mizu tap --daemon`. This type of mizu instance will run indefinitely in the cluster.
+Mizu can be run detached from the cli using the install command: `mizu install`. This type of mizu instance will run
+indefinitely in the cluster.
 
-In order to access a daemon mizu you will have to run `mizu view` after running the `tap --daemon` command.
-
-To stop the detached mizu instance and clean all cluster side resources, run `mizu clean`
-
-
-## How to Run local UI
-
-- run from mizu/agent `go run main.go --hars-read --hars-dir <folder>`
-
-- copy Har files into the folder from last command
-
-- change `MizuWebsocketURL` and `apiURL` in `api.js` file
-
-- run from mizu/ui - `npm run start`
-
-- open browser on `localhost:3000`
+For more information please refer to [INSTALL STANDALONE](docs/INSTALL_STANDALONE.md)

acceptanceTests/Makefile

@@ -1,2 +1,2 @@
 test: ## Run acceptance tests.
-	@go test ./... -timeout 1h
+	@go test ./... -timeout 1h -v

acceptanceTests/config_test.go

@@ -2,11 +2,12 @@ package acceptanceTests
 
 import (
 	"fmt"
-	"gopkg.in/yaml.v3"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"testing"
+
+	"gopkg.in/yaml.v3"
 )
 
 type tapConfig struct {

acceptanceTests/cypress.json

@@ -0,0 +1,17 @@
+{
+  "watchForFileChanges":false,
+  "viewportWidth": 1920,
+  "viewportHeight": 1080,
+  "video": false,
+  "screenshotOnRunFailure": false,
+
+  "testFiles":
+  ["tests/GuiPort.js",
+  "tests/MultipleNamespaces.js",
+  "tests/RedactTests.js",
+  "tests/Regex.js"],
+
+  "env": {
+    "testUrl": "http://localhost:8899/"
+  }
+}

acceptanceTests/cypress/integration/page_objects/StatusBar.js

@@ -0,0 +1,46 @@
+const columns = {podName : 1, namespace : 2, tapping : 3};
+const greenStatusImageSrc = '/static/media/success.662997eb.svg';
+
+function getDomPathInStatusBar(line, column) {
+    return `.expandedStatusBar > :nth-child(2) > > :nth-child(2) > :nth-child(${line}) > :nth-child(${column})`;
+}
+
+export function checkLine(line, expectedValues) {
+    cy.get(getDomPathInStatusBar(line, columns.podName)).invoke('text').then(podValue => {
+        const podName = getOnlyPodName(podValue);
+        expect(podName).to.equal(expectedValues.podName);
+
+        cy.get(getDomPathInStatusBar(line, columns.namespace)).invoke('text').then(namespaceValue => {
+            expect(namespaceValue).to.equal(expectedValues.namespace);
+            cy.get(getDomPathInStatusBar(line, columns.tapping)).children().should('have.attr', 'src', greenStatusImageSrc);
+        });
+    });
+}
+
+export function findLineAndCheck(expectedValues) {
+    cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(1)').then(pods => {
+        cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(2)').then(namespaces => {
+            // organizing namespaces array
+            const podObjectsArray = Object.values(pods ?? {});
+            const namespacesObjectsArray = Object.values(namespaces ?? {});
+            let lineNumber = -1;
+            namespacesObjectsArray.forEach((namespaceObj, index) => {
+                const currentLine = index + 1;
+                lineNumber = (namespaceObj.getAttribute && namespaceObj.innerHTML === expectedValues.namespace && (getOnlyPodName(podObjectsArray[index].innerHTML)) === expectedValues.podName) ? currentLine : lineNumber;
+            });
+            lineNumber === -1 ? throwError(expectedValues) : checkLine(lineNumber, expectedValues);
+        });
+    });
+}
+
+function throwError(expectedValues) {
+    throw new Error(`The pod named ${expectedValues.podName} doesn't match any namespace named ${expectedValues.namespace}`);
+}
+
+export function getExpectedDetailsDict(podName, namespace) {
+    return {podName : podName, namespace : namespace};
+}
+
+function getOnlyPodName(podElementFullStr) {
+    return podElementFullStr.substring(0, podElementFullStr.indexOf('-'));
+}

acceptanceTests/cypress/integration/tests/GuiPort.js

@@ -0,0 +1,8 @@
+it('check', function () {
+    cy.visit(`http://localhost:${Cypress.env('port')}/`);
+
+    cy.get('.header').should('be.visible');
+    cy.get('.TrafficPageHeader').should('be.visible');
+    cy.get('.TrafficPage-ListContainer').should('be.visible');
+    cy.get('.TrafficPage-Container').should('be.visible');
+});

acceptanceTests/cypress/integration/tests/MultipleNamespaces.js

@@ -0,0 +1,18 @@
+import {findLineAndCheck, getExpectedDetailsDict} from '../page_objects/StatusBar';
+
+it('opening', function () {
+    cy.visit(Cypress.env('testUrl'));
+    cy.get('.podsCount').trigger('mouseover');
+});
+
+[1, 2, 3].map(doItFunc);
+
+function doItFunc(number) {
+    const podName = Cypress.env(`name${number}`);
+    const namespace = Cypress.env(`namespace${number}`);
+
+    it(`verifying the pod (${podName}, ${namespace})`, function () {
+        findLineAndCheck(getExpectedDetailsDict(podName, namespace));
+    });
+}
+

acceptanceTests/cypress/integration/tests/RedactTests.js

@@ -0,0 +1,23 @@
+const inHeader = 'User-Header[REDACTED]';
+const inBody = '{ "User": "[REDACTED]" }';
+const shouldExist = Cypress.env('shouldExist');
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+})
+
+it(`should ${shouldExist ? '' : 'not'} include ${inHeader}`, function () {
+    cy.get('.CollapsibleContainer', { timeout : 15 * 1000}).first().next().then(headerElements => { //TODO change the path and refactor the body and head functions
+        const allText = headerElements.text();
+        if (allText.includes(inHeader) !== shouldExist)
+            throw new Error(`The headers panel doesnt include ${inHeader}`);
+    });
+});
+
+it(`should ${shouldExist ? '' : 'not'} include ${inBody}`, function () {
+    cy.get('.hljs').then(bodyElement => {
+        const line = bodyElement.text();
+        if (line.includes(inBody) !== shouldExist)
+            throw new Error(`The body panel doesnt include ${inBody}`);
+    });
+});

acceptanceTests/cypress/integration/tests/Regex.js

@@ -0,0 +1,11 @@
+import {getExpectedDetailsDict, checkLine} from '../page_objects/StatusBar';
+
+
+it('opening', function () {
+    cy.visit(Cypress.env('testUrl'));
+    cy.get('.podsCount').trigger('mouseover');
+
+    cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) >').should('have.length', 1); // one line
+
+    checkLine(1, getExpectedDetailsDict(Cypress.env('name'), Cypress.env('namespace')));
+});

acceptanceTests/go.mod

@@ -3,6 +3,7 @@ module github.com/up9inc/mizu/tests
 go 1.16
 
 require (
+	github.com/gorilla/websocket v1.4.2
 	github.com/up9inc/mizu/shared v0.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

acceptanceTests/go.sum

@@ -75,6 +75,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
@@ -110,6 +111,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
 github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
@@ -156,6 +158,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0=
 github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
@@ -191,10 +194,11 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -211,6 +215,7 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -242,6 +247,7 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@@ -287,8 +293,10 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
@@ -303,6 +311,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -312,6 +321,7 @@ github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -355,6 +365,7 @@ github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJ
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
@@ -364,6 +375,7 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
@@ -460,6 +472,7 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -519,6 +532,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -568,6 +582,7 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
@@ -628,6 +643,7 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8X
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
@@ -639,6 +655,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
@@ -651,7 +668,9 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y=
 k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU=
+k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc=
 k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM=
 k8s.io/cli-runtime v0.21.2/go.mod h1:8u/jFcM0QpoI28f6sfrAAIslLCXUYKD5SsPPMWiHYrI=
 k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA=
@@ -662,6 +681,7 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
 k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
 k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
 k8s.io/kubectl v0.21.2/go.mod h1:PgeUclpG8VVmmQIl8zpLar3IQEpFc9mrmvlwY3CK1xo=
@@ -675,5 +695,7 @@ sigs.k8s.io/kustomize/cmd/config v0.9.10/go.mod h1:Mrby0WnRH7hA6OwOYnYpfpiY0WJIM
 sigs.k8s.io/kustomize/kustomize/v4 v4.1.2/go.mod h1:PxBvo4WGYlCLeRPL+ziT64wBXqbgfcalOS/SXa/tcyo=
 sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4cPcfUVt5Hg=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

acceptanceTests/logs_test.go

@@ -81,11 +81,16 @@ func TestLogs(t *testing.T) {
 		logsFileNames = append(logsFileNames, file.Name)
 	}
 
-	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
+	if !Contains(logsFileNames, "mizu.mizu-api-server.mizu-api-server.log") {
 		t.Errorf("api server logs not found")
 		return
 	}
 
+	if !Contains(logsFileNames, "mizu.mizu-api-server.basenine.log") {
+		t.Errorf("basenine logs not found")
+		return
+	}
+
 	if !Contains(logsFileNames, "mizu_cli.log") {
 		t.Errorf("cli logs not found")
 		return
@@ -174,11 +179,16 @@ func TestLogsPath(t *testing.T) {
 		logsFileNames = append(logsFileNames, file.Name)
 	}
 
-	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
+	if !Contains(logsFileNames, "mizu.mizu-api-server.mizu-api-server.log") {
 		t.Errorf("api server logs not found")
 		return
 	}
 
+	if !Contains(logsFileNames, "mizu.mizu-api-server.basenine.log") {
+		t.Errorf("basenine logs not found")
+		return
+	}
+
 	if !Contains(logsFileNames, "mizu_cli.log") {
 		t.Errorf("cli logs not found")
 		return

acceptanceTests/tap_test.go

@@ -3,7 +3,6 @@ package acceptanceTests
 import (
 	"archive/zip"
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -66,21 +65,18 @@ func TestTap(t *testing.T) {
 			entriesCheckFunc := func() error {
 				timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-				entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, entriesCount, timestamp)
-				requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-				if requestErr != nil {
-					return fmt.Errorf("failed to get entries, err: %v", requestErr)
+				entries, err := getDBEntries(timestamp, entriesCount, 1*time.Second)
+				if err != nil {
+					return err
 				}
-
-				entries := requestResult.([]interface{})
-				if len(entries) == 0 {
-					return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+				err = checkEntriesAtLeast(entries, 1)
+				if err != nil {
+					return err
 				}
-
-				entry :=  entries[0].(map[string]interface{})
+				entry := entries[0]
 
 				entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"])
-				requestResult, requestErr = executeHttpGetRequest(entryUrl)
+				requestResult, requestErr := executeHttpGetRequest(entryUrl)
 				if requestErr != nil {
 					return fmt.Errorf("failed to get entry, err: %v", requestErr)
 				}
@@ -141,6 +137,8 @@ func TestTapGuiPort(t *testing.T) {
 				t.Errorf("failed to start tap pods on time, err: %v", err)
 				return
 			}
+
+			runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env port=%d", guiPort))
 		})
 	}
 }
@@ -150,11 +148,9 @@ func TestTapAllNamespaces(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	expectedPods := []struct{
-		Name      string
-		Namespace string
-	}{
+	expectedPods := []PodDescriptor{
 		{Name: "httpbin", Namespace: "mizu-tests"},
+		{Name: "httpbin2", Namespace: "mizu-tests"},
 		{Name: "httpbin", Namespace: "mizu-tests2"},
 	}
 
@@ -188,37 +184,8 @@ func TestTapAllNamespaces(t *testing.T) {
 		return
 	}
 
-	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
-	requestResult, requestErr := executeHttpGetRequest(podsUrl)
-	if requestErr != nil {
-		t.Errorf("failed to get tap status, err: %v", requestErr)
-		return
-	}
-
-	pods, err := getPods(requestResult)
-	if err != nil {
-		t.Errorf("failed to get pods, err: %v", err)
-		return
-	}
-
-	for _, expectedPod := range expectedPods {
-		podFound := false
-
-		for _, pod := range pods {
-			podNamespace :=  pod["namespace"].(string)
-			podName := pod["name"].(string)
-
-			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
-				podFound = true
-				break
-			}
-		}
-
-		if !podFound {
-			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
-			return
-		}
-	}
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
+		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
 }
 
 func TestTapMultipleNamespaces(t *testing.T) {
@@ -226,10 +193,7 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	expectedPods := []struct{
-		Name      string
-		Namespace string
-	}{
+	expectedPods := []PodDescriptor{
 		{Name: "httpbin", Namespace: "mizu-tests"},
 		{Name: "httpbin2", Namespace: "mizu-tests"},
 		{Name: "httpbin", Namespace: "mizu-tests2"},
@@ -269,42 +233,8 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		return
 	}
 
-	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
-	requestResult, requestErr := executeHttpGetRequest(podsUrl)
-	if requestErr != nil {
-		t.Errorf("failed to get tap status, err: %v", requestErr)
-		return
-	}
-
-	pods, err := getPods(requestResult)
-	if err != nil {
-		t.Errorf("failed to get pods, err: %v", err)
-		return
-	}
-
-	if len(expectedPods) != len(pods) {
-		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
-		return
-	}
-
-	for _, expectedPod := range expectedPods {
-		podFound := false
-
-		for _, pod := range pods {
-			podNamespace :=  pod["namespace"].(string)
-			podName := pod["name"].(string)
-
-			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
-				podFound = true
-				break
-			}
-		}
-
-		if !podFound {
-			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
-			return
-		}
-	}
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
+		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
 }
 
 func TestTapRegex(t *testing.T) {
@@ -313,10 +243,7 @@ func TestTapRegex(t *testing.T) {
 	}
 
 	regexPodName := "httpbin2"
-	expectedPods := []struct{
-		Name      string
-		Namespace string
-	}{
+	expectedPods := []PodDescriptor{
 		{Name: regexPodName, Namespace: "mizu-tests"},
 	}
 
@@ -352,42 +279,8 @@ func TestTapRegex(t *testing.T) {
 		return
 	}
 
-	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
-	requestResult, requestErr := executeHttpGetRequest(podsUrl)
-	if requestErr != nil {
-		t.Errorf("failed to get tap status, err: %v", requestErr)
-		return
-	}
-
-	pods, err := getPods(requestResult)
-	if err != nil {
-		t.Errorf("failed to get pods, err: %v", err)
-		return
-	}
-
-	if len(expectedPods) != len(pods) {
-		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
-		return
-	}
-
-	for _, expectedPod := range expectedPods {
-		podFound := false
-
-		for _, pod := range pods {
-			podNamespace :=  pod["namespace"].(string)
-			podName := pod["name"].(string)
-
-			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
-				podFound = true
-				break
-			}
-		}
-
-		if !podFound {
-			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
-			return
-		}
-	}
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Regex.js\" --env name=%v,namespace=%v",
+		expectedPods[0].Name, expectedPods[0].Namespace))
 }
 
 func TestTapDryRun(t *testing.T) {
@@ -431,7 +324,7 @@ func TestTapDryRun(t *testing.T) {
 		resultChannel <- "fail"
 	}()
 
-	testResult := <- resultChannel
+	testResult := <-resultChannel
 	if testResult != "success" {
 		t.Errorf("unexpected result - dry run cmd not done")
 	}
@@ -475,79 +368,16 @@ func TestTapRedact(t *testing.T) {
 	}
 
 	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+	requestHeaders := map[string]string{"User-Header": "Mizu"}
 	requestBody := map[string]string{"User": "Mizu"}
 	for i := 0; i < defaultEntriesCount; i++ {
-		if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil {
+		if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	redactCheckFunc := func() error {
-		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
-
-		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
-		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entries, err: %v", requestErr)
-		}
-
-		entries := requestResult.([]interface{})
-		if len(entries) == 0 {
-			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
-		}
-
-		firstEntry :=  entries[0].(map[string]interface{})
-
-		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr = executeHttpGetRequest(entryUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entry, err: %v", requestErr)
-		}
-
-		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		entryJson := data["entry"].(string)
-
-		var entry map[string]interface{}
-		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
-			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
-		}
-
-		entryRequest := entry["request"].(map[string]interface{})
-		entryPayload := entryRequest["payload"].(map[string]interface{})
-		entryDetails := entryPayload["details"].(map[string]interface{})
-
-		headers :=  entryDetails["headers"].([]interface{})
-		for _, headerInterface := range headers {
-			header := headerInterface.(map[string]interface{})
-			if header["name"].(string) != "User-Agent" {
-				continue
-			}
-
-			userAgent := header["value"].(string)
-			if userAgent != "[REDACTED]" {
-				return fmt.Errorf("unexpected result - user agent is not redacted")
-			}
-		}
-
-		postData := entryDetails["postData"].(map[string]interface{})
-		textDataStr := postData["text"].(string)
-
-		var textData map[string]string
-		if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
-			return fmt.Errorf("failed to parse text data, err: %v", parseErr)
-		}
-
-		if textData["User"] != "[REDACTED]" {
-			return fmt.Errorf("unexpected result - user in body is not redacted")
-		}
-
-		return nil
-	}
-	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/RedactTests.js\" --env shouldExist=true"))
 }
 
 func TestTapNoRedact(t *testing.T) {
@@ -590,79 +420,16 @@ func TestTapNoRedact(t *testing.T) {
 	}
 
 	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+	requestHeaders := map[string]string{"User-Header": "Mizu"}
 	requestBody := map[string]string{"User": "Mizu"}
 	for i := 0; i < defaultEntriesCount; i++ {
-		if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil {
+		if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	redactCheckFunc := func() error {
-		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
-
-		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
-		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entries, err: %v", requestErr)
-		}
-
-		entries := requestResult.([]interface{})
-		if len(entries) == 0 {
-			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
-		}
-
-		firstEntry :=  entries[0].(map[string]interface{})
-
-		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr = executeHttpGetRequest(entryUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entry, err: %v", requestErr)
-		}
-
-		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		entryJson := data["entry"].(string)
-
-		var entry map[string]interface{}
-		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
-			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
-		}
-
-		entryRequest := entry["request"].(map[string]interface{})
-		entryPayload := entryRequest["payload"].(map[string]interface{})
-		entryDetails := entryPayload["details"].(map[string]interface{})
-
-		headers :=  entryDetails["headers"].([]interface{})
-		for _, headerInterface := range headers {
-			header := headerInterface.(map[string]interface{})
-			if header["name"].(string) != "User-Agent" {
-				continue
-			}
-
-			userAgent := header["value"].(string)
-			if userAgent == "[REDACTED]" {
-				return fmt.Errorf("unexpected result - user agent is redacted")
-			}
-		}
-
-		postData := entryDetails["postData"].(map[string]interface{})
-		textDataStr := postData["text"].(string)
-
-		var textData map[string]string
-		if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
-			return fmt.Errorf("failed to parse text data, err: %v", parseErr)
-		}
-
-		if textData["User"] == "[REDACTED]" {
-			return fmt.Errorf("unexpected result - user in body is redacted")
-		}
-
-		return nil
-	}
-	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/RedactTests.js\" --env shouldExist=false")
 }
 
 func TestTapRegexMasking(t *testing.T) {
@@ -716,38 +483,26 @@ func TestTapRegexMasking(t *testing.T) {
 	redactCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
-		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
+		if err != nil {
+			return err
 		}
-
-		entries := requestResult.([]interface{})
-		if len(entries) == 0 {
-			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		err = checkEntriesAtLeast(entries, 1)
+		if err != nil {
+			return err
 		}
-
-		firstEntry :=  entries[0].(map[string]interface{})
+		firstEntry := entries[0]
 
 		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr = executeHttpGetRequest(entryUrl)
+		requestResult, requestErr := executeHttpGetRequest(entryUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entry, err: %v", requestErr)
 		}
 
-		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		entryJson := data["entry"].(string)
+		entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+		request := entry["request"].(map[string]interface{})
 
-		var entry map[string]interface{}
-		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
-			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
-		}
-
-		entryRequest := entry["request"].(map[string]interface{})
-		entryPayload := entryRequest["payload"].(map[string]interface{})
-		entryDetails := entryPayload["details"].(map[string]interface{})
-
-		postData := entryDetails["postData"].(map[string]interface{})
+		postData := request["postData"].(map[string]interface{})
 		textData := postData["text"].(string)
 
 		if textData != "[REDACTED]" {
@@ -805,7 +560,7 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
 
 	ignoredUserAgentCustomHeader := "Ignored-User-Agent"
-	headers := map[string]string {"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
+	headers := map[string]string{"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
 	for i := 0; i < defaultEntriesCount; i++ {
 		if _, requestErr := executeHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
@@ -823,38 +578,27 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 	ignoredUserAgentsCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount * 2, timestamp)
-		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
+		if err != nil {
+			return err
 		}
-
-		entries := requestResult.([]interface{})
-		if len(entries) == 0 {
-			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		err = checkEntriesAtLeast(entries, 1)
+		if err != nil {
+			return err
 		}
 
 		for _, entryInterface := range entries {
-			entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface.(map[string]interface{})["id"])
-			requestResult, requestErr = executeHttpGetRequest(entryUrl)
+			entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface["id"])
+			requestResult, requestErr := executeHttpGetRequest(entryUrl)
 			if requestErr != nil {
 				return fmt.Errorf("failed to get entry, err: %v", requestErr)
 			}
 
-			data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-			entryJson := data["entry"].(string)
+			entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+			request := entry["request"].(map[string]interface{})
 
-			var entry map[string]interface{}
-			if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
-				return fmt.Errorf("failed to parse entry, err: %v", parseErr)
-			}
-
-			entryRequest := entry["request"].(map[string]interface{})
-			entryPayload := entryRequest["payload"].(map[string]interface{})
-			entryDetails := entryPayload["details"].(map[string]interface{})
-
-			entryHeaders :=  entryDetails["headers"].([]interface{})
-			for _, headerInterface := range entryHeaders {
+			headers := request["_headers"].([]interface{})
+			for _, headerInterface := range headers {
 				header := headerInterface.(map[string]interface{})
 				if header["name"].(string) != ignoredUserAgentCustomHeader {
 					continue
@@ -922,21 +666,21 @@ func TestTapDumpLogs(t *testing.T) {
 		return
 	}
 
-	var dumpsLogsPath string
+	var dumpLogsPath string
 	for _, file := range files {
 		fileName := file.Name()
 		if strings.Contains(fileName, "mizu_logs") {
-			dumpsLogsPath = path.Join(mizuFolderPath, fileName)
+			dumpLogsPath = path.Join(mizuFolderPath, fileName)
 			break
 		}
 	}
 
-	if dumpsLogsPath == "" {
+	if dumpLogsPath == "" {
 		t.Errorf("dump logs file not found")
 		return
 	}
 
-	zipReader, zipError := zip.OpenReader(dumpsLogsPath)
+	zipReader, zipError := zip.OpenReader(dumpLogsPath)
 	if zipError != nil {
 		t.Errorf("failed to get zip reader, err: %v", zipError)
 		return
@@ -953,11 +697,16 @@ func TestTapDumpLogs(t *testing.T) {
 		logsFileNames = append(logsFileNames, file.Name)
 	}
 
-	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
+	if !Contains(logsFileNames, "mizu.mizu-api-server.mizu-api-server.log") {
 		t.Errorf("api server logs not found")
 		return
 	}
 
+	if !Contains(logsFileNames, "mizu.mizu-api-server.basenine.log") {
+		t.Errorf("basenine logs not found")
+		return
+	}
+
 	if !Contains(logsFileNames, "mizu_cli.log") {
 		t.Errorf("cli logs not found")
 		return

acceptanceTests/testsUtils.go

@@ -3,6 +3,7 @@ package acceptanceTests
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -10,9 +11,12 @@ import (
 	"os/exec"
 	"path"
 	"strings"
+	"sync"
 	"syscall"
+	"testing"
 	"time"
 
+	"github.com/gorilla/websocket"
 	"github.com/up9inc/mizu/shared"
 )
 
@@ -24,8 +28,26 @@ const (
 	defaultServiceName    = "httpbin"
 	defaultEntriesCount   = 50
 	waitAfterTapPodsReady = 3 * time.Second
+	cleanCommandTimeout   = 1 * time.Minute
 )
 
+type PodDescriptor struct {
+	Name      string
+	Namespace string
+}
+
+func isPodDescriptorInPodArray(pods []map[string]interface{}, podDescriptor PodDescriptor) bool {
+	for _, pod := range pods {
+		podNamespace := pod["namespace"].(string)
+		podName := pod["name"].(string)
+
+		if podDescriptor.Namespace == podNamespace && strings.Contains(podName, podDescriptor.Name) {
+			return true
+		}
+	}
+	return false
+}
+
 func getCliPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {
@@ -59,16 +81,21 @@ func getProxyUrl(namespace string, service string) string {
 }
 
 func getApiServerUrl(port uint16) string {
-	return fmt.Sprintf("http://localhost:%v/mizu", port)
+	return fmt.Sprintf("http://localhost:%v", port)
+}
+
+func getWebSocketUrl(port uint16) string {
+	return fmt.Sprintf("ws://localhost:%v/ws", port)
 }
 
 func getDefaultCommandArgs() []string {
 	setFlag := "--set"
 	telemetry := "telemetry=false"
 	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
-	imagePullPolicy := "image-pull-policy=Never"
+	imagePullPolicy := "image-pull-policy=IfNotPresent"
+	headless := "headless=true"
 
-	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy}
+	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
 }
 
 func getDefaultTapCommandArgs() []string {
@@ -103,6 +130,31 @@ func getDefaultConfigCommandArgs() []string {
 	return append([]string{configCommand}, defaultCmdArgs...)
 }
 
+func getDefaultCleanCommandArgs() []string {
+	cleanCommand := "clean"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{cleanCommand}, defaultCmdArgs...)
+}
+
+func getDefaultViewCommandArgs() []string {
+	viewCommand := "view"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{viewCommand}, defaultCmdArgs...)
+}
+
+func runCypressTests(t *testing.T, cypressRunCmd string) {
+	cypressCmd := exec.Command("bash", "-c", cypressRunCmd)
+	t.Logf("running command: %v", cypressCmd.String())
+	out, err := cypressCmd.Output()
+	if err != nil {
+		t.Errorf("%s", out)
+		return
+	}
+	t.Logf("%s", out)
+}
+
 func retriesExecute(retriesCount int, executeFunc func() error) error {
 	var lastError interface{}
 
@@ -131,16 +183,16 @@ func tryExecuteFunc(executeFunc func() error) (err interface{}) {
 }
 
 func waitTapPodsReady(apiServerUrl string) error {
-	resolvingUrl := fmt.Sprintf("%v/status/tappersCount", apiServerUrl)
+	resolvingUrl := fmt.Sprintf("%v/status/connectedTappersCount", apiServerUrl)
 	tapPodsReadyFunc := func() error {
 		requestResult, requestErr := executeHttpGetRequest(resolvingUrl)
 		if requestErr != nil {
 			return requestErr
 		}
 
-		tappersCount := requestResult.(float64)
-		if tappersCount == 0 {
-			return fmt.Errorf("no tappers running")
+		connectedTappersCount := requestResult.(float64)
+		if connectedTappersCount == 0 {
+			return fmt.Errorf("no connected tappers running")
 		}
 		time.Sleep(waitAfterTapPodsReady)
 		return nil
@@ -195,16 +247,57 @@ func executeHttpGetRequest(url string) (interface{}, error) {
 	return executeHttpRequest(response, requestErr)
 }
 
-func executeHttpPostRequest(url string, body interface{}) (interface{}, error) {
+func executeHttpPostRequestWithHeaders(url string, headers map[string]string, body interface{}) (interface{}, error) {
 	requestBody, jsonErr := json.Marshal(body)
 	if jsonErr != nil {
 		return nil, jsonErr
 	}
 
-	response, requestErr := http.Post(url, "application/json", bytes.NewBuffer(requestBody))
+	request, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(requestBody))
+	if err != nil {
+		return nil, err
+	}
+
+	request.Header.Add("Content-Type", "application/json")
+	for headerKey, headerValue := range headers {
+		request.Header.Add(headerKey, headerValue)
+	}
+
+	client := &http.Client{}
+	response, requestErr := client.Do(request)
 	return executeHttpRequest(response, requestErr)
 }
 
+func runMizuClean() error {
+	cliPath, err := getCliPath()
+	if err != nil {
+		return err
+	}
+
+	cleanCmdArgs := getDefaultCleanCommandArgs()
+
+	cleanCmd := exec.Command(cliPath, cleanCmdArgs...)
+
+	commandDone := make(chan error)
+	go func() {
+		if err := cleanCmd.Run(); err != nil {
+			commandDone <- err
+		}
+		commandDone <- nil
+	}()
+
+	select {
+	case err = <-commandDone:
+		if err != nil {
+			return err
+		}
+	case <-time.After(cleanCommandTimeout):
+		return errors.New("clean command timed out")
+	}
+
+	return nil
+}
+
 func cleanupCommand(cmd *exec.Cmd) error {
 	if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
 		return err
@@ -218,11 +311,10 @@ func cleanupCommand(cmd *exec.Cmd) error {
 }
 
 func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) {
-	tapStatus := tapStatusInterface.(map[string]interface{})
-	podsInterface := tapStatus["pods"].([]interface{})
+	tapPodsInterface := tapStatusInterface.([]interface{})
 
 	var pods []map[string]interface{}
-	for _, podInterface := range podsInterface {
+	for _, podInterface := range tapPodsInterface {
 		pods = append(pods, podInterface.(map[string]interface{}))
 	}
 
@@ -239,6 +331,77 @@ func getLogsPath() (string, error) {
 	return logsPath, nil
 }
 
+// waitTimeout waits for the waitgroup for the specified max timeout.
+// Returns true if waiting timed out.
+func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
+	channel := make(chan struct{})
+	go func() {
+		defer close(channel)
+		wg.Wait()
+	}()
+	select {
+	case <-channel:
+		return false // completed normally
+	case <-time.After(timeout):
+		return true // timed out
+	}
+}
+
+// checkEntriesAtLeast checks whether the number of entries greater than or equal to n
+func checkEntriesAtLeast(entries []map[string]interface{}, n int) error {
+	if len(entries) < n {
+		return fmt.Errorf("Unexpected entries result - Expected more than %d entries", n-1)
+	}
+	return nil
+}
+
+// getDBEntries retrieves the entries from the database before the given timestamp.
+// Also limits the results according to the limit parameter.
+// Timeout for the WebSocket connection is defined by the timeout parameter.
+func getDBEntries(timestamp int64, limit int, timeout time.Duration) (entries []map[string]interface{}, err error) {
+	query := fmt.Sprintf("timestamp < %d and limit(%d)", timestamp, limit)
+	webSocketUrl := getWebSocketUrl(defaultApiServerPort)
+
+	var connection *websocket.Conn
+	connection, _, err = websocket.DefaultDialer.Dial(webSocketUrl, nil)
+	if err != nil {
+		return
+	}
+	defer connection.Close()
+
+	handleWSConnection := func(wg *sync.WaitGroup) {
+		defer wg.Done()
+		for {
+			_, message, err := connection.ReadMessage()
+			if err != nil {
+				return
+			}
+
+			var data map[string]interface{}
+			if err = json.Unmarshal([]byte(message), &data); err != nil {
+				return
+			}
+
+			if data["messageType"] == "entry" {
+				entries = append(entries, data)
+			}
+		}
+	}
+
+	err = connection.WriteMessage(websocket.TextMessage, []byte(query))
+	if err != nil {
+		return
+	}
+
+	var wg sync.WaitGroup
+	go handleWSConnection(&wg)
+	wg.Add(1)
+
+	waitTimeout(&wg, timeout)
+
+	return
+}
+
 func Contains(slice []string, containsValue string) bool {
 	for _, sliceValue := range slice {
 		if sliceValue == containsValue {

agent/go.mod

@@ -3,26 +3,28 @@ module mizuserver
 go 1.16
 
 require (
+	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
+	github.com/chanced/openapi v0.0.6
 	github.com/djherbis/atime v1.0.0
-	github.com/fsnotify/fsnotify v1.4.9
 	github.com/getkin/kin-openapi v0.76.0
 	github.com/gin-contrib/static v0.0.1
-	github.com/gin-gonic/gin v1.7.2
+	github.com/gin-gonic/gin v1.7.7
 	github.com/go-playground/locales v0.13.0
 	github.com/go-playground/universal-translator v0.17.0
 	github.com/go-playground/validator/v10 v10.5.0
 	github.com/google/martian v2.1.0+incompatible
+	github.com/google/uuid v1.1.2
 	github.com/gorilla/websocket v1.4.2
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
+	github.com/ory/kratos-client-go v0.8.2-alpha.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
-	go.mongodb.org/mongo-driver v1.7.1
-	gorm.io/driver/sqlite v1.1.4
-	gorm.io/gorm v1.21.8
+	golang.org/x/text v0.3.5 // indirect
 	k8s.io/api v0.21.2
 	k8s.io/apimachinery v0.21.2
 	k8s.io/client-go v0.21.2

agent/go.sum

@@ -8,20 +8,30 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg
 cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
 cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
 cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0=
 cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0 h1:Dg9iHVQfrhq82rUNu9ZxUDrJLaxFUe/HlCVaLyRruq8=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
@@ -52,6 +62,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
+github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b h1:8m+eVxVVDDyJFidv7Ck1OwqnDaQR6pTSRGlCC2Dnw0A=
+github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b/go.mod h1:+tQQjzrp2501Nd6JXrb9s/XsNvFK3ZbxOnCdQl/vDRo=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -70,10 +82,17 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
+github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a h1:zG6t+4krPXcCKtLbjFvAh+fKN1d0qfD+RaCj+680OU8=
+github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a/go.mod h1:yhcmlFk1hxuZ+5XZbupzT/cEm/eE4ZvWbmsW1+Q/aZE=
+github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44 h1:4NOJMtvZaOA6cI2gkIuXk/2b5KTOvm/R4zyPy/yLCM4=
+github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44/go.mod h1:XVNfXN5kgZST4PQ0W/oBAHJku2OteCeHxjAbvfd0ARM=
+github.com/chanced/openapi v0.0.6 h1:2giGS47+T8/7MN2hfRHSIUPx86Qksk+J/ciIWcAM7hY=
+github.com/chanced/openapi v0.0.6/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -94,24 +113,26 @@ github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg=
 github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
+github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/getkin/kin-openapi v0.76.0 h1:j77zg3Ec+k+r+GA3d8hBoXpAc6KX9TbBPrwQGBIy2sY=
@@ -123,12 +144,11 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm
 github.com/gin-contrib/static v0.0.1 h1:JVxuvHPuUfkoul12N7dtQw7KRn/pSMq7Ue1Va9Swm1U=
 github.com/gin-contrib/static v0.0.1/go.mod h1:CSxeF+wep05e0kCOsqWdAWbSszmc31zTIbD8TvWl7Hs=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
-github.com/gin-gonic/gin v1.7.2 h1:Tg03T9yM2xa8j6I3Z3oqLaQRSmKvxPd6g/2HJ6zICFA=
-github.com/gin-gonic/gin v1.7.2/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
+github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs=
+github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
-github.com/go-errors/errors v1.4.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -195,31 +215,7 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn
 github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E=
 github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
-github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
-github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
-github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
-github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
-github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
-github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
-github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
-github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
-github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
-github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
-github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
-github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
-github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
-github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
-github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
 github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
-github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
-github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
-github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
-github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
-github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
-github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
-github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
-github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -237,11 +233,14 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -251,7 +250,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -259,10 +257,14 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -270,11 +272,14 @@ github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -321,12 +326,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
-github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
-github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI=
-github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -336,14 +336,10 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
-github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -364,16 +360,12 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
-github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
-github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ=
-github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -395,7 +387,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
-github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
@@ -414,12 +405,13 @@ github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWEr
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
+github.com/ory/kratos-client-go v0.8.2-alpha.1 h1:YlKhGOSZjounlB9iY4xSWlqHbyLYkeLzlLk8ZL7/nEM=
+github.com/ory/kratos-client-go v0.8.2-alpha.1/go.mod h1:dOQIsar76K07wMPJD/6aMhrWyY+sFGEagLDLso1CpsA=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -447,19 +439,17 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE=
+github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
@@ -468,7 +458,6 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
@@ -485,38 +474,51 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.12.0 h1:61wEp/qfvFnqKH/WCI3M8HuRut+mHT6Mr82QrFmM2SY=
+github.com/tidwall/gjson v1.12.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.3 h1:5+deguEhHSEjmuICXZ21uSSsXotWMA0orU783+Z7Cp8=
+github.com/tidwall/sjson v1.2.3/go.mod h1:5WdjKx3AQMvCJ4RG6/2UYT7dLrGvJUV1x4jdTAyGvZs=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d h1:WTz53dcfqCIWZpZLQoHbIcNc21s0ZHEZH7EqMPp99qQ=
+github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
-github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
-github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=
+github.com/wI2L/jsondiff v0.1.1/go.mod h1:bAbJSAJXZtfOCZ5y3v7Mfb6UQa3DGdGFjQj1cNv8EcM=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
-github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
-go.mongodb.org/mongo-driver v1.7.1 h1:jwqTeEM3x6L9xDXrCxN0Hbg7vdGfPBOTIkr0+/LYZDA=
-go.mongodb.org/mongo-driver v1.7.1/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -525,13 +527,11 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
@@ -586,6 +586,7 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -595,6 +596,13 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
@@ -603,15 +611,17 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 h1:D7nTwh4J0i+5mW4Zjzn5omvlr6YBcWywE6KOcatyNxY=
+golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -623,13 +633,10 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -646,15 +653,21 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@@ -683,13 +696,9 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
@@ -714,9 +723,19 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -733,13 +752,20 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -757,16 +783,31 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx
 google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -775,6 +816,7 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
@@ -803,11 +845,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=
-gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw=
-gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
-gorm.io/gorm v1.21.8 h1:2CEwZSzogdhsKPlJ9OvBKTdlWIpELXb6HbfLfMNhSYI=
-gorm.io/gorm v1.21.8/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -816,6 +853,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y=
 k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU=
 k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc=
@@ -849,5 +887,6 @@ sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

agent/kratos/Dockerfile

@@ -0,0 +1,14 @@
+FROM gcr.io/up9-docker-hub/mizu-kratos-base/simple-password-policy:latest
+
+USER root
+
+RUN apk add sqlite
+
+RUN mkdir -p /etc/config/kratos
+
+COPY ./kratos.yml /etc/config/kratos/kratos.yml
+COPY ./identity.schema.json /etc/config/kratos/identity.schema.json
+COPY ./start.sh /opt/start.sh
+RUN chmod +x /opt/start.sh
+
+ENTRYPOINT ["/opt/start.sh"]

agent/kratos/build-push-featurebranch.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+GCP_PROJECT=up9-docker-hub
+REPOSITORY=gcr.io/$GCP_PROJECT
+SERVER_NAME=mizu-kratos
+GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
+
+DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
+SEM_VER=${SEM_VER=0.0.0}
+
+DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
+
+if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
+then
+  echo "Pushing to $GIT_BRANCH is allowed only via CI"
+  exit 1
+fi
+
+echo "building ${DOCKER_TAGGED_BUILDS[@]}"
+DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
+docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
+
+for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
+do
+  echo pushing "$DOCKER_TAG"
+  docker push "$DOCKER_TAG"
+done

agent/kratos/identity.schema.json

@@ -0,0 +1,43 @@
+{
+	"$id": "https://schemas.ory.sh/presets/kratos/quickstart/email-password/identity.schema.json",
+	"$schema": "http://json-schema.org/draft-07/schema#",
+	"title": "Person",
+	"type": "object",
+	"properties": {
+		"traits": {
+			"type": "object",
+			"properties": {
+				"username": {
+					"type": "string",
+					"format": "username",
+					"title": "Username",
+					"minLength": 3,
+					"ory.sh/kratos": {
+						"credentials": {
+							"password": {
+								"identifier": true
+							}
+						}
+					}
+				},
+				"name": {
+					"type": "object",
+					"properties": {
+						"first": {
+							"title": "First Name",
+							"type": "string"
+						},
+						"last": {
+							"title": "Last Name",
+							"type": "string"
+						}
+					}
+				}
+			},
+			"required": [
+				"username"
+			],
+			"additionalProperties": false
+		}
+	}
+}

agent/kratos/kratos.yml

@@ -0,0 +1,84 @@
+version: v0.8.2-alpha.1
+
+dsn: sqlite:///app/data/kratos.sqlite?_fk=true
+
+serve:
+  public:
+    base_url: http://127.0.0.1:4433/
+    cors:
+      enabled: true
+  admin:
+    base_url: http://kratos:4434/
+
+selfservice:
+  default_browser_return_url: http://127.0.0.1:4455/
+  whitelisted_return_urls:
+    - http://127.0.0.1:4455
+
+  methods:
+    password:
+      enabled: true
+
+  flows:
+    error:
+      ui_url: http://127.0.0.1:4455/error
+
+    settings:
+      ui_url: http://127.0.0.1:4455/settings
+      privileged_session_max_age: 15m
+
+    recovery:
+      enabled: true
+      ui_url: http://127.0.0.1:4455/recovery
+
+    verification:
+      enabled: false
+      ui_url: http://127.0.0.1:4455/verification
+      after:
+        default_browser_return_url: http://127.0.0.1:4455/
+
+    logout:
+      after:
+        default_browser_return_url: http://127.0.0.1:4455/login
+
+    login:
+      ui_url: http://127.0.0.1:4455/login
+      lifespan: 10m
+
+    registration:
+      lifespan: 10m
+      ui_url: http://127.0.0.1:4455/registration
+      after:
+        password:
+          hooks:
+            -
+              hook: session
+
+log:
+  level: info
+  format: text
+  leak_sensitive_values: true
+
+secrets:
+  cookie:
+    - PLEASE-CHANGE-ME-I-AM-VERY-INSECURE
+  cipher:
+    - 32-LONG-SECRET-NOT-SECURE-AT-ALL
+
+ciphers:
+  algorithm: xchacha20-poly1305
+
+hashers:
+  argon2:
+    parallelism: 1
+    memory: 128MB
+    iterations: 2
+    salt_length: 16
+    key_length: 16
+
+identity:
+  default_schema_url: file:///etc/config/kratos/identity.schema.json
+
+courier:
+  smtp:
+    connection_uri: smtps://test:test@mailslurper:1025/?skip_ssl_verify=true

agent/kratos/start.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+kratos migrate sql sqlite:///app/data/kratos.sqlite?_fk=true --yes # this initializes the db
+kratos serve -c /etc/config/kratos/kratos.yml --watch-courier # start kratos

agent/main.go

@@ -1,20 +1,17 @@
 package main
 
 import (
-	"context"
 	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
-	"github.com/up9inc/mizu/shared/kubernetes"
 	"io/ioutil"
-	v1 "k8s.io/api/core/v1"
 	"mizuserver/pkg/api"
 	"mizuserver/pkg/config"
 	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/database"
+	"mizuserver/pkg/middlewares"
 	"mizuserver/pkg/models"
-	"mizuserver/pkg/providers"
+	"mizuserver/pkg/oas"
 	"mizuserver/pkg/routes"
 	"mizuserver/pkg/up9"
 	"mizuserver/pkg/utils"
@@ -25,13 +22,19 @@ import (
 	"path/filepath"
 	"plugin"
 	"sort"
+	"strconv"
+	"strings"
 	"syscall"
 	"time"
 
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/antelman107/net-wait-go/wait"
 	"github.com/gin-contrib/static"
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	"github.com/op/go-logging"
+	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap"
@@ -49,10 +52,13 @@ var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
 var extensions []*tapApi.Extension             // global
 var extensionsMap map[string]*tapApi.Extension // global
 
+var startTime int64
+
 const (
-	socketConnectionRetries = 10
+	socketConnectionRetries    = 30
 	socketConnectionRetryDelay = time.Second * 2
-	socketHandshakeTimeout = time.Second * 2
+	socketHandshakeTimeout     = time.Second * 2
+	uiIndexPath                = "./site/index.html"
 )
 
 func main() {
@@ -89,17 +95,17 @@ func main() {
 			panic("API server address must be provided with --api-server-address when using --tap")
 		}
 
+		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+		tapOpts := &tap.TapOpts{HostMode: hostMode}
 		tapTargets := getTapTargets()
 		if tapTargets != nil {
-			tap.SetFilterAuthorities(tapTargets)
-			logger.Log.Infof("Filtering for the following authorities: %v", tap.GetFilterIPs())
+			tapOpts.FilterAuthorities = tapTargets
+			logger.Log.Infof("Filtering for the following authorities: %v", tapOpts.FilterAuthorities)
 		}
 
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
 
 		filteringOptions := getTrafficFilteringOptions()
-		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-		tapOpts := &tap.TapOpts{HostMode: hostMode}
 		tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions)
 		socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
 		if err != nil {
@@ -109,19 +115,20 @@ func main() {
 
 		go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
 	} else if *apiServerMode {
-		database.InitDataBase(config.Config.AgentDatabasePath)
+		configureBasenineServer(shared.BasenineHost, shared.BaseninePort)
+		startTime = time.Now().UnixNano() / int64(time.Millisecond)
 		api.StartResolving(*namespace)
 
 		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-
+		enableExpFeatureIfNeeded()
 		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
 		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
 
 		syncEntriesConfig := getSyncEntriesConfig()
 		if syncEntriesConfig != nil {
 			if err := up9.SyncEntries(syncEntriesConfig); err != nil {
-				panic(fmt.Sprintf("Error syncing entries, err: %v", err))
+				logger.Log.Error("Error syncing entries, err: %v", err)
 			}
 		}
 
@@ -142,6 +149,41 @@ func main() {
 	logger.Log.Info("Exiting")
 }
 
+func enableExpFeatureIfNeeded() {
+	if config.Config.OAS {
+		oas.GetOasGeneratorInstance().Start()
+	}
+}
+
+func configureBasenineServer(host string, port string) {
+	if !wait.New(
+		wait.WithProto("tcp"),
+		wait.WithWait(200*time.Millisecond),
+		wait.WithBreak(50*time.Millisecond),
+		wait.WithDeadline(5*time.Second),
+		wait.WithDebug(true),
+	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
+		logger.Log.Panicf("Basenine is not available!")
+	}
+
+	// Limit the database size to default 200MB
+	err := basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
+	if err != nil {
+		logger.Log.Panicf("Error while limiting database size: %v", err)
+	}
+
+	// Define the macros
+	for _, extension := range extensions {
+		macros := extension.Dissector.Macros()
+		for macro, expanded := range macros {
+			err = basenine.Macro(host, port, macro, expanded)
+			if err != nil {
+				logger.Log.Panicf("Error while adding a macro: %v", err)
+			}
+		}
+	}
+}
+
 func loadExtensions() {
 	dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
 	extensionsDir := path.Join(dir, "./extensions/")
@@ -154,7 +196,7 @@ func loadExtensions() {
 	extensionsMap = make(map[string]*tapApi.Extension)
 	for i, file := range files {
 		filename := file.Name()
-		logger.Log.Infof("Loading extension: %s\n", filename)
+		logger.Log.Infof("Loading extension: %s", filename)
 		extension := &tapApi.Extension{
 			Path: path.Join(extensionsDir, filename),
 		}
@@ -166,7 +208,7 @@ func loadExtensions() {
 		var ok bool
 		dissector, ok = symDissector.(tapApi.Dissector)
 		if err != nil || !ok {
-			panic(fmt.Sprintf("Failed to load the extension: %s\n", extension.Path))
+			panic(fmt.Sprintf("Failed to load the extension: %s", extension.Path))
 		}
 		dissector.Register(extension)
 		extension.Dissector = dissector
@@ -179,7 +221,7 @@ func loadExtensions() {
 	})
 
 	for _, extension := range extensions {
-		logger.Log.Infof("Extension Properties: %+v\n", extension)
+		logger.Log.Infof("Extension Properties: %+v", extension)
 	}
 
 	controllers.InitExtensionsMap(extensionsMap)
@@ -197,24 +239,31 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 	}
 
 	app.Use(DisableRootStaticCache())
-	app.Use(static.ServeRoot("/", "./site"))
-	app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
 
-	api.WebSocketRoutes(app, &eventHandlers)
+	if err := setUIFlags(); err != nil {
+		logger.Log.Errorf("Error setting ui mode, err: %v", err)
+	}
+	app.Use(static.ServeRoot("/", "./site"))
+
+	app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
+
+	api.WebSocketRoutes(app, &eventHandlers, startTime)
+
+	if config.Config.StandaloneMode {
+		routes.ConfigRoutes(app)
+		routes.UserRoutes(app)
+		routes.InstallRoutes(app)
+	}
+
+	if config.Config.OAS {
+		routes.OASRoutes(app)
+	}
+
+	routes.QueryRoutes(app)
 	routes.EntriesRoutes(app)
 	routes.MetadataRoutes(app)
 	routes.StatusRoutes(app)
 	routes.NotFoundRoute(app)
-
-	if config.Config.DaemonMode {
-		ctx, cancel := context.WithCancel(context.Background())
-		defer cancel()
-
-		if _, err := startMizuTapperSyncer(ctx); err != nil {
-			logger.Log.Fatalf("error initializing tapper syncer: %+v", err)
-		}
-	}
-
 	utils.StartServer(app)
 }
 
@@ -229,24 +278,25 @@ func DisableRootStaticCache() gin.HandlerFunc {
 	}
 }
 
-func CORSMiddleware() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
-		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
-		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
-		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")
-
-		if c.Request.Method == "OPTIONS" {
-			c.AbortWithStatus(204)
-			return
-		}
-
-		c.Next()
+func setUIFlags() error {
+	read, err := ioutil.ReadFile(uiIndexPath)
+	if err != nil {
+		return err
 	}
+
+	replacedContent := strings.Replace(string(read), "__IS_STANDALONE__", strconv.FormatBool(config.Config.StandaloneMode), 1)
+	replacedContent = strings.Replace(replacedContent, "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS), 1)
+
+	err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
-func parseEnvVar(env string) map[string][]string {
-	var mapOfList map[string][]string
+func parseEnvVar(env string) map[string][]v1.Pod {
+	var mapOfList map[string][]v1.Pod
 
 	val, present := os.LookupEnv(env)
 
@@ -256,12 +306,12 @@ func parseEnvVar(env string) map[string][]string {
 
 	err := json.Unmarshal([]byte(val), &mapOfList)
 	if err != nil {
-		panic(fmt.Sprintf("env var %s's value of %s is invalid! must be map[string][]string %v", env, mapOfList, err))
+		panic(fmt.Sprintf("env var %s's value of %v is invalid! must be map[string][]v1.Pod %v", env, mapOfList, err))
 	}
 	return mapOfList
 }
 
-func getTapTargets() []string {
+func getTapTargets() []v1.Pod {
 	nodeName := os.Getenv(shared.NodeNameEnvVar)
 	tappedAddressesPerNodeDict := parseEnvVar(shared.TappedAddressesPerNodeDictEnvVar)
 	return tappedAddressesPerNodeDict[nodeName]
@@ -344,10 +394,11 @@ func getSyncEntriesConfig() *shared.SyncEntriesConfig {
 }
 
 func determineLogLevel() (logLevel logging.Level) {
-	logLevel = logging.INFO
-	if os.Getenv(shared.DebugModeEnvVar) == "1" {
-		logLevel = logging.DEBUG
+	logLevel, err := logging.LogLevel(os.Getenv(shared.LogLevelEnvVar))
+	if err != nil {
+		logLevel = logging.INFO
 	}
+
 	return
 }
 
@@ -361,69 +412,42 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time.
 		socketConnection, _, err := dialer.Dial(socketAddress, nil)
 		if err != nil {
 			if i < retryAmount {
-				logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay / time.Second)
+				logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay/time.Second)
 				time.Sleep(retryDelay)
 			}
 		} else {
+			go handleIncomingMessageAsTapper(socketConnection)
 			return socketConnection, nil
 		}
 	}
 	return nil, lastErr
 }
 
-
-func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, error){
-	provider, err := kubernetes.NewProviderInCluster()
-	if err != nil {
-		return nil, err
-	}
-
-	tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
-		TargetNamespaces:         config.Config.TargetNamespaces,
-		PodFilterRegex:           config.Config.TapTargetRegex.Regexp,
-		MizuResourcesNamespace:   config.Config.MizuResourcesNamespace,
-		AgentImage:               config.Config.AgentImage,
-		TapperResources:          config.Config.TapperResources,
-		ImagePullPolicy:          v1.PullPolicy(config.Config.PullPolicy),
-		DumpLogs:                 config.Config.DumpLogs,
-		IgnoredUserAgents:        config.Config.IgnoredUserAgents,
-		MizuApiFilteringOptions:  config.Config.MizuApiFilteringOptions,
-		MizuServiceAccountExists: true, //assume service account exists since daemon mode will not function without it anyway
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	// handle tapperSyncer events (pod changes and errors)
-	go func() {
-		for {
-			select {
-			case syncerErr, ok := <-tapperSyncer.ErrorOut:
-				if !ok {
-					logger.Log.Debug("mizuTapperSyncer err channel closed, ending listener loop")
-					return
-				}
-				logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr)
-			case _, ok := <-tapperSyncer.TapPodChangesOut:
-				if !ok {
-					logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop")
-					return
-				}
-				tapStatus := shared.TapStatus{Pods: kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods)}
-
-				serializedTapStatus, err := json.Marshal(shared.CreateWebSocketStatusMessage(tapStatus))
-				if err != nil {
-					logger.Log.Fatalf("error serializing tap status: %v", err)
-				}
-				api.BroadcastToBrowserClients(serializedTapStatus)
-				providers.TapStatus.Pods = tapStatus.Pods
-			case <-ctx.Done():
-				logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")
+func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
+	for {
+		if _, message, err := socketConnection.ReadMessage(); err != nil {
+			logger.Log.Errorf("error reading message from socket connection, err: %s, (%v,%+v)", err, err, err)
+			if errors.Is(err, syscall.EPIPE) {
+				// socket has disconnected, we can safely stop this goroutine
 				return
 			}
+		} else {
+			var socketMessageBase shared.WebSocketMessageMetadata
+			if err := json.Unmarshal(message, &socketMessageBase); err != nil {
+				logger.Log.Errorf("Could not unmarshal websocket message %v", err)
+			} else {
+				switch socketMessageBase.MessageType {
+				case shared.WebSocketMessageTypeTapConfig:
+					var tapConfigMessage *shared.WebSocketTapConfigMessage
+					if err := json.Unmarshal(message, &tapConfigMessage); err != nil {
+						logger.Log.Errorf("received unknown message from socket connection: %s, err: %s, (%v,%+v)", string(message), err, err, err)
+					} else {
+						tap.UpdateTapTargets(tapConfigMessage.TapTargets)
+					}
+				default:
+					logger.Log.Warningf("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
+				}
+			}
 		}
-	}()
-
-	return tapperSyncer, nil
+	}
 }

agent/pkg/api/main.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
 	"os"
@@ -14,15 +13,17 @@ import (
 	"strings"
 	"time"
 
-	"go.mongodb.org/mongo-driver/bson/primitive"
-
 	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	"mizuserver/pkg/models"
+	"mizuserver/pkg/oas"
 	"mizuserver/pkg/resolver"
 	"mizuserver/pkg/utils"
+
+	basenine "github.com/up9inc/basenine/client/go"
 )
 
 var k8sResolver *resolver.Resolver
@@ -76,7 +77,7 @@ func startReadingFiles(workingDir string) {
 		sort.Sort(utils.ByModTime(harFiles))
 
 		if len(harFiles) == 0 {
-			logger.Log.Infof("Waiting for new files\n")
+			logger.Log.Infof("Waiting for new files")
 			time.Sleep(3 * time.Second)
 			continue
 		}
@@ -99,11 +100,17 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		panic("Channel of captured messages is nil")
 	}
 
+	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+	if err != nil {
+		panic(err)
+	}
+	connection.InsertMode()
+
 	disableOASValidation := false
 	ctx := context.Background()
 	doc, contractContent, router, err := loadOAS(ctx)
 	if err != nil {
-		logger.Log.Infof("Disabled OAS validation: %s\n", err.Error())
+		logger.Log.Infof("Disabled OAS validation: %s", err.Error())
 		disableOASValidation = true
 	}
 
@@ -112,34 +119,33 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
-		mizuEntry := extension.Dissector.Analyze(item, primitive.NewObjectID().Hex(), resolvedSource, resolvedDestionation)
-		baseEntry := extension.Dissector.Summarize(mizuEntry)
-		mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
+		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
 				var httpPair tapApi.HTTPRequestResponsePair
-				json.Unmarshal([]byte(mizuEntry.Entry), &httpPair)
+				json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair)
 
 				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
-				baseEntry.ContractStatus = contract.Status
 				mizuEntry.ContractStatus = contract.Status
 				mizuEntry.ContractRequestReason = contract.RequestReason
 				mizuEntry.ContractResponseReason = contract.ResponseReason
 				mizuEntry.ContractContent = contract.Content
 			}
 
-			var pair tapApi.RequestResponsePair
-			json.Unmarshal([]byte(mizuEntry.Entry), &pair)
-			harEntry, err := utils.NewEntry(&pair)
+			harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
 			if err == nil {
-				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Service)
-				baseEntry.Rules = rules
+				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
+				mizuEntry.Rules = rules
 			}
-		}
-		database.CreateEntry(mizuEntry)
 
-		baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(baseEntry)
-		BroadcastToBrowserClients(baseEntryBytes)
+			oas.GetOasGeneratorInstance().PushEntry(harEntry)
+		}
+
+		data, err := json.Marshal(mizuEntry)
+		if err != nil {
+			panic(err)
+		}
+		connection.SendText(string(data))
 	}
 }
 
@@ -148,7 +154,7 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re
 		unresolvedSource := connectionInfo.ClientIP
 		resolvedSource = k8sResolver.Resolve(unresolvedSource)
 		if resolvedSource == "" {
-			logger.Log.Debugf("Cannot find resolved name to source: %s\n", unresolvedSource)
+			logger.Log.Debugf("Cannot find resolved name to source: %s", unresolvedSource)
 			if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
 				return
 			}
@@ -156,7 +162,7 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re
 		unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
 		resolvedDestination = k8sResolver.Resolve(unresolvedDestination)
 		if resolvedDestination == "" {
-			logger.Log.Debugf("Cannot find resolved name to dest: %s\n", unresolvedDestination)
+			logger.Log.Debugf("Cannot find resolved name to dest: %s", unresolvedDestination)
 			if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
 				return
 			}
@@ -171,21 +177,3 @@ func CheckIsServiceIP(address string) bool {
 	}
 	return k8sResolver.CheckIsServiceIP(address)
 }
-
-// gives a rough estimate of the size this will take up in the db, good enough for maintaining db size limit accurately
-func getEstimatedEntrySizeBytes(mizuEntry *tapApi.MizuEntry) int {
-	sizeBytes := len(mizuEntry.Entry)
-	sizeBytes += len(mizuEntry.EntryId)
-	sizeBytes += len(mizuEntry.Service)
-	sizeBytes += len(mizuEntry.Url)
-	sizeBytes += len(mizuEntry.Method)
-	sizeBytes += len(mizuEntry.RequestSenderIp)
-	sizeBytes += len(mizuEntry.ResolvedDestination)
-	sizeBytes += len(mizuEntry.ResolvedSource)
-	sizeBytes += 8 // Status bytes (sqlite integer is always 8 bytes)
-	sizeBytes += 8 // Timestamp bytes
-	sizeBytes += 8 // SizeBytes bytes
-	sizeBytes += 1 // IsOutgoing bytes
-
-	return sizeBytes
-}

agent/pkg/api/socket_routes.go

@@ -1,15 +1,22 @@
 package api
 
 import (
+	"encoding/json"
 	"errors"
+	"fmt"
+	"mizuserver/pkg/middlewares"
+	"mizuserver/pkg/models"
 	"net/http"
 	"sync"
 	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 type EventHandlers interface {
@@ -39,17 +46,18 @@ func init() {
 	connectedWebsockets = make(map[int]*SocketConnection, 0)
 }
 
-func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
+func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
 	app.GET("/ws", func(c *gin.Context) {
-		websocketHandler(c.Writer, c.Request, eventHandlers, false)
-	})
-	app.GET("/wsTapper", func(c *gin.Context) {
-		websocketHandler(c.Writer, c.Request, eventHandlers, true)
+		websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime)
+	}, middlewares.RequiresAuth())
+
+	app.GET("/wsTapper", func(c *gin.Context) { // TODO: add m2m authentication to this route
+		websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime)
 	})
 }
 
-func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) {
-	conn, err := websocketUpgrader.Upgrade(w, r, nil)
+func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool, startTime int64) {
+	ws, err := websocketUpgrader.Upgrade(w, r, nil)
 	if err != nil {
 		logger.Log.Errorf("Failed to set websocket upgrade: %v", err)
 		return
@@ -59,30 +67,114 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 
 	connectedWebsocketIdCounter++
 	socketId := connectedWebsocketIdCounter
-	connectedWebsockets[socketId] = &SocketConnection{connection: conn, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
+	connectedWebsockets[socketId] = &SocketConnection{connection: ws, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
 
 	websocketIdsLock.Unlock()
 
+	var connection *basenine.Connection
+	var isQuerySet bool
+
+	// `!isTapper` means it's a connection from the web UI
+	if !isTapper {
+		connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	data := make(chan []byte)
+	meta := make(chan []byte)
+
 	defer func() {
 		socketCleanup(socketId, connectedWebsockets[socketId])
+		data <- []byte(basenine.CloseChannel)
+		meta <- []byte(basenine.CloseChannel)
+		connection.Close()
 	}()
 
 	eventHandlers.WebSocketConnect(socketId, isTapper)
 
+	startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime)
+	SendToSocket(socketId, startTimeBytes)
+
 	for {
-		_, msg, err := conn.ReadMessage()
+		_, msg, err := ws.ReadMessage()
 		if err != nil {
-			logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
+			if _, ok := err.(*websocket.CloseError); ok {
+				logger.Log.Debugf("Received websocket close message, socket id: %d", socketId)
+			} else {
+				logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
+			}
+
 			break
 		}
-		eventHandlers.WebSocketMessage(socketId, msg)
+
+		if !isTapper && !isQuerySet {
+			query := string(msg)
+			err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
+			if err != nil {
+				toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
+					Type:      "error",
+					AutoClose: 5000,
+					Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
+				})
+				SendToSocket(socketId, toastBytes)
+				break
+			}
+
+			isQuerySet = true
+
+			handleDataChannel := func(c *basenine.Connection, data chan []byte) {
+				for {
+					bytes := <-data
+
+					if string(bytes) == basenine.CloseChannel {
+						return
+					}
+
+					var entry *tapApi.Entry
+					err = json.Unmarshal(bytes, &entry)
+
+					base := tapApi.Summarize(entry)
+
+					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
+					SendToSocket(socketId, baseEntryBytes)
+				}
+			}
+
+			handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
+				for {
+					bytes := <-meta
+
+					if string(bytes) == basenine.CloseChannel {
+						return
+					}
+
+					var metadata *basenine.Metadata
+					err = json.Unmarshal(bytes, &metadata)
+					if err != nil {
+						logger.Log.Debugf("Error recieving metadata: %v", err.Error())
+					}
+
+					metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
+					SendToSocket(socketId, metadataBytes)
+				}
+			}
+
+			go handleDataChannel(connection, data)
+			go handleMetaChannel(connection, meta)
+
+			connection.Query(query, data, meta)
+		} else {
+			eventHandlers.WebSocketMessage(socketId, msg)
+		}
 	}
 }
 
 func socketCleanup(socketId int, socketConnection *SocketConnection) {
 	err := socketConnection.connection.Close()
 	if err != nil {
-		logger.Log.Errorf("Error closing socket connection for socket id %d: %v\n", socketId, err)
+		logger.Log.Errorf("Error closing socket connection for socket id %d: %v", socketId, err)
 	}
 
 	websocketIdsLock.Lock()

agent/pkg/api/socket_server_handlers.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tappers"
 	"mizuserver/pkg/up9"
 	"sync"
 
@@ -29,7 +30,7 @@ func init() {
 func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
-		providers.TapperAdded()
+		tappers.Connected()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
 		socketListLock.Lock()
@@ -41,7 +42,7 @@ func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
-		providers.TapperRemoved()
+		tappers.Disconnected()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
 		socketListLock.Lock()
@@ -65,14 +66,14 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 	var socketMessageBase shared.WebSocketMessageMetadata
 	err := json.Unmarshal(message, &socketMessageBase)
 	if err != nil {
-		logger.Log.Infof("Could not unmarshal websocket message %v\n", err)
+		logger.Log.Infof("Could not unmarshal websocket message %v", err)
 	} else {
 		switch socketMessageBase.MessageType {
 		case shared.WebSocketMessageTypeTappedEntry:
 			var tappedEntryMessage models.WebSocketTappedEntryMessage
 			err := json.Unmarshal(message, &tappedEntryMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
 			} else {
 				// NOTE: This is where the message comes back from the intermediate WebSocket to code.
 				h.SocketOutChannel <- tappedEntryMessage.Data
@@ -81,16 +82,15 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 			var statusMessage shared.WebSocketStatusMessage
 			err := json.Unmarshal(message, &statusMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
 			} else {
-				providers.TapStatus.Pods = statusMessage.TappingStatus.Pods
 				BroadcastToBrowserClients(message)
 			}
 		case shared.WebsocketMessageTypeOutboundLink:
 			var outboundLinkMessage models.WebsocketOutboundLinkMessage
 			err := json.Unmarshal(message, &outboundLinkMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
+				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
 			} else {
 				handleTLSLink(outboundLinkMessage)
 			}

agent/pkg/config/config.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap/api"
 	"io/ioutil"
 	"os"
 )
@@ -12,7 +11,6 @@ import (
 // these values are used when the config.json file is not present
 const (
 	defaultMaxDatabaseSizeBytes int64  = 200 * 1000 * 1000
-	defaultRegexTarget          string = ".*"
 	DefaultDatabasePath         string = "./entries"
 )
 
@@ -48,14 +46,8 @@ func applyDefaultConfig() error {
 }
 
 func getDefaultConfig() (*shared.MizuAgentConfig, error) {
-	regex, err := api.CompileRegexToSerializableRegexp(defaultRegexTarget)
-	if err != nil {
-		return nil, err
-	}
 	return &shared.MizuAgentConfig{
-		TapTargetRegex:    *regex,
 		MaxDBSizeBytes:    defaultMaxDatabaseSizeBytes,
 		AgentDatabasePath: DefaultDatabasePath,
-		DaemonMode:        false,
 	}, nil
 }

agent/pkg/controllers/config_controller.go

@@ -0,0 +1,154 @@
+package controllers
+
+import (
+	"context"
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
+	v1 "k8s.io/api/core/v1"
+	"mizuserver/pkg/config"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tapConfig"
+	"mizuserver/pkg/providers/tappedPods"
+	"mizuserver/pkg/providers/tappers"
+	"net/http"
+	"regexp"
+	"time"
+)
+
+var cancelTapperSyncer context.CancelFunc
+
+func PostTapConfig(c *gin.Context) {
+	requestTapConfig := &models.TapConfig{}
+
+	if err := c.Bind(requestTapConfig); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+
+	if cancelTapperSyncer != nil {
+		cancelTapperSyncer()
+
+		tappedPods.Set([]*shared.PodInfo{})
+		tappers.ResetStatus()
+
+		broadcastTappedPodsStatus()
+	}
+
+	var tappedNamespaces []string
+	for namespace, tapped := range requestTapConfig.TappedNamespaces {
+		if tapped {
+			tappedNamespaces = append(tappedNamespaces, namespace)
+		}
+	}
+
+	podRegex, _ := regexp.Compile(".*")
+
+	kubernetesProvider, err := providers.GetKubernetesProvider()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{}, tapApi.TrafficFilteringOptions{}, false); err != nil {
+		c.JSON(http.StatusInternalServerError, err)
+		cancel()
+		return
+	}
+
+	cancelTapperSyncer = cancel
+	tapConfig.Save(requestTapConfig)
+
+	c.JSON(http.StatusOK, "OK")
+}
+
+func GetTapConfig(c *gin.Context) {
+	kubernetesProvider, err := providers.GetKubernetesProvider()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	namespaces, err := kubernetesProvider.ListAllNamespaces(ctx)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, err)
+		return
+	}
+
+	savedTapConfig := tapConfig.Get()
+
+	tappedNamespaces := make(map[string]bool)
+	for _, namespace := range namespaces {
+		if namespace.Name == config.Config.MizuResourcesNamespace {
+			continue
+		}
+
+		tappedNamespaces[namespace.Name] = savedTapConfig.TappedNamespaces[namespace.Name]
+	}
+
+	tapConfigToReturn := models.TapConfig{TappedNamespaces: tappedNamespaces}
+	c.JSON(http.StatusOK, tapConfigToReturn)
+}
+
+func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, serviceMesh bool) (*kubernetes.MizuTapperSyncer, error) {
+	tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
+		TargetNamespaces:         targetNamespaces,
+		PodFilterRegex:           podFilterRegex,
+		MizuResourcesNamespace:   config.Config.MizuResourcesNamespace,
+		AgentImage:               config.Config.AgentImage,
+		TapperResources:          config.Config.TapperResources,
+		ImagePullPolicy:          v1.PullPolicy(config.Config.PullPolicy),
+		LogLevel:                 config.Config.LogLevel,
+		IgnoredUserAgents:        ignoredUserAgents,
+		MizuApiFilteringOptions:  mizuApiFilteringOptions,
+		MizuServiceAccountExists: true, //assume service account exists since install mode will not function without it anyway
+		ServiceMesh:              serviceMesh,
+	}, time.Now())
+
+	if err != nil {
+		return nil, err
+	}
+
+	// handle tapperSyncer events (pod changes and errors)
+	go func() {
+		for {
+			select {
+			case syncerErr, ok := <-tapperSyncer.ErrorOut:
+				if !ok {
+					logger.Log.Debug("mizuTapperSyncer err channel closed, ending listener loop")
+					return
+				}
+				logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr)
+			case _, ok := <-tapperSyncer.TapPodChangesOut:
+				if !ok {
+					logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop")
+					return
+				}
+
+				tappedPods.Set(kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods))
+				broadcastTappedPodsStatus()
+			case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut:
+				if !ok {
+					logger.Log.Debug("mizuTapperSyncer tapper status changed channel closed, ending listener loop")
+					return
+				}
+
+				tappers.SetStatus(&tapperStatus)
+				broadcastTappedPodsStatus()
+			case <-ctx.Done():
+				logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")
+				return
+			}
+		}
+	}()
+
+	return tapperSyncer, nil
+}

agent/pkg/controllers/entries_controller.go

@@ -2,14 +2,19 @@ package controllers
 
 import (
 	"encoding/json"
-	"fmt"
-	"github.com/gin-gonic/gin"
-	tapApi "github.com/up9inc/mizu/tap/api"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/utils"
 	"mizuserver/pkg/validation"
 	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/gin-gonic/gin"
+
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 var extensionsMap map[string]*tapApi.Extension // global
@@ -18,78 +23,122 @@ func InitExtensionsMap(ref map[string]*tapApi.Extension) {
 	extensionsMap = ref
 }
 
-func GetEntries(c *gin.Context) {
-	entriesFilter := &models.EntriesFilter{}
-
-	if err := c.BindQuery(entriesFilter); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-	}
-	err := validation.Validate(entriesFilter)
+func Error(c *gin.Context, err error) bool {
 	if err != nil {
+		logger.Log.Errorf("Error getting entry: %v", err)
+		c.Error(err)
+		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       err.Error(),
+		})
+		return true // signal that there was an error and the caller should return
+	}
+	return false // no error, can continue
+}
+
+func GetEntries(c *gin.Context) {
+	entriesRequest := &models.EntriesRequest{}
+
+	if err := c.BindQuery(entriesRequest); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 	}
-
-	order := database.OperatorToOrderMapping[entriesFilter.Operator]
-	operatorSymbol := database.OperatorToSymbolMapping[entriesFilter.Operator]
-	var entries []tapApi.MizuEntry
-	database.GetEntriesTable().
-		Order(fmt.Sprintf("timestamp %s", order)).
-		Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)).
-		Limit(entriesFilter.Limit).
-		Find(&entries)
-
-	if len(entries) > 0 && order == database.OrderDesc {
-		// the entries always order from oldest to newest - we should reverse
-		utils.ReverseSlice(entries)
+	validationError := validation.Validate(entriesRequest)
+	if validationError != nil {
+		c.JSON(http.StatusBadRequest, validationError)
 	}
 
-	baseEntries := make([]tapApi.BaseEntryDetails, 0)
-	for _, entry := range entries {
-		baseEntryDetails := tapApi.BaseEntryDetails{}
-		if err := models.GetEntry(&entry, &baseEntryDetails); err != nil {
-			continue
-		}
-
-		var pair tapApi.RequestResponsePair
-		json.Unmarshal([]byte(entry.Entry), &pair)
-		harEntry, err := utils.NewEntry(&pair)
-		if err == nil {
-			rules, _, _ := models.RunValidationRulesState(*harEntry, entry.Service)
-			baseEntryDetails.Rules = rules
-		}
-
-		baseEntries = append(baseEntries, baseEntryDetails)
+	if entriesRequest.TimeoutMs == 0 {
+		entriesRequest.TimeoutMs = 3000
 	}
 
-	c.JSON(http.StatusOK, baseEntries)
+	data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
+		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
+		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, validationError)
+	}
+
+	response := &models.EntriesResponse{}
+	var dataSlice []interface{}
+
+	for _, row := range data {
+		var entry *tapApi.Entry
+		err = json.Unmarshal(row, &entry)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{
+				"error":     true,
+				"type":      "error",
+				"autoClose": "5000",
+				"msg":       string(row),
+			})
+			return // exit
+		}
+
+		base := tapApi.Summarize(entry)
+
+		dataSlice = append(dataSlice, base)
+	}
+
+	var metadata *basenine.Metadata
+	err = json.Unmarshal(meta, &metadata)
+	if err != nil {
+		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
+	}
+
+	response.Data = dataSlice
+	response.Meta = metadata
+
+	c.JSON(http.StatusOK, response)
 }
 
 func GetEntry(c *gin.Context) {
-	var entryData tapApi.MizuEntry
-	database.GetEntriesTable().
-		Where(map[string]string{"entryId": c.Param("entryId")}).
-		First(&entryData)
+	singleEntryRequest := &models.SingleEntryRequest{}
 
-	extension := extensionsMap[entryData.ProtocolName]
-	protocol, representation, bodySize, _ := extension.Dissector.Represent(&entryData)
+	if err := c.BindQuery(singleEntryRequest); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+	validationError := validation.Validate(singleEntryRequest)
+	if validationError != nil {
+		c.JSON(http.StatusBadRequest, validationError)
+	}
+
+	id, _ := strconv.Atoi(c.Param("id"))
+	var entry *tapApi.Entry
+	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id, singleEntryRequest.Query)
+	if Error(c, err) {
+		return // exit
+	}
+	err = json.Unmarshal(bytes, &entry)
+	if err != nil {
+		c.JSON(http.StatusNotFound, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       string(bytes),
+		})
+		return // exit
+	}
+
+	extension := extensionsMap[entry.Protocol.Name]
+	representation, bodySize, _ := extension.Dissector.Represent(entry.Request, entry.Response)
 
 	var rules []map[string]interface{}
 	var isRulesEnabled bool
-	if entryData.ProtocolName == "http" {
-		var pair tapApi.RequestResponsePair
-		json.Unmarshal([]byte(entryData.Entry), &pair)
-		harEntry, _ := utils.NewEntry(&pair)
-		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entryData.Service)
+	if entry.Protocol.Name == "http" {
+		harEntry, _ := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
 		isRulesEnabled = _isRulesEnabled
 		inrec, _ := json.Marshal(rulesMatched)
 		json.Unmarshal(inrec, &rules)
 	}
 
-	c.JSON(http.StatusOK, tapApi.MizuEntryWrapper{
-		Protocol:       protocol,
+	c.JSON(http.StatusOK, tapApi.EntryWrapper{
+		Protocol:       entry.Protocol,
 		Representation: string(representation),
 		BodySize:       bodySize,
-		Data:           entryData,
+		Data:           entry,
 		Rules:          rules,
 		IsRulesEnabled: isRulesEnabled,
 	})

agent/pkg/controllers/install_controller.go

@@ -0,0 +1,18 @@
+package controllers
+
+import (
+	"mizuserver/pkg/providers"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/shared/logger"
+)
+
+func IsSetupNecessary(c *gin.Context) {
+	if IsInstallNeeded, err := providers.IsInstallNeeded(); err != nil {
+		logger.Log.Errorf("unknown internal while checking if install is needed %s", err)
+		c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while checking if install is needed"})
+	} else {
+		c.JSON(http.StatusOK, IsInstallNeeded)
+	}
+}

agent/pkg/controllers/oas_controller.go

@@ -0,0 +1,62 @@
+package controllers
+
+import (
+	"github.com/chanced/openapi"
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/oas"
+	"net/http"
+)
+
+func GetOASServers(c *gin.Context) {
+	m := make([]string, 0)
+	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
+		m = append(m, key.(string))
+		return true
+	})
+
+	c.JSON(http.StatusOK, m)
+}
+
+func GetOASSpec(c *gin.Context) {
+	res, ok := oas.GetOasGeneratorInstance().ServiceSpecs.Load(c.Param("id"))
+	if !ok {
+		c.JSON(http.StatusNotFound, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       "Service not found among specs",
+		})
+		return // exit
+	}
+
+	gen := res.(*oas.SpecGen)
+	spec, err := gen.GetSpec()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       err,
+		})
+		return // exit
+	}
+
+	c.JSON(http.StatusOK, spec)
+}
+
+func GetOASAllSpecs(c *gin.Context) {
+	res := map[string]*openapi.OpenAPI{}
+	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
+		svc := key.(string)
+		gen := value.(*oas.SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			logger.Log.Warningf("Failed to obtain spec for service %s: %s", svc, err)
+			return true
+		}
+		res[svc] = spec
+		return true
+	})
+	c.JSON(http.StatusOK, res)
+}

agent/pkg/controllers/oas_controller_test.go

@@ -0,0 +1,43 @@
+package controllers
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/oas"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestGetOASServers(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+
+	GetOASServers(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+	return
+}
+
+func TestGetOASAllSpecs(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+
+	GetOASAllSpecs(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+	return
+}
+
+func TestGetOASSpec(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+
+	c.Params = []gin.Param{{Key: "id", Value: "some"}}
+
+	GetOASSpec(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+	return
+}

agent/pkg/controllers/query_controller.go

@@ -0,0 +1,31 @@
+package controllers
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/shared"
+)
+
+type ValidateResponse struct {
+	Valid   bool   `json:"valid"`
+	Message string `json:"message"`
+}
+
+func PostValidate(c *gin.Context) {
+	query := c.PostForm("query")
+	valid := true
+	message := ""
+
+	err := basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
+	if err != nil {
+		valid = false
+		message = err.Error()
+	}
+
+	c.JSON(http.StatusOK, ValidateResponse{
+		Valid:   valid,
+		Message: message,
+	})
+}

agent/pkg/controllers/status_controller.go

@@ -2,49 +2,75 @@ package controllers
 
 import (
 	"encoding/json"
-	"mizuserver/pkg/api"
-	"mizuserver/pkg/holder"
-	"mizuserver/pkg/providers"
-	"mizuserver/pkg/up9"
-	"mizuserver/pkg/validation"
-	"net/http"
-
 	"github.com/gin-gonic/gin"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/api"
+	"mizuserver/pkg/holder"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tappedPods"
+	"mizuserver/pkg/providers/tappers"
+	"mizuserver/pkg/up9"
+	"mizuserver/pkg/validation"
+	"net/http"
 )
 
 func HealthCheck(c *gin.Context) {
+	tappersStatus := make([]*shared.TapperStatus, 0)
+	for _, value := range tappers.GetStatus() {
+		tappersStatus = append(tappersStatus, value)
+	}
+
 	response := shared.HealthResponse{
-		TapStatus:    providers.TapStatus,
-		TappersCount: providers.TappersCount,
+		TappedPods:            tappedPods.Get(),
+		ConnectedTappersCount: tappers.GetConnectedCount(),
+		TappersStatus:         tappersStatus,
 	}
 	c.JSON(http.StatusOK, response)
 }
 
-
 func PostTappedPods(c *gin.Context) {
-	tapStatus := &shared.TapStatus{}
-	if err := c.Bind(tapStatus); err != nil {
+	var requestTappedPods []*shared.PodInfo
+	if err := c.Bind(&requestTappedPods); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
-	if err := validation.Validate(tapStatus); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-		return
-	}
-	logger.Log.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods))
-	providers.TapStatus.Pods = tapStatus.Pods
-	message := shared.CreateWebSocketStatusMessage(*tapStatus)
+
+	logger.Log.Infof("[Status] POST request: %d tapped pods", len(requestTappedPods))
+	tappedPods.Set(requestTappedPods)
+	broadcastTappedPodsStatus()
+}
+
+func broadcastTappedPodsStatus() {
+	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
+
+	message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
 	if jsonBytes, err := json.Marshal(message); err != nil {
-		logger.Log.Errorf("Could not Marshal message %v\n", err)
+		logger.Log.Errorf("Could not Marshal message %v", err)
 	} else {
 		api.BroadcastToBrowserClients(jsonBytes)
 	}
 }
 
-func GetTappersCount(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.TappersCount)
+func PostTapperStatus(c *gin.Context) {
+	tapperStatus := &shared.TapperStatus{}
+	if err := c.Bind(tapperStatus); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+
+	if err := validation.Validate(tapperStatus); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+
+	logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus)
+	tappers.SetStatus(tapperStatus)
+	broadcastTappedPodsStatus()
+}
+
+func GetConnectedTappersCount(c *gin.Context) {
+	c.JSON(http.StatusOK, tappers.GetConnectedCount())
 }
 
 func GetAuthStatus(c *gin.Context) {
@@ -58,7 +84,8 @@ func GetAuthStatus(c *gin.Context) {
 }
 
 func GetTappingStatus(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.TapStatus)
+	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
+	c.JSON(http.StatusOK, tappedPodsStatus)
 }
 
 func AnalyzeInformation(c *gin.Context) {

agent/pkg/controllers/user_controller.go

@@ -0,0 +1,39 @@
+package controllers
+
+import (
+	"mizuserver/pkg/providers"
+
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/shared/logger"
+)
+
+func Login(c *gin.Context) {
+	if token, err := providers.PerformLogin(c.PostForm("username"), c.PostForm("password"), c.Request.Context()); err != nil {
+		c.AbortWithStatusJSON(401, gin.H{"error": "bad login"})
+	} else {
+		c.JSON(200, gin.H{"token": token})
+	}
+}
+
+func Logout(c *gin.Context) {
+	token := c.GetHeader("x-session-token")
+	if err := providers.Logout(token, c.Request.Context()); err != nil {
+		c.AbortWithStatusJSON(500, gin.H{"error": "error occured while logging out, the session might still be valid"})
+	} else {
+		c.JSON(200, "")
+	}
+}
+
+func Register(c *gin.Context) {
+	if token, _, err, formErrorMessages := providers.RegisterUser(c.PostForm("username"), c.PostForm("password"), c.Request.Context()); err != nil {
+		if formErrorMessages != nil {
+			logger.Log.Infof("user attempted to register but had form errors %v %v", formErrorMessages, err)
+			c.AbortWithStatusJSON(400, formErrorMessages)
+		} else {
+			logger.Log.Errorf("unknown internal error registering user %s", err)
+			c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while registering"})
+		}
+	} else {
+		c.JSON(200, gin.H{"token": token})
+	}
+}

agent/pkg/database/main.go

@@ -1,78 +0,0 @@
-package database
-
-import (
-	"fmt"
-	"mizuserver/pkg/utils"
-	"time"
-
-	"gorm.io/driver/sqlite"
-	"gorm.io/gorm"
-	"gorm.io/gorm/logger"
-
-	tapApi "github.com/up9inc/mizu/tap/api"
-)
-
-const (
-	OrderDesc  = "desc"
-	OrderAsc   = "asc"
-	LT         = "lt"
-	GT         = "gt"
-	TimeFormat = "2006-01-02 15:04:05.000000000"
-)
-
-var (
-	DB                      *gorm.DB
-	IsDBLocked              = false
-	OperatorToSymbolMapping = map[string]string{
-		LT: "<",
-		GT: ">",
-	}
-	OperatorToOrderMapping = map[string]string{
-		LT: OrderDesc,
-		GT: OrderAsc,
-	}
-)
-
-var DBPath string
-
-func GetEntriesTable() *gorm.DB {
-	return DB.Table("mizu_entries")
-}
-
-func CreateEntry(entry *tapApi.MizuEntry) {
-	if IsDBLocked {
-		return
-	}
-	GetEntriesTable().Create(entry)
-}
-
-func InitDataBase(databasePath string) *gorm.DB {
-	DBPath = databasePath
-	DB, _ = gorm.Open(sqlite.Open(databasePath), &gorm.Config{
-		Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond},
-	})
-	_ = DB.AutoMigrate(&tapApi.MizuEntry{}) // this will ensure table is created
-	go StartEnforcingDatabaseSize()
-	return DB
-}
-
-func GetEntriesFromDb(timeFrom time.Time, timeTo time.Time, protocolName *string) []tapApi.MizuEntry {
-	order := OrderDesc
-	protocolNameCondition := "1 = 1"
-	if protocolName != nil {
-		protocolNameCondition = fmt.Sprintf("protocolName = '%s'", *protocolName)
-	}
-
-	var entries []tapApi.MizuEntry
-	GetEntriesTable().
-		Where(protocolNameCondition).
-		Where(fmt.Sprintf("created_at BETWEEN '%s' AND '%s'", timeFrom.Format(TimeFormat), timeTo.Format(TimeFormat))).
-		Order(fmt.Sprintf("timestamp %s", order)).
-		Find(&entries)
-
-	if len(entries) > 0 {
-		// the entries always order from oldest to newest so we should revers
-		utils.ReverseSlice(entries)
-	}
-	return entries
-}

agent/pkg/database/size_enforcer.go

@@ -1,102 +0,0 @@
-package database
-
-import (
-	"mizuserver/pkg/config"
-	"os"
-	"time"
-
-	"github.com/fsnotify/fsnotify"
-	"github.com/up9inc/mizu/shared/debounce"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/shared/units"
-	tapApi "github.com/up9inc/mizu/tap/api"
-)
-
-const percentageOfMaxSizeBytesToPrune = 15
-
-func StartEnforcingDatabaseSize() {
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		logger.Log.Fatalf("Error creating filesystem watcher for db size enforcement: %v\n", err)
-		return
-	}
-
-	checkFileSizeDebouncer := debounce.NewDebouncer(5*time.Second, func() {
-		checkFileSize(config.Config.MaxDBSizeBytes)
-	})
-
-	go func() {
-		for {
-			select {
-			case event, ok := <-watcher.Events:
-				if !ok {
-					return // closed channel
-				}
-				if event.Op == fsnotify.Write {
-					checkFileSizeDebouncer.SetOn()
-				}
-			case err, ok := <-watcher.Errors:
-				if !ok {
-					return // closed channel
-				}
-				logger.Log.Errorf("filesystem watcher encountered error:%v", err)
-			}
-		}
-	}()
-
-	err = watcher.Add(DBPath)
-	if err != nil {
-		logger.Log.Fatalf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err)
-	}
-}
-
-func checkFileSize(maxSizeBytes int64) {
-	fileStat, err := os.Stat(DBPath)
-	if err != nil {
-		logger.Log.Errorf("Error checking %s file size: %v", DBPath, err)
-	} else {
-		if fileStat.Size() > maxSizeBytes {
-			pruneOldEntries(fileStat.Size())
-		}
-	}
-}
-
-func pruneOldEntries(currentFileSize int64) {
-	// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
-	IsDBLocked = true
-	defer func() { IsDBLocked = false }()
-
-	amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
-
-	rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()
-	if err != nil {
-		logger.Log.Errorf("Error getting 10000 first db rows: %v", err)
-		return
-	}
-
-	entryIdsToRemove := make([]uint, 0)
-	bytesToBeRemoved := int64(0)
-	for rows.Next() {
-		if bytesToBeRemoved >= amountOfBytesToTrim {
-			break
-		}
-		var entry tapApi.MizuEntry
-		err = DB.ScanRows(rows, &entry)
-		if err != nil {
-			logger.Log.Errorf("Error scanning db row: %v", err)
-			continue
-		}
-
-		entryIdsToRemove = append(entryIdsToRemove, entry.ID)
-		bytesToBeRemoved += int64(entry.EstimatedSizeBytes)
-	}
-
-	if len(entryIdsToRemove) > 0 {
-		GetEntriesTable().Where(entryIdsToRemove).Delete(tapApi.MizuEntry{})
-		// VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this
-		DB.Exec("VACUUM")
-		logger.Log.Errorf("Removed %d rows and cleared %s", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
-	} else {
-		logger.Log.Error("Found no rows to remove when pruning")
-	}
-}

agent/pkg/middlewares/cors.go

@@ -0,0 +1,19 @@
+package middlewares
+
+import "github.com/gin-gonic/gin"
+
+func CORSMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
+		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
+		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With, x-session-token")
+		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")
+
+		if c.Request.Method == "OPTIONS" {
+			c.AbortWithStatus(204)
+			return
+		}
+
+		c.Next()
+	}
+}

agent/pkg/middlewares/requiresAuth.go

@@ -0,0 +1,49 @@
+package middlewares
+
+import (
+	"mizuserver/pkg/config"
+	"mizuserver/pkg/providers"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/patrickmn/go-cache"
+	"github.com/up9inc/mizu/shared/logger"
+)
+
+const cachedValidTokensRetainmentTime = time.Minute * 1
+
+var cachedValidTokens = cache.New(cachedValidTokensRetainmentTime, cachedValidTokensRetainmentTime)
+
+func RequiresAuth() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// auth is irrelevant for ephermeral mizu
+		if !config.Config.StandaloneMode {
+			c.Next()
+			return
+		}
+
+		token := c.GetHeader("x-session-token")
+		if token == "" {
+			c.AbortWithStatusJSON(401, gin.H{"error": "token header is empty"})
+			return
+		}
+
+		if _, isTokenCached := cachedValidTokens.Get(token); isTokenCached {
+			c.Next()
+			return
+		}
+
+		if isTokenValid, err := providers.VerifyToken(token, c.Request.Context()); err != nil {
+			logger.Log.Errorf("error verifying token %s", err)
+			c.AbortWithStatusJSON(401, gin.H{"error": "unknown auth error occured"})
+			return
+		} else if !isTokenValid {
+			c.AbortWithStatusJSON(401, gin.H{"error": "invalid token"})
+			return
+		}
+
+		cachedValidTokens.Set(token, true, cachedValidTokensRetainmentTime)
+
+		c.Next()
+	}
+}

agent/pkg/models/models.go

@@ -2,29 +2,44 @@ package models
 
 import (
 	"encoding/json"
+	"mizuserver/pkg/rules"
 
 	tapApi "github.com/up9inc/mizu/tap/api"
 
-	"mizuserver/pkg/rules"
-
 	"github.com/google/martian/har"
+	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
 )
 
-func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error {
+func GetEntry(r *tapApi.Entry, v tapApi.DataUnmarshaler) error {
 	return v.UnmarshalData(r)
 }
 
-type EntriesFilter struct {
-	Limit     int    `form:"limit" validate:"required,min=1,max=200"`
-	Operator  string `form:"operator" validate:"required,oneof='lt' 'gt'"`
-	Timestamp int64  `form:"timestamp" validate:"required,min=1"`
+type TapConfig struct {
+	TappedNamespaces map[string]bool `json:"tappedNamespaces"`
+}
+
+type EntriesRequest struct {
+	LeftOff   int    `form:"leftOff" validate:"required,min=-1"`
+	Direction int    `form:"direction" validate:"required,oneof='1' '-1'"`
+	Query     string `form:"query"`
+	Limit     int    `form:"limit" validate:"required,min=1"`
+	TimeoutMs int    `form:"timeoutMs" validate:"min=1"`
+}
+
+type SingleEntryRequest struct {
+	Query string `form:"query"`
+}
+
+type EntriesResponse struct {
+	Data []interface{}      `json:"data"`
+	Meta *basenine.Metadata `json:"meta"`
 }
 
 type WebSocketEntryMessage struct {
 	*shared.WebSocketMessageMetadata
-	Data *tapApi.BaseEntryDetails `json:"data,omitempty"`
+	Data *tapApi.BaseEntry `json:"data,omitempty"`
 }
 
 type WebSocketTappedEntryMessage struct {
@@ -42,7 +57,28 @@ type AuthStatus struct {
 	Model string `json:"model"`
 }
 
-func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntryDetails) ([]byte, error) {
+type ToastMessage struct {
+	Type      string `json:"type"`
+	AutoClose uint   `json:"autoClose"`
+	Text      string `json:"text"`
+}
+
+type WebSocketToastMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *ToastMessage `json:"data,omitempty"`
+}
+
+type WebSocketQueryMetadataMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *basenine.Metadata `json:"data,omitempty"`
+}
+
+type WebSocketStartTimeMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data int64 `json:"data"`
+}
+
+func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) {
 	message := &WebSocketEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
 			MessageType: shared.WebSocketMessageTypeEntry,
@@ -72,6 +108,36 @@ func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error)
 	return json.Marshal(message)
 }
 
+func CreateWebsocketToastMessage(base *ToastMessage) ([]byte, error) {
+	message := &WebSocketToastMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeToast,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+func CreateWebsocketQueryMetadataMessage(base *basenine.Metadata) ([]byte, error) {
+	message := &WebSocketQueryMetadataMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeQueryMetadata,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+func CreateWebsocketStartTimeMessage(base int64) ([]byte, error) {
+	message := &WebSocketStartTimeMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeStartTime,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
 // ExtendedHAR is the top level object of a HAR log.
 type ExtendedHAR struct {
 	Log *ExtendedLog `json:"log"`
@@ -97,3 +163,7 @@ func RunValidationRulesState(harEntry har.Entry, service string) (tapApi.Applica
 	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend)
 	return tapApi.ApplicableRules{Status: statusPolicyToSend, Latency: latency, NumberOfRules: numberOfRules}, resultPolicyToSend, isEnabled
 }
+
+type InstallState struct {
+	Completed bool `json:"completed"`
+}

agent/pkg/oas/feeder_test.go

@@ -0,0 +1,158 @@
+package oas
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func getFiles(baseDir string) (result []string, err error) {
+	result = make([]string, 0, 0)
+	logger.Log.Infof("Reading files from tree: %s", baseDir)
+
+	// https://yourbasic.org/golang/list-files-in-directory/
+	err = filepath.Walk(baseDir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			ext := strings.ToLower(filepath.Ext(path))
+			if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
+				result = append(result, path)
+			}
+
+			return nil
+		})
+
+	sort.SliceStable(result, func(i, j int) bool {
+		return fileSize(result[i]) < fileSize(result[j])
+	})
+
+	logger.Log.Infof("Got files: %d", len(result))
+	return result, err
+}
+
+func fileSize(fname string) int64 {
+	fi, err := os.Stat(fname)
+	if err != nil {
+		panic(err)
+	}
+
+	return fi.Size()
+}
+
+func feedEntries(fromFiles []string) (err error) {
+	for _, file := range fromFiles {
+		logger.Log.Info("Processing file: " + file)
+		ext := strings.ToLower(filepath.Ext(file))
+		switch ext {
+		case ".har":
+			err = feedFromHAR(file)
+			if err != nil {
+				logger.Log.Warning("Failed processing file: " + err.Error())
+				continue
+			}
+		case ".ldjson":
+			err = feedFromLDJSON(file)
+			if err != nil {
+				logger.Log.Warning("Failed processing file: " + err.Error())
+				continue
+			}
+		default:
+			return errors.New("Unsupported file extension: " + ext)
+		}
+	}
+
+	return nil
+}
+
+func feedFromHAR(file string) error {
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		return err
+	}
+
+	var harDoc har.HAR
+	err = json.Unmarshal(data, &harDoc)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range harDoc.Log.Entries {
+		GetOasGeneratorInstance().PushEntry(entry)
+	}
+
+	return nil
+}
+
+func feedFromLDJSON(file string) error {
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	reader := bufio.NewReader(fd)
+
+	var meta map[string]interface{}
+
+	buf := strings.Builder{}
+	for {
+		substr, isPrefix, err := reader.ReadLine()
+		if err == io.EOF {
+			break
+		}
+
+		buf.WriteString(string(substr))
+		if isPrefix {
+			continue
+		}
+
+		line := buf.String()
+		buf.Reset()
+
+		if meta == nil {
+			err := json.Unmarshal([]byte(line), &meta)
+			if err != nil {
+				return err
+			}
+		} else {
+			var entry har.Entry
+			err := json.Unmarshal([]byte(line), &entry)
+			if err != nil {
+				logger.Log.Warningf("Failed decoding entry: %s", line)
+			}
+			GetOasGeneratorInstance().PushEntry(&entry)
+		}
+	}
+
+	return nil
+}
+
+func TestFilesList(t *testing.T) {
+	res, err := getFiles("./test_artifacts/")
+	t.Log(len(res))
+	t.Log(res)
+	if err != nil || len(res) != 2 {
+		t.Logf("Should return 2 files but returned %d", len(res))
+		t.FailNow()
+	}
+}

agent/pkg/oas/gibberish.go

@@ -0,0 +1,108 @@
+package oas
+
+import (
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+var (
+	patBase64   = regexp.MustCompile(`^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$`)
+	patUuid4    = regexp.MustCompile(`(?i)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
+	patEmail    = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
+	patHexLower = regexp.MustCompile(`(0x)?[0-9a-f]{6,}`)
+	patHexUpper = regexp.MustCompile(`(0x)?[0-9A-F]{6,}`)
+)
+
+func IsGibberish(str string) bool {
+	if patBase64.MatchString(str) && len(str) > 32 {
+		return true
+	}
+
+	if patUuid4.MatchString(str) {
+		return true
+	}
+
+	if patEmail.MatchString(str) {
+		return true
+	}
+
+	if patHexLower.MatchString(str) || patHexUpper.MatchString(str) {
+		return true
+	}
+
+	noise := noiseLevel(str)
+	if noise >= 0.2 {
+		return true
+	}
+
+	return false
+}
+
+func noiseLevel(str string) (score float64) {
+	// opinionated algo of certain char pairs marking the non-human strings
+	prev := *new(rune)
+	cnt := 0.0
+	for _, char := range str {
+		cnt += 1
+		if prev > 0 {
+			switch {
+			// continued class of upper/lower/digit adds no noise
+			case unicode.IsUpper(prev) && unicode.IsUpper(char):
+			case unicode.IsLower(prev) && unicode.IsLower(char):
+			case unicode.IsDigit(prev) && unicode.IsDigit(char):
+
+			// upper =>
+			case unicode.IsUpper(prev) && unicode.IsLower(char):
+				score += 0.25
+			case unicode.IsUpper(prev) && unicode.IsDigit(char):
+				score += 0.25
+
+			// lower =>
+			case unicode.IsLower(prev) && unicode.IsUpper(char):
+				score += 0.75
+			case unicode.IsLower(prev) && unicode.IsDigit(char):
+				score += 0.25
+
+			// digit =>
+			case unicode.IsDigit(prev) && unicode.IsUpper(char):
+				score += 0.75
+			case unicode.IsDigit(prev) && unicode.IsLower(char):
+				score += 0.75
+
+			// the rest is 100% noise
+			default:
+				score += 1
+			}
+		}
+		prev = char
+	}
+
+	score /= cnt // weigh it
+
+	return score
+}
+
+func IsVersionString(component string) bool {
+	if component == "" {
+		return false
+	}
+
+	hasV := false
+	if strings.HasPrefix(component, "v") {
+		component = component[1:]
+		hasV = true
+	}
+
+	for _, c := range component {
+		if string(c) != "." && !unicode.IsDigit(c) {
+			return false
+		}
+	}
+
+	if !hasV && strings.Contains(component, ".") {
+		return false
+	}
+
+	return true
+}

agent/pkg/oas/gibberish_test.go

@@ -0,0 +1,71 @@
+package oas
+
+import "testing"
+
+func TestNegative(t *testing.T) {
+	cases := []string{
+		"",
+		"b", // can be valid hexadecimal
+		"GetUniversalVariableUser",
+		"callback",
+		"runs",
+		"tcfv2",
+		"StartUpCheckout",
+		"GetCart",
+	}
+
+	for _, str := range cases {
+		if IsGibberish(str) {
+			t.Errorf("Mistakenly true: %s", str)
+		}
+	}
+}
+
+func TestPositive(t *testing.T) {
+	cases := []string{
+		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
+		"952bea17-3776-11ea-9341-42010a84012a",
+		"456795af-b48f-4a8d-9b37-3e932622c2f0",
+		"0a0d0174-b338-4520-a1c3-24f7e3d5ec50.html",
+		"6120c057c7a97b03f6986f1b",
+		"610bc3fd5a77a7fa25033fb0",
+		"610bd0315a77a7fa25034368",
+		"610bd0315a77a7fa25034368zh",
+		"710a462e",
+		"1554507871",
+		"qwerqwerasdfqwer@protonmai.com",
+		"john.dow.1981@protonmail.com",
+		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
+		"11ca096cbc224a67360493d44a9903",
+		"c738338322370b47a79251f7510dd", // prefixed hex
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
+		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
+		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
+		"730970532670-compute@developer.gserviceaccount.com",
+		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
+		"AAAA028295945",
+		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
+		"n63nd45qsj",
+		"n9z9QGNiz",
+		"proxy.3d2100fd7107262ecb55ce6847f01fa5.html",
+		"r-ext-5579e00a95c90",
+		"r-ext-5579e8b12f11e",
+		"r-v4-5c92513c9e04a",
+		"r-v4-5c92513c9e04a-0-c.monitoring.json",
+		"segments-1563566437171.639994",
+		"t_52d94268-8810-4a7e-ba87-ffd657a6752f",
+		"timeouts-1563566437171.639994",
+
+		// TODO
+		// "fb6cjraf9cejut2a",
+		// "Fxvd1timk", // questionable
+		// "JEHJW4BKVFDRTMTUQLHKK5WVAU",
+	}
+
+	for _, str := range cases {
+		if !IsGibberish(str) {
+			t.Errorf("Mistakenly false: %s", str)
+		}
+	}
+}

agent/pkg/oas/ignores.go

@@ -0,0 +1,75 @@
+package oas
+
+import "strings"
+
+var ignoredExtensions = []string{"gif", "svg", "css", "png", "ico", "js", "woff2", "woff", "jpg", "jpeg", "swf", "ttf", "map", "webp", "otf", "mp3"}
+
+var ignoredCtypePrefixes = []string{"image/", "font/", "video/", "audio/", "text/javascript"}
+var ignoredCtypes = []string{"application/javascript", "application/x-javascript", "text/css", "application/font-woff2", "application/font-woff", "application/x-font-woff"}
+
+var ignoredHeaders = []string{
+	"a-im", "accept",
+	"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-type", "cookie",
+	"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
+	"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
+	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests",
+	"server", "user-agent", "via", "warning", "strict-transport-security",
+	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data",
+	"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
+	"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
+	"x-wap-profile", "x-scheme",
+	"newrelic", "x-cloud-trace-context", "sentry-trace",
+	"expires", "set-cookie", "p3p", "location", "content-security-policy", "content-security-policy-report-only",
+	"last-modified", "content-language",
+	"keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by",
+	"age", "allow", "www-authenticate",
+}
+
+var ignoredHeaderPrefixes = []string{
+	":", "accept-", "access-control-", "if-", "sec-", "grpc-",
+	"x-forwarded-", "x-original-",
+	"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
+	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-",
+}
+
+func isCtypeIgnored(ctype string) bool {
+	for _, prefix := range ignoredCtypePrefixes {
+		if strings.HasPrefix(ctype, prefix) {
+			return true
+		}
+	}
+
+	for _, toIgnore := range ignoredCtypes {
+		if ctype == toIgnore {
+			return true
+		}
+	}
+	return false
+}
+
+func isExtIgnored(path string) bool {
+	for _, extIgn := range ignoredExtensions {
+		if strings.HasSuffix(path, "."+extIgn) {
+			return true
+		}
+	}
+	return false
+}
+
+func isHeaderIgnored(name string) bool {
+	name = strings.ToLower(name)
+
+	for _, ignore := range ignoredHeaders {
+		if name == ignore {
+			return true
+		}
+	}
+
+	for _, prefix := range ignoredHeaderPrefixes {
+		if strings.HasPrefix(name, prefix) {
+			return true
+		}
+	}
+
+	return false
+}

agent/pkg/oas/oas_generator.go

@@ -0,0 +1,107 @@
+package oas
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"net/url"
+	"sync"
+)
+
+var (
+	syncOnce sync.Once
+	instance *oasGenerator
+)
+
+func GetOasGeneratorInstance() *oasGenerator {
+	syncOnce.Do(func() {
+		instance = newOasGenerator()
+		logger.Log.Debug("Oas Generator Initialized")
+	})
+	return instance
+}
+
+func (g *oasGenerator) Start() {
+	if g.started {
+		return
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	g.cancel = cancel
+	g.ctx = ctx
+	g.entriesChan = make(chan har.Entry, 100) // buffer up to 100 entries for OAS processing
+	g.ServiceSpecs = &sync.Map{}
+	g.started = true
+	go instance.runGeneretor()
+}
+
+func (g *oasGenerator) runGeneretor() {
+	for {
+		select {
+		case <-g.ctx.Done():
+			logger.Log.Infof("OAS Generator was canceled")
+			return
+
+		case entry, ok := <-g.entriesChan:
+			if !ok {
+				logger.Log.Infof("OAS Generator - entries channel closed")
+				break
+			}
+			u, err := url.Parse(entry.Request.URL)
+			if err != nil {
+				logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
+			}
+
+			val, found := g.ServiceSpecs.Load(u.Host)
+			var gen *SpecGen
+			if !found {
+				gen = NewGen(u.Scheme + "://" + u.Host)
+				g.ServiceSpecs.Store(u.Host, gen)
+			} else {
+				gen = val.(*SpecGen)
+			}
+
+			opId, err := gen.feedEntry(entry)
+			if err != nil {
+				txt, suberr := json.Marshal(entry)
+				if suberr == nil {
+					logger.Log.Debugf("Problematic entry: %s", txt)
+				}
+
+				logger.Log.Warningf("Failed processing entry: %s", err)
+				continue
+			}
+
+			logger.Log.Debugf("Handled entry %s as opId: %s", entry.Request.URL, opId) // TODO: set opId back to entry?
+		}
+	}
+}
+
+func (g *oasGenerator) PushEntry(entry *har.Entry) {
+	if !g.started {
+		return
+	}
+	select {
+	case g.entriesChan <- *entry:
+	default:
+		logger.Log.Warningf("OAS Generator - entry wasn't sent to channel because the channel has no buffer or there is no receiver")
+	}
+}
+
+func newOasGenerator() *oasGenerator {
+	return &oasGenerator{
+		started:      false,
+		ctx:          nil,
+		cancel:       nil,
+		ServiceSpecs: nil,
+		entriesChan:  nil,
+	}
+}
+
+type oasGenerator struct {
+	started      bool
+	ctx          context.Context
+	cancel       context.CancelFunc
+	ServiceSpecs *sync.Map
+	entriesChan  chan har.Entry
+}

agent/pkg/oas/specgen.go

@@ -0,0 +1,497 @@
+package oas
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/google/uuid"
+	"github.com/up9inc/mizu/shared/logger"
+	"mime"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+type reqResp struct { // hello, generics in Go
+	Req  *har.Request
+	Resp *har.Response
+}
+
+type SpecGen struct {
+	oas  *openapi.OpenAPI
+	tree *Node
+	lock sync.Mutex
+}
+
+func NewGen(server string) *SpecGen {
+	spec := new(openapi.OpenAPI)
+	spec.Version = "3.1.0"
+
+	info := openapi.Info{Title: server}
+	info.Version = "0.0"
+	spec.Info = &info
+	spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
+
+	spec.Servers = make([]*openapi.Server, 0)
+	spec.Servers = append(spec.Servers, &openapi.Server{URL: server})
+
+	gen := SpecGen{oas: spec, tree: new(Node)}
+	return &gen
+}
+
+func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
+	g.oas = oas
+	for pathStr, pathObj := range oas.Paths.Items {
+		pathSplit := strings.Split(string(pathStr), "/")
+		g.tree.getOrSet(pathSplit, pathObj)
+	}
+}
+
+func (g *SpecGen) feedEntry(entry har.Entry) (string, error) {
+	g.lock.Lock()
+	defer g.lock.Unlock()
+
+	opId, err := g.handlePathObj(&entry)
+	if err != nil {
+		return "", err
+	}
+
+	// NOTE: opId can be empty for some failed entries
+	return opId, err
+}
+
+func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
+	g.lock.Lock()
+	defer g.lock.Unlock()
+
+	g.tree.compact()
+
+	for _, pathop := range g.tree.listOps() {
+		if pathop.op.Summary == "" {
+			pathop.op.Summary = pathop.path
+		}
+	}
+
+	// put paths back from tree into OAS
+	g.oas.Paths = g.tree.listPaths()
+
+	suggestTags(g.oas)
+
+	// to make a deep copy, no better idea than marshal+unmarshal
+	specText, err := json.MarshalIndent(g.oas, "", "\t")
+	if err != nil {
+		return nil, err
+	}
+
+	spec := new(openapi.OpenAPI)
+	err = json.Unmarshal(specText, spec)
+	if err != nil {
+		return nil, err
+	}
+
+	return spec, err
+}
+
+func suggestTags(oas *openapi.OpenAPI) {
+	paths := getPathsKeys(oas.Paths.Items)
+	for len(paths) > 0 {
+		group := make([]string, 0)
+		group = append(group, paths[0])
+		paths = paths[1:]
+
+		pathsClone := append(paths[:0:0], paths...)
+		for _, path := range pathsClone {
+			if getSimilarPrefix([]string{group[0], path}) != "" {
+				group = append(group, path)
+				paths = deleteFromSlice(paths, path)
+			}
+		}
+
+		common := getSimilarPrefix(group)
+
+		if len(group) > 1 {
+			for _, path := range group {
+				pathObj := oas.Paths.Items[openapi.PathValue(path)]
+				for _, op := range getOps(pathObj) {
+					if op.Tags == nil {
+						op.Tags = make([]string, 0)
+					}
+					// only add tags if not present
+					if len(op.Tags) == 0 {
+						op.Tags = append(op.Tags, common)
+					}
+				}
+			}
+		}
+
+		//groups[common] = group
+	}
+}
+
+func getSimilarPrefix(strs []string) string {
+	chunked := make([][]string, 0)
+	for _, item := range strs {
+		chunked = append(chunked, strings.Split(item, "/"))
+	}
+
+	cmn := longestCommonXfix(chunked, true)
+	res := make([]string, 0)
+	for _, chunk := range cmn {
+		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
+			res = append(res, chunk)
+		}
+	}
+	return strings.Join(res[1:], ".")
+}
+
+func deleteFromSlice(s []string, val string) []string {
+	temp := s[:0]
+	for _, x := range s {
+		if x != val {
+			temp = append(temp, x)
+		}
+	}
+	return temp
+}
+
+func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
+	keys := make([]string, len(mymap))
+
+	i := 0
+	for k := range mymap {
+		keys[i] = string(k)
+		i++
+	}
+	return keys
+}
+
+func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
+	urlParsed, err := url.Parse(entry.Request.URL)
+	if err != nil {
+		return "", err
+	}
+
+	if isExtIgnored(urlParsed.Path) {
+		logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
+	}
+
+	ctype := getRespCtype(entry.Response)
+	if isCtypeIgnored(ctype) {
+		logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
+	}
+
+	if entry.Response.Status < 100 {
+		logger.Log.Debugf("Dropped traffic entry due to status<100: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	if entry.Response.Status == 301 || entry.Response.Status == 308 {
+		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	split := strings.Split(urlParsed.Path, "/")
+	node := g.tree.getOrSet(split, new(openapi.PathObj))
+	opObj, err := handleOpObj(entry, node.ops)
+
+	if opObj != nil {
+		return opObj.OperationID, err
+	}
+
+	return "", err
+}
+
+func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation, error) {
+	isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
+	opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
+	if err != nil {
+		return nil, err
+	}
+
+	if !isSuccess && wasMissing {
+		logger.Log.Debugf("Dropped traffic entry due to failed status and no known endpoint at: %s", entry.StartedDateTime)
+		return nil, nil
+	}
+
+	err = handleRequest(entry.Request, opObj, isSuccess)
+	if err != nil {
+		return nil, err
+	}
+
+	err = handleResponse(entry.Response, opObj, isSuccess)
+	if err != nil {
+		return nil, err
+	}
+
+	return opObj, nil
+}
+
+func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error {
+	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
+
+	qstrGW := nvParams{
+		In: openapi.InQuery,
+		Pairs: func() []NVPair {
+			return qstrToNVP(req.QueryString)
+		},
+		IsIgnored:      func(name string) bool { return false },
+		GeneralizeName: func(name string) string { return name },
+	}
+	handleNameVals(qstrGW, &opObj.Parameters)
+
+	hdrGW := nvParams{
+		In: openapi.InHeader,
+		Pairs: func() []NVPair {
+			return hdrToNVP(req.Headers)
+		},
+		IsIgnored:      isHeaderIgnored,
+		GeneralizeName: strings.ToLower,
+	}
+	handleNameVals(hdrGW, &opObj.Parameters)
+
+	if req.PostData != nil && req.PostData.Text != "" && isSuccess {
+		reqBody, err := getRequestBody(req, opObj, isSuccess)
+		if err != nil {
+			return err
+		}
+
+		if reqBody != nil {
+			reqCtype := getReqCtype(req)
+			reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, err)
+			if err != nil {
+				return err
+			}
+
+			_ = reqMedia
+		}
+	}
+	return nil
+}
+
+func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool) error {
+	// TODO: we don't support "default" response
+	respObj, err := getResponseObj(resp, opObj, isSuccess)
+	if err != nil {
+		return err
+	}
+
+	handleRespHeaders(resp.Headers, respObj)
+
+	respCtype := getRespCtype(resp)
+	respContent := respObj.Content
+	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, err)
+	if err != nil {
+		return err
+	}
+	_ = respMedia
+	return nil
+}
+
+func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj) {
+	visited := map[string]*openapi.HeaderObj{}
+	for _, pair := range reqHeaders {
+		if isHeaderIgnored(pair.Name) {
+			continue
+		}
+
+		nameGeneral := strings.ToLower(pair.Name)
+
+		initHeaders(respObj)
+		objHeaders := respObj.Headers
+		param := findHeaderByName(&respObj.Headers, pair.Name)
+		if param == nil {
+			param = createHeader(openapi.TypeString)
+			objHeaders[nameGeneral] = param
+		}
+		exmp := &param.Examples
+		err := fillParamExample(&exmp, pair.Value)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+		visited[nameGeneral] = param
+	}
+
+	// maintain "required" flag
+	if respObj.Headers != nil {
+		for name, param := range respObj.Headers {
+			paramObj, err := param.ResolveHeader(headerResolver)
+			if err != nil {
+				logger.Log.Warningf("Failed to resolve param: %s", err)
+				continue
+			}
+
+			_, ok := visited[strings.ToLower(name)]
+			if !ok {
+				flag := false
+				paramObj.Required = &flag
+			}
+		}
+	}
+
+	return
+}
+
+func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err error) (*openapi.MediaType, error) {
+	content, found := respContent[ctype]
+	if !found {
+		respContent[ctype] = &openapi.MediaType{}
+		content = respContent[ctype]
+	}
+
+	var text string
+	if reqResp.Req != nil {
+		text = reqResp.Req.PostData.Text
+	} else {
+		text = decRespText(reqResp.Resp.Content)
+	}
+
+	var exampleMsg []byte
+	// try treating it as json
+	any, isJSON := anyJSON(text)
+	if isJSON {
+		// re-marshal with forced indent
+		exampleMsg, err = json.MarshalIndent(any, "", "\t")
+		if err != nil {
+			panic("Failed to re-marshal value, super-strange")
+		}
+	} else {
+		exampleMsg, err = json.Marshal(text)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	content.Example = exampleMsg
+	return respContent[ctype], nil
+}
+
+func decRespText(content *har.Content) (res string) {
+	res = string(content.Text)
+	if content.Encoding == "base64" {
+		data, err := base64.StdEncoding.DecodeString(res)
+		if err != nil {
+			logger.Log.Warningf("error decoding response text as base64: %s", err)
+		} else {
+			res = string(data)
+		}
+	}
+	return
+}
+
+func getRespCtype(resp *har.Response) string {
+	var ctype string
+	ctype = resp.Content.MimeType
+	for _, hdr := range resp.Headers {
+		if strings.ToLower(hdr.Name) == "content-type" {
+			ctype = hdr.Value
+		}
+	}
+
+	mediaType, _, err := mime.ParseMediaType(ctype)
+	if err != nil {
+		return ""
+	}
+	return mediaType
+}
+
+func getReqCtype(req *har.Request) string {
+	var ctype string
+	ctype = req.PostData.MimeType
+	for _, hdr := range req.Headers {
+		if strings.ToLower(hdr.Name) == "content-type" {
+			ctype = hdr.Value
+		}
+	}
+
+	mediaType, _, err := mime.ParseMediaType(ctype)
+	if err != nil {
+		return ""
+	}
+	return mediaType
+}
+
+func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool) (*openapi.ResponseObj, error) {
+	statusStr := strconv.Itoa(resp.Status)
+
+	var response openapi.Response
+	response, found := opObj.Responses[statusStr]
+	if !found {
+		if opObj.Responses == nil {
+			opObj.Responses = map[string]openapi.Response{}
+		}
+
+		opObj.Responses[statusStr] = &openapi.ResponseObj{Content: map[string]*openapi.MediaType{}}
+		response = opObj.Responses[statusStr]
+	}
+
+	resResponse, err := response.ResolveResponse(responseResolver)
+	if err != nil {
+		return nil, err
+	}
+
+	if isSuccess {
+		resResponse.Description = "Successful call with status " + statusStr
+	} else {
+		resResponse.Description = "Failed call with status " + statusStr
+	}
+	return resResponse, nil
+}
+
+func getRequestBody(req *har.Request, opObj *openapi.Operation, isSuccess bool) (*openapi.RequestBodyObj, error) {
+	if opObj.RequestBody == nil {
+		opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
+	}
+
+	reqBody, err := opObj.RequestBody.ResolveRequestBody(reqBodyResolver)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: maintain required flag for it, but only consider successful responses
+	//reqBody.Content[]
+
+	return reqBody, nil
+}
+
+func getOpObj(pathObj *openapi.PathObj, method string, createIfNone bool) (*openapi.Operation, bool, error) {
+	method = strings.ToLower(method)
+	var op **openapi.Operation
+
+	switch method {
+	case "get":
+		op = &pathObj.Get
+	case "put":
+		op = &pathObj.Put
+	case "post":
+		op = &pathObj.Post
+	case "delete":
+		op = &pathObj.Delete
+	case "options":
+		op = &pathObj.Options
+	case "head":
+		op = &pathObj.Head
+	case "patch":
+		op = &pathObj.Patch
+	case "trace":
+		op = &pathObj.Trace
+	default:
+		return nil, false, errors.New("unsupported HTTP method: " + method)
+	}
+
+	isMissing := false
+	if *op == nil {
+		isMissing = true
+		if createIfNone {
+			*op = &openapi.Operation{Responses: map[string]openapi.Response{}}
+			newUUID := uuid.New().String()
+			(**op).OperationID = newUUID
+		} else {
+			return nil, isMissing, nil
+		}
+	}
+
+	return *op, isMissing, nil
+}

agent/pkg/oas/specgen_test.go

@@ -0,0 +1,197 @@
+package oas
+
+import (
+	"encoding/json"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+)
+
+// if started via env, write file into subdir
+func writeFiles(label string, spec *openapi.OpenAPI) {
+	if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
+		path := "./oas-samples"
+		err := os.MkdirAll(path, 0o755)
+		if err != nil {
+			panic(err)
+		}
+		content, err := json.MarshalIndent(spec, "", "\t")
+		if err != nil {
+			panic(err)
+		}
+		err = ioutil.WriteFile(path+"/"+label+".json", content, 0644)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+func TestEntries(t *testing.T) {
+	files, err := getFiles("./test_artifacts/")
+	// files, err = getFiles("/media/bigdisk/UP9")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	GetOasGeneratorInstance().Start()
+
+	if err := feedEntries(files); err != nil {
+		t.Log(err)
+		t.Fail()
+	}
+
+	loadStartingOAS()
+
+	svcs := strings.Builder{}
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+		gen := val.(*SpecGen)
+		svc := key.(string)
+		svcs.WriteString(svc + ",")
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+			return false
+		}
+
+		err = spec.Validate()
+		if err != nil {
+			specText, _ := json.MarshalIndent(spec, "", "\t")
+			t.Log(string(specText))
+			t.Log(err)
+			t.FailNow()
+		}
+
+		return true
+	})
+
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+		svc := key.(string)
+		gen := val.(*SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		specText, _ := json.MarshalIndent(spec, "", "\t")
+		t.Logf("%s", string(specText))
+
+		err = spec.Validate()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+		writeFiles(svc, spec)
+
+		return true
+	})
+
+}
+
+func TestFileLDJSON(t *testing.T) {
+	GetOasGeneratorInstance().Start()
+	file := "test_artifacts/output_rdwtyeoyrj.har.ldjson"
+	err := feedFromLDJSON(file)
+	if err != nil {
+		logger.Log.Warning("Failed processing file: " + err.Error())
+		t.Fail()
+	}
+
+	loadStartingOAS()
+
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(_, val interface{}) bool {
+		gen := val.(*SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		specText, _ := json.MarshalIndent(spec, "", "\t")
+		t.Logf("%s", string(specText))
+
+		err = spec.Validate()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		return true
+	})
+}
+
+func loadStartingOAS() {
+	file := "test_artifacts/catalogue.json"
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		panic(err)
+	}
+
+	var doc *openapi.OpenAPI
+	err = json.Unmarshal(data, &doc)
+	if err != nil {
+		panic(err)
+	}
+
+	gen := NewGen("catalogue")
+	gen.StartFromSpec(doc)
+
+	GetOasGeneratorInstance().ServiceSpecs.Store("catalogue", gen)
+
+	return
+}
+
+func TestEntriesNegative(t *testing.T) {
+	files := []string{"invalid"}
+	err := feedEntries(files)
+	if err == nil {
+		t.Logf("Should have failed")
+		t.Fail()
+	}
+}
+
+func TestLoadValidHAR(t *testing.T) {
+	inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}`
+	var entry *har.Entry
+	var err = json.Unmarshal([]byte(inp), &entry)
+	if err != nil {
+		t.Logf("Failed to decode entry: %s", err)
+		// t.FailNow() demonstrates the problem of library
+	}
+}
+
+func TestLoadValid3_1(t *testing.T) {
+	fd, err := os.Open("test_artifacts/catalogue.json")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	var oas openapi.OpenAPI
+	err = json.Unmarshal(data, &oas)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	return
+}

agent/pkg/oas/test_artifacts/catalogue.json

@@ -0,0 +1,51 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "Preloaded",
+    "version": "0.1",
+    "description": "Test file for loading pre-existing OAS"
+  },
+  "paths": {
+    "/catalogue/{id}": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ],
+      "get": {
+        "parameters": [        {
+          "name": "non-required-header",
+          "in": "header",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+        ]
+      }
+    },
+    "/catalogue/{id}/details": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "style": "simple",
+          "required": true,
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ]
+    }
+  }
+}

agent/pkg/oas/test_artifacts/output_rdwtyeoyrj.har.ldjson

@@ -0,0 +1,13 @@
+{"messageType": "http", "_source": null, "firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78415179Z", "time": 13, "request": {"method": "GET", "url": "http://catalogue/catalogue/size?tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "x-some", "value": "demo val"},{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [{"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "22"}], "content": {"size": 22, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInNpemUiOjl9", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 22}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 13}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.784918698Z", "time": 19, "request": {"method": "GET", "url": "http://catalogue/catalogue?page=1&size=6&tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "page", "value": "1"}, {"name": "size", "value": "6"}, {"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "1927"}], "content": {"size": 1927, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19LHsiaWQiOiI4MzdhYjE0MS0zOTllLTRjMWYtOWFiYy1iYWNlNDAyOTZiYWMiLCJuYW1lIjoiQ2F0IHNvY2tzIiwiZGVzY3JpcHRpb24iOiJjb25zZXF1YXQgYW1ldCBjdXBpZGF0YXQgbWluaW0gbGFib3J1bSB0ZW1wb3IgZWxpdCBleCBjb25zZXF1YXQgaW4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2NrczIuanBnIl0sInByaWNlIjoxNSwiY291bnQiOjE3NSwidGFnIjpbImJyb3duIiwiZm9ybWFsIiwiZ3JlZW4iXX1dCg==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1927}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 19}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78418182Z", "time": 7, "request": {"method": "GET", "url": "http://catalogue/tags", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "107"}], "content": {"size": 107, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInRhZ3MiOlsiYnJvd24iLCJnZWVrIiwiZm9ybWFsIiwiYmx1ZSIsInNraW4iLCJyZWQiLCJhY3Rpb24iLCJzcG9ydCIsImJsYWNrIiwibWFnaWMiLCJncmVlbiJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 107}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 7}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:18.131501482Z", "time": 5, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}, {"name": "x-some", "value": "demoval"}], ",queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 5}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:18.379836908Z", "time": 14, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 14}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.920540124Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/808a2de1-1aaa-4c25-a9b9-6612e8f29a38", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "275"}], "content": {"size": 275, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NzM4LCJkZXNjcmlwdGlvbiI6IkEgbWF0dXJlIHNvY2ssIGNyb3NzZWQsIHdpdGggYW4gYWlyIG9mIG5vbmNoYWxhbmNlLiIsImlkIjoiODA4YTJkZTEtMWFhYS00YzI1LWE5YjktNjYxMmU4ZjI5YTM4IiwiaW1hZ2VVcmwiOlsiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMS5qcGVnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMi5qcGVnIl0sIm5hbWUiOiJDcm9zc2VkIiwicHJpY2UiOjE3LjMyLCJ0YWciOlsiYmx1ZSIsInJlZCIsImFjdGlvbiIsImZvcm1hbCJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 275}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.921609501Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?sort=id&size=3&tags=blue", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "3"}, {"name": "tags", "value": "blue"}, {"name": "sort", "value": "id"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "789"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}], "content": {"size": 789, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJuYW1lIjoiQ29sb3VyZnVsIiwiZGVzY3JpcHRpb24iOiJwcm9pZGVudCBvY2NhZWNhdCBpcnVyZSBldCBleGNlcHRldXIgbGFib3JlIG1pbmltIG5pc2kgYW1ldCBpcnVyZSIsImltYWdlVXJsIjpbIi9jYXRhbG9ndWUvaW1hZ2VzL2NvbG91cmZ1bF9zb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIl0sInByaWNlIjoxOCwiY291bnQiOjQzOCwidGFnIjpbImJsdWUiXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiYmx1ZSJdfV0K", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 789}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.923197848Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:23.175549218Z", "time": 26, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 26}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.239777333Z", "time": 10, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Transfer-Encoding", "value": "chunked"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "X-Application-Context", "value": "carts:80"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 10}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.725866772Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?size=5", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "5"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "1643"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}], "content": {"size": 1643, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19XQo=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1643}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.729303217Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}

agent/pkg/oas/tree.go

@@ -0,0 +1,205 @@
+package oas
+
+import (
+	"github.com/chanced/openapi"
+	"github.com/up9inc/mizu/shared/logger"
+	"strconv"
+	"strings"
+)
+
+type NodePath = []string
+
+type Node struct {
+	constant *string
+	param    *openapi.ParameterObj
+	ops      *openapi.PathObj
+	parent   *Node
+	children []*Node
+}
+
+func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Node) {
+	if pathObjToSet == nil {
+		panic("Invalid function call")
+	}
+
+	pathChunk := path[0]
+	chunkIsParam := strings.HasPrefix(pathChunk, "{") && strings.HasSuffix(pathChunk, "}")
+	chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
+
+	var paramObj *openapi.ParameterObj
+	if chunkIsParam && pathObjToSet != nil {
+		paramObj = findParamByName(pathObjToSet.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
+	}
+
+	if paramObj == nil {
+		node = n.searchInConstants(pathChunk)
+	}
+
+	if node == nil {
+		node = n.searchInParams(paramObj, chunkIsGibberish)
+	}
+
+	// still no node found, should create it
+	if node == nil {
+		node = new(Node)
+		node.parent = n
+		n.children = append(n.children, node)
+
+		if paramObj != nil {
+			node.param = paramObj
+		} else if chunkIsGibberish {
+			initParams(&pathObjToSet.Parameters)
+
+			newParam := n.createParam()
+			node.param = newParam
+
+			appended := append(*pathObjToSet.Parameters, newParam)
+			pathObjToSet.Parameters = &appended
+		} else {
+			node.constant = &pathChunk
+		}
+	}
+
+	// add example if it's a param
+	if node.param != nil && !chunkIsParam {
+		exmp := &node.param.Examples
+		err := fillParamExample(&exmp, pathChunk)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+	}
+
+	// TODO: eat up trailing slash, in a smart way: node.ops!=nil && path[1]==""
+	if len(path) > 1 {
+		return node.getOrSet(path[1:], pathObjToSet)
+	} else if node.ops == nil {
+		node.ops = pathObjToSet
+	}
+
+	return node
+}
+
+func (n *Node) createParam() *openapi.ParameterObj {
+	name := "param"
+
+	// REST assumption, not always correct
+	if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
+		name = *n.constant
+		name = name[:len(name)-2] + "Id"
+	} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
+		name = *n.constant
+		name = name[:len(name)-1] + "Id"
+	}
+
+	newParam := createSimpleParam(name, "path", "string")
+	x := n.countParentParams()
+	if x > 1 {
+		newParam.Name = newParam.Name + strconv.Itoa(x)
+	}
+
+	return newParam
+}
+
+func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunkIsGibberish bool) *Node {
+	// look among params
+	if paramObj != nil || chunkIsGibberish {
+		for _, subnode := range n.children {
+			if subnode.constant != nil {
+				continue
+			}
+
+			// TODO: check the regex pattern of param? for exceptions etc
+
+			if paramObj != nil {
+				// TODO: mergeParam(subnode.param, paramObj)
+				return subnode
+			} else {
+				return subnode
+			}
+		}
+	}
+	return nil
+}
+
+func (n *Node) searchInConstants(pathChunk string) *Node {
+	// look among constants
+	for _, subnode := range n.children {
+		if subnode.constant == nil {
+			continue
+		}
+
+		if *subnode.constant == pathChunk {
+			return subnode
+		}
+	}
+	return nil
+}
+
+func (n *Node) compact() {
+	// TODO
+}
+
+func (n *Node) listPaths() *openapi.Paths {
+	paths := &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
+
+	var strChunk string
+	if n.constant != nil {
+		strChunk = *n.constant
+	} else if n.param != nil {
+		strChunk = "{" + n.param.Name + "}"
+	} else {
+		// this is the root node
+	}
+
+	// add self
+	if n.ops != nil {
+		paths.Items[openapi.PathValue(strChunk)] = n.ops
+	}
+
+	// recurse into children
+	for _, child := range n.children {
+		subPaths := child.listPaths()
+		for path, pathObj := range subPaths.Items {
+			var concat string
+			if n.parent == nil {
+				concat = string(path)
+			} else {
+				concat = strChunk + "/" + string(path)
+			}
+			paths.Items[openapi.PathValue(concat)] = pathObj
+		}
+	}
+
+	return paths
+}
+
+type PathAndOp struct {
+	path string
+	op   *openapi.Operation
+}
+
+func (n *Node) listOps() []PathAndOp {
+	res := make([]PathAndOp, 0)
+	for path, pathObj := range n.listPaths().Items {
+		for _, op := range getOps(pathObj) {
+			res = append(res, PathAndOp{path: string(path), op: op})
+		}
+	}
+	return res
+}
+
+func (n *Node) countParentParams() int {
+	res := 0
+	node := n
+	for {
+		if node.param != nil {
+			res++
+		}
+
+		if node.parent == nil {
+			break
+		}
+		node = node.parent
+	}
+	return res
+}

agent/pkg/oas/utils.go

@@ -0,0 +1,344 @@
+package oas
+
+import (
+	"encoding/json"
+	"errors"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"strconv"
+	"strings"
+)
+
+func exampleResolver(ref string) (*openapi.ExampleObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func responseResolver(ref string) (*openapi.ResponseObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func reqBodyResolver(ref string) (*openapi.RequestBodyObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func paramResolver(ref string) (*openapi.ParameterObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func headerResolver(ref string) (*openapi.HeaderObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func initParams(obj **openapi.ParameterList) {
+	if *obj == nil {
+		var params openapi.ParameterList
+		params = make([]openapi.Parameter, 0)
+		*obj = &params
+	}
+}
+
+func initHeaders(respObj *openapi.ResponseObj) {
+	if respObj.Headers == nil {
+		var created openapi.Headers
+		created = map[string]openapi.Header{}
+		respObj.Headers = created
+	}
+}
+
+func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *openapi.ParameterObj {
+	if name == "" {
+		panic("Cannot create parameter with empty name")
+	}
+	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
+	schema := new(openapi.SchemaObj)
+	schema.Type = make(openapi.Types, 0)
+	schema.Type = append(schema.Type, ptype)
+
+	style := openapi.StyleSimple
+	if in == openapi.InQuery {
+		style = openapi.StyleForm
+	}
+
+	newParam := openapi.ParameterObj{
+		Name:     name,
+		In:       in,
+		Style:    string(style),
+		Examples: map[string]openapi.Example{},
+		Schema:   schema,
+		Required: &required,
+	}
+	return &newParam
+}
+
+func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (pathParam *openapi.ParameterObj) {
+	caseInsensitive := in == openapi.InHeader
+	for _, param := range *params {
+		paramObj, err := param.ResolveParameter(paramResolver)
+		if err != nil {
+			logger.Log.Warningf("Failed to resolve reference: %s", err)
+			continue
+		}
+
+		if paramObj.In != in {
+			continue
+		}
+
+		if paramObj.Name == name || (caseInsensitive && strings.ToLower(paramObj.Name) == strings.ToLower(name)) {
+			pathParam = paramObj
+			break
+		}
+	}
+	return pathParam
+}
+
+func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj {
+	for hname, param := range *headers {
+		hdrObj, err := param.ResolveHeader(headerResolver)
+		if err != nil {
+			logger.Log.Warningf("Failed to resolve reference: %s", err)
+			continue
+		}
+
+		if strings.ToLower(hname) == strings.ToLower(name) {
+			return hdrObj
+		}
+	}
+	return nil
+}
+
+type NVPair struct {
+	Name  string
+	Value string
+}
+
+type nvParams struct {
+	In             openapi.In
+	Pairs          func() []NVPair
+	IsIgnored      func(name string) bool
+	GeneralizeName func(name string) string
+}
+
+func qstrToNVP(list []har.QueryString) []NVPair {
+	res := make([]NVPair, len(list))
+	for idx, val := range list {
+		res[idx] = NVPair{Name: val.Name, Value: val.Value}
+	}
+	return res
+}
+
+func hdrToNVP(list []har.Header) []NVPair {
+	res := make([]NVPair, len(list))
+	for idx, val := range list {
+		res[idx] = NVPair{Name: val.Name, Value: val.Value}
+	}
+	return res
+}
+
+func handleNameVals(gw nvParams, params **openapi.ParameterList) {
+	visited := map[string]*openapi.ParameterObj{}
+	for _, pair := range gw.Pairs() {
+		if gw.IsIgnored(pair.Name) {
+			continue
+		}
+
+		nameGeneral := gw.GeneralizeName(pair.Name)
+
+		initParams(params)
+		param := findParamByName(*params, gw.In, pair.Name)
+		if param == nil {
+			param = createSimpleParam(nameGeneral, gw.In, openapi.TypeString)
+			appended := append(**params, param)
+			*params = &appended
+		}
+		exmp := &param.Examples
+		err := fillParamExample(&exmp, pair.Value)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+		visited[nameGeneral] = param
+	}
+
+	// maintain "required" flag
+	if *params != nil {
+		for _, param := range **params {
+			paramObj, err := param.ResolveParameter(paramResolver)
+			if err != nil {
+				logger.Log.Warningf("Failed to resolve param: %s", err)
+				continue
+			}
+			if paramObj.In != gw.In {
+				continue
+			}
+
+			_, ok := visited[strings.ToLower(paramObj.Name)]
+			if !ok {
+				flag := false
+				paramObj.Required = &flag
+			}
+		}
+	}
+}
+
+func createHeader(ptype openapi.SchemaType) *openapi.HeaderObj {
+	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
+	schema := new(openapi.SchemaObj)
+	schema.Type = make(openapi.Types, 0)
+	schema.Type = append(schema.Type, ptype)
+
+	style := openapi.StyleSimple
+	newParam := openapi.HeaderObj{
+		Style:    string(style),
+		Examples: map[string]openapi.Example{},
+		Schema:   schema,
+		Required: &required,
+	}
+	return &newParam
+}
+
+func fillParamExample(param **openapi.Examples, exampleValue string) error {
+	if **param == nil {
+		**param = map[string]openapi.Example{}
+	}
+
+	cnt := 0
+	for _, example := range **param {
+		cnt++
+		exampleObj, err := example.ResolveExample(exampleResolver)
+		if err != nil {
+			continue
+		}
+
+		var value string
+		err = json.Unmarshal(exampleObj.Value, &value)
+		if err != nil {
+			logger.Log.Warningf("Failed decoding parameter example into string: %s", err)
+			continue
+		}
+
+		if value == exampleValue || cnt > 5 { // 5 examples is enough
+			return nil
+		}
+	}
+
+	valMsg, err := json.Marshal(exampleValue)
+	if err != nil {
+		return err
+	}
+
+	themap := **param
+	themap["example #"+strconv.Itoa(cnt)] = &openapi.ExampleObj{Value: valMsg}
+
+	return nil
+}
+
+func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.com/jpillora/longestcommon
+	empty := make([]string, 0)
+	//short-circuit empty list
+	if len(strs) == 0 {
+		return empty
+	}
+	xfix := strs[0]
+	//short-circuit single-element list
+	if len(strs) == 1 {
+		return xfix
+	}
+	//compare first to rest
+	for _, str := range strs[1:] {
+		xfixl := len(xfix)
+		strl := len(str)
+		//short-circuit empty strings
+		if xfixl == 0 || strl == 0 {
+			return empty
+		}
+		//maximum possible length
+		maxl := xfixl
+		if strl < maxl {
+			maxl = strl
+		}
+		//compare letters
+		if pre {
+			//prefix, iterate left to right
+			for i := 0; i < maxl; i++ {
+				if xfix[i] != str[i] {
+					xfix = xfix[:i]
+					break
+				}
+			}
+		} else {
+			//suffix, iternate right to left
+			for i := 0; i < maxl; i++ {
+				xi := xfixl - i - 1
+				si := strl - i - 1
+				if xfix[xi] != str[si] {
+					xfix = xfix[xi+1:]
+					break
+				}
+			}
+		}
+	}
+	return xfix
+}
+
+// returns all non-nil ops in PathObj
+func getOps(pathObj *openapi.PathObj) []*openapi.Operation {
+	ops := []**openapi.Operation{&pathObj.Get, &pathObj.Patch, &pathObj.Put, &pathObj.Options, &pathObj.Post, &pathObj.Trace, &pathObj.Head, &pathObj.Delete}
+	res := make([]*openapi.Operation, 0)
+	for _, opp := range ops {
+		if *opp == nil {
+			continue
+		}
+		res = append(res, *opp)
+	}
+	return res
+}
+
+// parses JSON into any possible value
+func anyJSON(text string) (anyVal interface{}, isJSON bool) {
+	isJSON = true
+	asMap := map[string]interface{}{}
+	err := json.Unmarshal([]byte(text), &asMap)
+	if err == nil && asMap != nil {
+		return asMap, isJSON
+	}
+
+	asArray := make([]interface{}, 0)
+	err = json.Unmarshal([]byte(text), &asArray)
+	if err == nil && asArray != nil {
+		return asArray, isJSON
+	}
+
+	asString := ""
+	sPtr := &asString
+	err = json.Unmarshal([]byte(text), &sPtr)
+	if err == nil && sPtr != nil {
+		return asString, isJSON
+	}
+
+	asInt := 0
+	intPtr := &asInt
+	err = json.Unmarshal([]byte(text), &intPtr)
+	if err == nil && intPtr != nil {
+		return asInt, isJSON
+	}
+
+	asFloat := 0.0
+	floatPtr := &asFloat
+	err = json.Unmarshal([]byte(text), &floatPtr)
+	if err == nil && floatPtr != nil {
+		return asFloat, isJSON
+	}
+
+	asBool := false
+	boolPtr := &asBool
+	err = json.Unmarshal([]byte(text), &boolPtr)
+	if err == nil && boolPtr != nil {
+		return asBool, isJSON
+	}
+
+	if text == "null" {
+		return nil, isJSON
+	}
+
+	return nil, false
+}

agent/pkg/oas/utils_test.go

@@ -0,0 +1,35 @@
+package oas
+
+import (
+	"testing"
+)
+
+func TestAnyJSON(t *testing.T) {
+	testCases := []struct {
+		inp    string
+		isJSON bool
+		out    interface{}
+	}{
+		{`{"key": 1, "keyNull": null}`, true, nil},
+		{`[{"key": "val"}, ["subarray"], "string", 1, 2.2, true, null]`, true, nil},
+		{`"somestring"`, true, "somestring"},
+		{"0", true, 0},
+		{"0.5", true, 0.5},
+		{"true", true, true},
+		{"null", true, nil},
+		{"sabbra cadabra", false, nil},
+		{"0.1.2.3", false, nil},
+	}
+	for _, tc := range testCases {
+		any, isJSON := anyJSON(tc.inp)
+		if isJSON != tc.isJSON {
+			t.Errorf("Parse flag mismatch: %t != %t", tc.isJSON, isJSON)
+		} else if isJSON && tc.out != nil && tc.out != any {
+			t.Errorf("%s != %s", any, tc.out)
+		} else if tc.inp == "null" && any != nil {
+			t.Errorf("null has to parse as nil (but got %s)", any)
+		} else {
+			t.Logf("%s => %s", tc.inp, any)
+		}
+	}
+}

agent/pkg/providers/install_provider.go

@@ -0,0 +1,18 @@
+package providers
+
+import (
+	"context"
+	"mizuserver/pkg/config"
+)
+
+func IsInstallNeeded() (bool, error) {
+	if !config.Config.StandaloneMode { // install not needed in ephermeral mizu
+		return false, nil
+	}
+
+	if anyUserExists, err := AnyUserExists(context.Background()); err != nil {
+		return false, err
+	} else {
+		return !anyUserExists, nil
+	}
+}

agent/pkg/providers/kubernetes_provider.go

@@ -0,0 +1,27 @@
+package providers
+
+import (
+	"github.com/up9inc/mizu/shared/kubernetes"
+	"sync"
+)
+
+var lock = &sync.Mutex{}
+
+var kubernetesProvider *kubernetes.Provider
+
+func GetKubernetesProvider() (*kubernetes.Provider, error) {
+	if kubernetesProvider == nil {
+		lock.Lock()
+		defer lock.Unlock()
+
+		if kubernetesProvider == nil {
+			var err error
+			kubernetesProvider, err = kubernetes.NewProviderInCluster()
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return kubernetesProvider, nil
+}

agent/pkg/providers/stats_provider_test.go

@@ -2,16 +2,10 @@ package providers_test
 
 import (
 	"fmt"
-	"mizuserver/pkg/config"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/providers"
 	"testing"
 )
 
-func init() {
-	database.InitDataBase(config.DefaultDatabasePath)
-}
-
 func TestNoEntryAddedCount(t *testing.T) {
 	entriesStats := providers.GetGeneralStats()
 

agent/pkg/providers/status_provider.go

@@ -8,19 +8,14 @@ import (
 	"github.com/up9inc/mizu/tap"
 	"mizuserver/pkg/models"
 	"os"
-	"sync"
 	"time"
 )
 
 const tlsLinkRetainmentTime = time.Minute * 15
 
 var (
-	TappersCount   int
-	TapStatus      shared.TapStatus
 	authStatus     *models.AuthStatus
 	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
-
-	tappersCountLock = sync.Mutex{}
 )
 
 func GetAuthStatus() (*models.AuthStatus, error) {
@@ -68,15 +63,3 @@ func GetAllRecentTLSAddresses() []string {
 
 	return recentTLSLinks
 }
-
-func TapperAdded() {
-	tappersCountLock.Lock()
-	TappersCount++
-	tappersCountLock.Unlock()
-}
-
-func TapperRemoved() {
-	tappersCountLock.Lock()
-	TappersCount--
-	tappersCountLock.Unlock()
-}

agent/pkg/providers/tapConfig/tap_config_provider.go

@@ -0,0 +1,42 @@
+package tapConfig
+
+import (
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/utils"
+	"os"
+	"sync"
+)
+
+const FilePath = shared.DataDirPath + "tap-config.json"
+
+var (
+	lock     = &sync.Mutex{}
+	syncOnce sync.Once
+	config   *models.TapConfig
+)
+
+func Get() *models.TapConfig {
+	syncOnce.Do(func() {
+		if err := utils.ReadJsonFile(FilePath, &config); err != nil {
+			config = &models.TapConfig{TappedNamespaces: make(map[string]bool)}
+
+			if !os.IsNotExist(err) {
+				logger.Log.Errorf("Error reading tap config from file, err: %v", err)
+			}
+		}
+	})
+
+	return config
+}
+
+func Save(tapConfigToSave *models.TapConfig) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	config = tapConfigToSave
+	if err := utils.SaveJsonFile(FilePath, config); err != nil {
+		logger.Log.Errorf("Error saving tap config, err: %v", err)
+	}
+}

agent/pkg/providers/tappedPods/tapped_pods_provider.go

@@ -0,0 +1,56 @@
+package tappedPods
+
+import (
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/providers/tappers"
+	"mizuserver/pkg/utils"
+	"os"
+	"strings"
+	"sync"
+)
+
+const FilePath = shared.DataDirPath + "tapped-pods.json"
+
+var (
+	lock       = &sync.Mutex{}
+	syncOnce   sync.Once
+	tappedPods []*shared.PodInfo
+)
+
+func Get() []*shared.PodInfo {
+	syncOnce.Do(func() {
+		if err := utils.ReadJsonFile(FilePath, &tappedPods); err != nil {
+			if !os.IsNotExist(err) {
+				logger.Log.Errorf("Error reading tapped pods from file, err: %v", err)
+			}
+		}
+	})
+
+	return tappedPods
+}
+
+func Set(tappedPodsToSet []*shared.PodInfo) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	tappedPods = tappedPodsToSet
+	if err := utils.SaveJsonFile(FilePath, tappedPods); err != nil {
+		logger.Log.Errorf("Error saving tapped pods, err: %v", err)
+	}
+}
+
+func GetTappedPodsStatus() []shared.TappedPodStatus {
+	tappedPodsStatus := make([]shared.TappedPodStatus, 0)
+	for _, pod := range Get() {
+		var status string
+		if tapperStatus, ok := tappers.GetStatus()[pod.NodeName]; ok {
+			status = strings.ToLower(tapperStatus.Status)
+		}
+
+		isTapped := status == "running"
+		tappedPodsStatus = append(tappedPodsStatus, shared.TappedPodStatus{Name: pod.Name, Namespace: pod.Namespace, IsTapped: isTapped})
+	}
+
+	return tappedPodsStatus
+}

agent/pkg/providers/tappers/tappers_provider.go

@@ -0,0 +1,82 @@
+package tappers
+
+import (
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/utils"
+	"os"
+	"sync"
+)
+
+const FilePath = shared.DataDirPath + "tappers-status.json"
+
+var (
+	lockStatus = &sync.Mutex{}
+	syncOnce   sync.Once
+	status     map[string]*shared.TapperStatus
+
+	lockConnectedCount = &sync.Mutex{}
+	connectedCount     int
+)
+
+func GetStatus() map[string]*shared.TapperStatus {
+	initStatus()
+
+	return status
+}
+
+func SetStatus(tapperStatus *shared.TapperStatus) {
+	initStatus()
+
+	lockStatus.Lock()
+	defer lockStatus.Unlock()
+
+	status[tapperStatus.NodeName] = tapperStatus
+
+	saveStatus()
+}
+
+func ResetStatus() {
+	lockStatus.Lock()
+	defer lockStatus.Unlock()
+
+	status = make(map[string]*shared.TapperStatus)
+
+	saveStatus()
+}
+
+func GetConnectedCount() int {
+	return connectedCount
+}
+
+func Connected() {
+	lockConnectedCount.Lock()
+	defer lockConnectedCount.Unlock()
+
+	connectedCount++
+}
+
+func Disconnected() {
+	lockConnectedCount.Lock()
+	defer lockConnectedCount.Unlock()
+
+	connectedCount--
+}
+
+func initStatus() {
+	syncOnce.Do(func() {
+		if err := utils.ReadJsonFile(FilePath, &status); err != nil {
+			status = make(map[string]*shared.TapperStatus)
+
+			if !os.IsNotExist(err) {
+				logger.Log.Errorf("Error reading tappers status from file, err: %v", err)
+			}
+		}
+	})
+}
+
+func saveStatus() {
+	if err := utils.SaveJsonFile(FilePath, status); err != nil {
+		logger.Log.Errorf("Error saving tappers status, err: %v", err)
+	}
+}

agent/pkg/providers/user_provider.go

@@ -0,0 +1,162 @@
+package providers
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/cookiejar"
+
+	ory "github.com/ory/kratos-client-go"
+	"github.com/up9inc/mizu/shared/logger"
+)
+
+var client = getKratosClient("http://127.0.0.1:4433", "http://127.0.0.1:4434")
+
+// returns session token if successful
+func RegisterUser(username string, password string, ctx context.Context) (token *string, identityId string, err error, formErrorMessages map[string][]ory.UiText) {
+	flow, _, err := client.V0alpha2Api.InitializeSelfServiceRegistrationFlowWithoutBrowser(ctx).Execute()
+	if err != nil {
+		return nil, "", err, nil
+	}
+
+	result, _, err := client.V0alpha2Api.SubmitSelfServiceRegistrationFlow(ctx).Flow(flow.Id).SubmitSelfServiceRegistrationFlowBody(
+		ory.SubmitSelfServiceRegistrationFlowWithPasswordMethodBodyAsSubmitSelfServiceRegistrationFlowBody(&ory.SubmitSelfServiceRegistrationFlowWithPasswordMethodBody{
+			Method:   "password",
+			Password: password,
+			Traits:   map[string]interface{}{"username": username},
+		}),
+	).Execute()
+
+	if err != nil {
+		parsedKratosError, parsingErr := parseKratosRegistrationFormError(err)
+		if parsingErr != nil {
+			logger.Log.Debugf("error parsing kratos error: %v", parsingErr)
+			return nil, "", err, nil
+		} else {
+			return nil, "", err, parsedKratosError
+		}
+	}
+
+	return result.SessionToken, result.Identity.Id, nil, nil
+}
+
+func PerformLogin(username string, password string, ctx context.Context) (*string, error) {
+	flow, _, err := client.V0alpha2Api.InitializeSelfServiceLoginFlowWithoutBrowser(ctx).Execute()
+	if err != nil {
+		return nil, err
+	}
+
+	result, _, err := client.V0alpha2Api.SubmitSelfServiceLoginFlow(ctx).Flow(flow.Id).SubmitSelfServiceLoginFlowBody(
+		ory.SubmitSelfServiceLoginFlowWithPasswordMethodBodyAsSubmitSelfServiceLoginFlowBody(&ory.SubmitSelfServiceLoginFlowWithPasswordMethodBody{
+			Method:             "password",
+			Password:           password,
+			PasswordIdentifier: username,
+		}),
+	).Execute()
+
+	if err != nil {
+		return nil, err
+	}
+	if result == nil {
+		return nil, errors.New("unknown error occured during login")
+	}
+
+	return result.SessionToken, nil
+}
+
+func VerifyToken(token string, ctx context.Context) (bool, error) {
+	flow, _, err := client.V0alpha2Api.ToSession(ctx).XSessionToken(token).Execute()
+	if err != nil {
+		return false, err
+	}
+
+	if flow == nil {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func DeleteUser(identityId string, ctx context.Context) error {
+	result, err := client.V0alpha2Api.AdminDeleteIdentity(ctx, identityId).Execute()
+	if err != nil {
+		return err
+	}
+	if result == nil {
+		return errors.New("unknown error occured during user deletion")
+	}
+
+	if result.StatusCode < 200 || result.StatusCode > 299 {
+		return errors.New(fmt.Sprintf("user deletion returned bad status %d", result.StatusCode))
+	} else {
+		return nil
+	}
+}
+
+func AnyUserExists(ctx context.Context) (bool, error) {
+	request := client.V0alpha2Api.AdminListIdentities(ctx)
+	request.PerPage(1)
+
+	if result, _, err := request.Execute(); err != nil {
+		return false, err
+	} else {
+		return len(result) > 0, nil
+	}
+}
+
+func Logout(token string, ctx context.Context) error {
+	logoutRequest := client.V0alpha2Api.SubmitSelfServiceLogoutFlowWithoutBrowser(ctx)
+	logoutRequest = logoutRequest.SubmitSelfServiceLogoutFlowWithoutBrowserBody(ory.SubmitSelfServiceLogoutFlowWithoutBrowserBody{
+		SessionToken: token,
+	})
+	if response, err := logoutRequest.Execute(); err != nil {
+		return err
+	} else if response == nil || response.StatusCode < 200 || response.StatusCode > 299 {
+		return errors.New("unknown error occured during logout")
+	}
+
+	return nil
+}
+
+func getKratosClient(url string, adminUrl string) *ory.APIClient {
+	conf := ory.NewConfiguration()
+	conf.Servers = ory.ServerConfigurations{{URL: url}}
+
+	// this ensures kratos client uses the admin url for admin actions (any new admin action we use will have to be added here)
+	conf.OperationServers = map[string]ory.ServerConfigurations{
+		"V0alpha2ApiService.AdminDeleteIdentity": {{URL: adminUrl}},
+		"V0alpha2ApiService.AdminListIdentities": {{URL: adminUrl}},
+	}
+
+	cj, _ := cookiejar.New(nil)
+	conf.HTTPClient = &http.Client{Jar: cj}
+	return ory.NewAPIClient(conf)
+}
+
+// returns map of form value key to error message
+func parseKratosRegistrationFormError(err error) (map[string][]ory.UiText, error) {
+	var openApiError *ory.GenericOpenAPIError
+	if errors.As(err, &openApiError) {
+		var registrationFlowModel *ory.SelfServiceRegistrationFlow
+		if jsonErr := json.Unmarshal(openApiError.Body(), &registrationFlowModel); jsonErr != nil {
+			return nil, jsonErr
+		} else {
+			formMessages := registrationFlowModel.Ui.Nodes
+			parsedMessages := make(map[string][]ory.UiText)
+
+			for _, message := range formMessages {
+				if len(message.Messages) > 0 {
+					if _, ok := parsedMessages[message.Group]; !ok {
+						parsedMessages[message.Group] = make([]ory.UiText, 0)
+					}
+					parsedMessages[message.Group] = append(parsedMessages[message.Group], message.Messages...)
+				}
+			}
+			return parsedMessages, nil
+		}
+	} else {
+		return nil, errors.New("error is not a generic openapi error")
+	}
+}

agent/pkg/resolver/resolver.go

@@ -164,10 +164,10 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
 func (resolver *Resolver) saveResolvedName(key string, resolved string, eventType watch.EventType) {
 	if eventType == watch.Deleted {
 		resolver.nameMap.Remove(key)
-		logger.Log.Infof("setting %s=nil\n", key)
+		logger.Log.Infof("setting %s=nil", key)
 	} else {
 		resolver.nameMap.Set(key, resolved)
-		logger.Log.Infof("setting %s=%s\n", key, resolved)
+		logger.Log.Infof("setting %s=%s", key, resolved)
 	}
 }
 
@@ -188,7 +188,7 @@ func (resolver *Resolver) infiniteErrorHandleRetryFunc(ctx context.Context, fun
 			var statusError *k8serrors.StatusError
 			if errors.As(err, &statusError) {
 				if statusError.ErrStatus.Reason == metav1.StatusReasonForbidden {
-					logger.Log.Infof("Resolver loop encountered permission error, aborting event listening - %v\n", err)
+					logger.Log.Infof("Resolver loop encountered permission error, aborting event listening - %v", err)
 					return
 				}
 			}

agent/pkg/routes/config_routes.go

@@ -0,0 +1,15 @@
+package routes
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/middlewares"
+)
+
+func ConfigRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/config")
+	routeGroup.Use(middlewares.RequiresAuth())
+
+	routeGroup.POST("/tapConfig", controllers.PostTapConfig)
+	routeGroup.GET("/tapConfig", controllers.GetTapConfig)
+}

agent/pkg/routes/entries_routes.go

@@ -1,14 +1,17 @@
 package routes
 
 import (
-	"github.com/gin-gonic/gin"
 	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/middlewares"
+
+	"github.com/gin-gonic/gin"
 )
 
 // EntriesRoutes defines the group of har entries routes.
 func EntriesRoutes(ginApp *gin.Engine) {
 	routeGroup := ginApp.Group("/entries")
+	routeGroup.Use(middlewares.RequiresAuth())
 
-	routeGroup.GET("/", controllers.GetEntries)        // get entries (base/thin entries)
-	routeGroup.GET("/:entryId", controllers.GetEntry) // get single (full) entry
+	routeGroup.GET("/", controllers.GetEntries)  // get entries (base/thin entries) and metadata
+	routeGroup.GET("/:id", controllers.GetEntry) // get single (full) entry
 }

agent/pkg/routes/install_routes.go

@@ -0,0 +1,13 @@
+package routes
+
+import (
+	"mizuserver/pkg/controllers"
+
+	"github.com/gin-gonic/gin"
+)
+
+func InstallRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/install")
+
+	routeGroup.GET("/isNeeded", controllers.IsSetupNecessary)
+}

agent/pkg/routes/metadata_routes.go

@@ -1,8 +1,9 @@
 package routes
 
 import (
-	"github.com/gin-gonic/gin"
 	"mizuserver/pkg/controllers"
+
+	"github.com/gin-gonic/gin"
 )
 
 // MetadataRoutes defines the group of metadata routes.

agent/pkg/routes/oas_routes.go

@@ -0,0 +1,18 @@
+package routes
+
+import (
+	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/middlewares"
+
+	"github.com/gin-gonic/gin"
+)
+
+// OASRoutes methods to access OAS spec
+func OASRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/oas")
+	routeGroup.Use(middlewares.RequiresAuth())
+
+	routeGroup.GET("/", controllers.GetOASServers)     // list of servers in OAS map
+	routeGroup.GET("/all", controllers.GetOASAllSpecs) // list of servers in OAS map
+	routeGroup.GET("/:id", controllers.GetOASSpec)     // get OAS spec for given server
+}

agent/pkg/routes/query_routes.go

@@ -0,0 +1,15 @@
+package routes
+
+import (
+	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/middlewares"
+
+	"github.com/gin-gonic/gin"
+)
+
+func QueryRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/query")
+	routeGroup.Use(middlewares.RequiresAuth())
+
+	routeGroup.POST("/validate", controllers.PostValidate)
+}

agent/pkg/routes/status_routes.go

@@ -1,17 +1,21 @@
 package routes
 
 import (
-	"github.com/gin-gonic/gin"
 	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/middlewares"
+
+	"github.com/gin-gonic/gin"
 )
 
 func StatusRoutes(ginApp *gin.Engine) {
 	routeGroup := ginApp.Group("/status")
+	routeGroup.Use(middlewares.RequiresAuth())
 
 	routeGroup.GET("/health", controllers.HealthCheck)
 
 	routeGroup.POST("/tappedPods", controllers.PostTappedPods)
-	routeGroup.GET("/tappersCount", controllers.GetTappersCount)
+	routeGroup.POST("/tapperStatus", controllers.PostTapperStatus)
+	routeGroup.GET("/connectedTappersCount", controllers.GetConnectedTappersCount)
 	routeGroup.GET("/tap", controllers.GetTappingStatus)
 
 	routeGroup.GET("/auth", controllers.GetAuthStatus)

agent/pkg/routes/user_routes.go

@@ -0,0 +1,15 @@
+package routes
+
+import (
+	"mizuserver/pkg/controllers"
+
+	"github.com/gin-gonic/gin"
+)
+
+func UserRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/user")
+
+	routeGroup.POST("/login", controllers.Login)
+	routeGroup.POST("/logout", controllers.Logout)
+	routeGroup.POST("/register", controllers.Register)
+}

agent/pkg/sensitiveDataFiltering/consts.go

@@ -1,10 +0,0 @@
-package sensitiveDataFiltering
-
-const maskedFieldPlaceholderValue = "[REDACTED]"
-
-//these values MUST be all lower case and contain no `-` or `_` characters
-var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
-	"username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt",
-	"bearer", "clientid", "clientsecret", "redirecturi", "phonenumber",
-	"zip", "zipcode", "address", "country", "firstname", "lastname",
-	"middlename", "fname", "lname", "birthdate"}

agent/pkg/up9/main.go

@@ -7,15 +7,16 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"mizuserver/pkg/database"
 	"mizuserver/pkg/utils"
 	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/google/martian/har"
+	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
@@ -23,6 +24,7 @@ import (
 
 const (
 	AnalyzeCheckSleepTime = 5 * time.Second
+	SentCountLogInterval  = 100
 )
 
 type GuestToken struct {
@@ -110,14 +112,14 @@ func GetAnalyzeInfo() *shared.AnalyzeStatus {
 }
 
 func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error {
-	logger.Log.Infof("Sync entries - started\n")
+	logger.Log.Infof("Sync entries - started")
 
 	var (
 		token, model string
 		guestMode    bool
 	)
 	if syncEntriesConfig.Token == "" {
-		logger.Log.Infof("Sync entries - creating anonymous token. env %s\n", syncEntriesConfig.Env)
+		logger.Log.Infof("Sync entries - creating anonymous token. env %s", syncEntriesConfig.Env)
 		guestToken, err := createAnonymousToken(syncEntriesConfig.Env)
 		if err != nil {
 			return fmt.Errorf("failed creating anonymous token, err: %v", err)
@@ -131,7 +133,7 @@ func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error {
 		model = syncEntriesConfig.Workspace
 		guestMode = false
 
-		logger.Log.Infof("Sync entries - upserting model. env %s, model %s\n", syncEntriesConfig.Env, model)
+		logger.Log.Infof("Sync entries - upserting model. env %s, model %s", syncEntriesConfig.Env, model)
 		if err := upsertModel(token, model, syncEntriesConfig.Env); err != nil {
 			return fmt.Errorf("failed upserting model, err: %v", err)
 		}
@@ -142,7 +144,7 @@ func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error {
 		return fmt.Errorf("invalid model name, model name: %s", model)
 	}
 
-	logger.Log.Infof("Sync entries - syncing. token: %s, model: %s, guest mode: %v\n", token, model, guestMode)
+	logger.Log.Infof("Sync entries - syncing. token: %s, model: %s, guest mode: %v", token, model, guestMode)
 	go syncEntriesImpl(token, model, syncEntriesConfig.Env, syncEntriesConfig.UploadIntervalSec, guestMode)
 
 	return nil
@@ -204,51 +206,80 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 	analyzeInformation.AnalyzeDestination = envPrefix
 	analyzeInformation.SentCount = 0
 
-	sleepTime := time.Second * time.Duration(uploadIntervalSec)
+	// "http or grpc" filter indicates that we're only interested in HTTP and gRPC entries
+	query := "http or grpc"
 
-	var timeFrom time.Time
-	protocolFilter := "http"
+	logger.Log.Infof("Getting entries from the database")
 
-	for {
-		timeTo := time.Now()
-		logger.Log.Infof("Getting entries from %v, to %v\n", timeFrom.Format(time.RFC3339Nano), timeTo.Format(time.RFC3339Nano))
-		entriesArray := database.GetEntriesFromDb(timeFrom, timeTo, &protocolFilter)
+	var connection *basenine.Connection
+	var err error
+	connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+	if err != nil {
+		panic(err)
+	}
 
-		if len(entriesArray) > 0 {
-			result := make([]har.Entry, 0)
-			for _, data := range entriesArray {
-				var pair tapApi.RequestResponsePair
-				if err := json.Unmarshal([]byte(data.Entry), &pair); err != nil {
-					continue
-				}
-				harEntry, err := utils.NewEntry(&pair)
-				if err != nil {
-					continue
-				}
-				if data.ResolvedSource != "" {
-					harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: data.ResolvedSource})
-				}
-				if data.ResolvedDestination != "" {
-					harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: data.ResolvedDestination})
-					harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, data.ResolvedDestination)
-				}
+	data := make(chan []byte)
+	meta := make(chan []byte)
 
-				// go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64
-				if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil {
-					continue
-				}
+	defer func() {
+		data <- []byte(basenine.CloseChannel)
+		meta <- []byte(basenine.CloseChannel)
+		connection.Close()
+	}()
 
-				result = append(result, *harEntry)
+	lastTimeSynced := time.Time{}
+
+	batch := make([]har.Entry, 0)
+
+	handleDataChannel := func(wg *sync.WaitGroup, connection *basenine.Connection, data chan []byte) {
+		defer wg.Done()
+		for {
+			dataBytes := <-data
+
+			if string(dataBytes) == basenine.CloseChannel {
+				return
 			}
 
-			logger.Log.Infof("About to upload %v entries\n", len(result))
+			var dataMap map[string]interface{}
+			err = json.Unmarshal(dataBytes, &dataMap)
 
-			body, jMarshalErr := json.Marshal(result)
+			var entry tapApi.Entry
+			if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil {
+				continue
+			}
+			harEntry, err := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+			if err != nil {
+				continue
+			}
+			if entry.Source.Name != "" {
+				harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.Source.Name})
+			}
+			if entry.Destination.Name != "" {
+				harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.Destination.Name})
+				harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entry.Destination.Name)
+			}
+
+			// go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64
+			if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil {
+				continue
+			}
+
+			batch = append(batch, *harEntry)
+
+			now := time.Now()
+			if lastTimeSynced.Add(time.Duration(uploadIntervalSec) * time.Second).After(now) {
+				continue
+			}
+			lastTimeSynced = now
+
+			body, jMarshalErr := json.Marshal(batch)
+			batchSize := len(batch)
 			if jMarshalErr != nil {
 				analyzeInformation.Reset()
 				logger.Log.Infof("Stopping sync entries")
 				logger.Log.Fatal(jMarshalErr)
 			}
+			batch = make([]har.Entry, 0)
 
 			var in bytes.Buffer
 			w := zlib.NewWriter(&in)
@@ -273,18 +304,33 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 				logger.Log.Info("Stopping sync entries")
 				logger.Log.Fatal(postErr)
 			}
-			analyzeInformation.SentCount += len(entriesArray)
-			logger.Log.Infof("Finish uploading %v entries to %s\n", len(entriesArray), GetTrafficDumpUrl(envPrefix, model))
+			analyzeInformation.SentCount += batchSize
 
-			logger.Log.Infof("Uploaded %v entries until now", analyzeInformation.SentCount)
-		} else {
-			logger.Log.Infof("Nothing to upload")
+			if analyzeInformation.SentCount%SentCountLogInterval == 0 {
+				logger.Log.Infof("Uploaded %v entries until now", analyzeInformation.SentCount)
+			}
 		}
-
-		logger.Log.Infof("Sleeping for %v...\n", sleepTime)
-		time.Sleep(sleepTime)
-		timeFrom = timeTo
 	}
+
+	handleMetaChannel := func(wg *sync.WaitGroup, connection *basenine.Connection, meta chan []byte) {
+		defer wg.Done()
+		for {
+			metaBytes := <-meta
+
+			if string(metaBytes) == basenine.CloseChannel {
+				return
+			}
+		}
+	}
+
+	var wg sync.WaitGroup
+	go handleDataChannel(&wg, connection, data)
+	go handleMetaChannel(&wg, connection, meta)
+	wg.Add(2)
+
+	connection.Query(query, data, meta)
+
+	wg.Wait()
 }
 
 func UpdateAnalyzeStatus(callback func(data []byte)) {

agent/pkg/utils/har.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/tap/api"
 )
 
 // Keep it because we might want cookies in the future
@@ -120,13 +119,11 @@ func BuildPostParams(rawParams []interface{}) []har.Param {
 	return params
 }
 
-func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error) {
-	reqDetails := request.Payload.(map[string]interface{})["details"].(map[string]interface{})
+func NewRequest(request map[string]interface{}) (harRequest *har.Request, err error) {
+	headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{}))
+	cookies := make([]har.Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
 
-	headers, host, scheme, authority, path, _ := BuildHeaders(reqDetails["headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(reqDetails["cookies"].([]interface{}))
-
-	postData, _ := reqDetails["postData"].(map[string]interface{})
+	postData, _ := request["postData"].(map[string]interface{})
 	mimeType, _ := postData["mimeType"]
 	if mimeType == nil || len(mimeType.(string)) == 0 {
 		mimeType = "text/html"
@@ -138,7 +135,7 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
 	}
 
 	queryString := make([]har.QueryString, 0)
-	for _, _qs := range reqDetails["queryString"].([]interface{}) {
+	for _, _qs := range request["_queryString"].([]interface{}) {
 		qs := _qs.(map[string]interface{})
 		queryString = append(queryString, har.QueryString{
 			Name:  qs["name"].(string),
@@ -146,7 +143,7 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
 		})
 	}
 
-	url := fmt.Sprintf("http://%s%s", host, reqDetails["url"].(string))
+	url := fmt.Sprintf("http://%s%s", host, request["url"].(string))
 	if strings.HasPrefix(mimeType.(string), "application/grpc") {
 		url = fmt.Sprintf("%s://%s%s", scheme, authority, path)
 	}
@@ -157,9 +154,9 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
 	}
 
 	harRequest = &har.Request{
-		Method:      reqDetails["method"].(string),
+		Method:      request["method"].(string),
 		URL:         url,
-		HTTPVersion: reqDetails["httpVersion"].(string),
+		HTTPVersion: request["httpVersion"].(string),
 		HeadersSize: -1,
 		BodySize:    int64(bytes.NewBufferString(postDataText).Len()),
 		QueryString: queryString,
@@ -175,13 +172,11 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
 	return
 }
 
-func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err error) {
-	resDetails := response.Payload.(map[string]interface{})["details"].(map[string]interface{})
+func NewResponse(response map[string]interface{}) (harResponse *har.Response, err error) {
+	headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{}))
+	cookies := make([]har.Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
 
-	headers, _, _, _, _, _status := BuildHeaders(resDetails["headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(resDetails["cookies"].([]interface{}))
-
-	content, _ := resDetails["content"].(map[string]interface{})
+	content, _ := response["content"].(map[string]interface{})
 	mimeType, _ := content["mimeType"]
 	if mimeType == nil || len(mimeType.(string)) == 0 {
 		mimeType = "text/html"
@@ -200,9 +195,11 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
 		Size:     int64(len(bodyText)),
 	}
 
-	status := int(resDetails["status"].(float64))
+	status := int(response["status"].(float64))
 	if strings.HasPrefix(mimeType.(string), "application/grpc") {
-		status, err = strconv.Atoi(_status)
+		if _status != "" {
+			status, err = strconv.Atoi(_status)
+		}
 		if err != nil {
 			logger.Log.Errorf("Failed converting status to int %s (%v,%+v)", err, err, err)
 			return nil, errors.New("failed converting response status to int for HAR")
@@ -210,9 +207,9 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
 	}
 
 	harResponse = &har.Response{
-		HTTPVersion: resDetails["httpVersion"].(string),
+		HTTPVersion: response["httpVersion"].(string),
 		Status:      status,
-		StatusText:  resDetails["statusText"].(string),
+		StatusText:  response["statusText"].(string),
 		HeadersSize: -1,
 		BodySize:    int64(bytes.NewBufferString(bodyText).Len()),
 		Headers:     headers,
@@ -222,34 +219,33 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
 	return
 }
 
-func NewEntry(pair *api.RequestResponsePair) (*har.Entry, error) {
-	harRequest, err := NewRequest(&pair.Request)
+func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*har.Entry, error) {
+	harRequest, err := NewRequest(request)
 	if err != nil {
 		logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
 		return nil, errors.New("failed converting request to HAR")
 	}
 
-	harResponse, err := NewResponse(&pair.Response)
+	harResponse, err := NewResponse(response)
 	if err != nil {
 		logger.Log.Errorf("Failed converting response to HAR %s (%v,%+v)", err, err, err)
 		return nil, errors.New("failed converting response to HAR")
 	}
 
-	totalTime := pair.Response.CaptureTime.Sub(pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
-	if totalTime < 1 {
-		totalTime = 1
+	if elapsedTime < 1 {
+		elapsedTime = 1
 	}
 
 	harEntry := har.Entry{
-		StartedDateTime: pair.Request.CaptureTime,
-		Time:            totalTime,
+		StartedDateTime: startTime,
+		Time:            elapsedTime,
 		Request:         harRequest,
 		Response:        harResponse,
 		Cache:           &har.Cache{},
 		Timings: &har.Timings{
 			Send:    -1,
 			Wait:    -1,
-			Receive: totalTime,
+			Receive: elapsedTime,
 		},
 	}
 

agent/pkg/utils/truncating_logger.go

@@ -1,60 +0,0 @@
-package utils
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	loggerShared "github.com/up9inc/mizu/shared/logger"
-	"gorm.io/gorm/logger"
-	"gorm.io/gorm/utils"
-)
-
-// TruncatingLogger implements the gorm logger.Interface interface. Its purpose is to act as gorm's logger while truncating logs to a max of 50 characters to minimise the performance impact
-type TruncatingLogger struct {
-	LogLevel      logger.LogLevel
-	SlowThreshold time.Duration
-}
-
-func (truncatingLogger *TruncatingLogger) LogMode(logLevel logger.LogLevel) logger.Interface {
-	truncatingLogger.LogLevel = logLevel
-	return truncatingLogger
-}
-
-func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string, __ ...interface{}) {
-	if truncatingLogger.LogLevel < logger.Info {
-		return
-	}
-	loggerShared.Log.Errorf("gorm info: %.150s", message)
-}
-
-func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) {
-	if truncatingLogger.LogLevel < logger.Warn {
-		return
-	}
-	loggerShared.Log.Errorf("gorm warning: %.150s", message)
-}
-
-func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) {
-	if truncatingLogger.LogLevel < logger.Error {
-		return
-	}
-	loggerShared.Log.Errorf("gorm error: %.150s", message)
-}
-
-func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) {
-	if truncatingLogger.LogLevel == logger.Silent {
-		return
-	}
-	elapsed := time.Since(begin)
-	if err != nil {
-		sql, rows := fc() // copied into every condition as this is a potentially heavy operation best done only when necessary
-		truncatingLogger.Error(ctx, fmt.Sprintf("Error in %s: %v - elapsed: %fs affected rows: %d, sql: %s", utils.FileWithLineNum(), err, elapsed.Seconds(), rows, sql))
-	} else if truncatingLogger.LogLevel >= logger.Warn && elapsed > truncatingLogger.SlowThreshold {
-		sql, rows := fc()
-		truncatingLogger.Warn(ctx, fmt.Sprintf("Slow sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql))
-	} else if truncatingLogger.LogLevel >= logger.Info {
-		sql, rows := fc()
-		truncatingLogger.Info(ctx, fmt.Sprintf("Sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql))
-	}
-}

agent/pkg/utils/utils.go

@@ -2,12 +2,13 @@ package utils
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
 	"os/signal"
-	"reflect"
 	"syscall"
 	"time"
 
@@ -44,17 +45,6 @@ func StartServer(app *gin.Engine) {
 	}
 }
 
-func ReverseSlice(data interface{}) {
-	value := reflect.ValueOf(data)
-	valueLen := value.Len()
-	for i := 0; i <= int((valueLen-1)/2); i++ {
-		reverseIndex := valueLen - 1 - i
-		tmp := value.Index(reverseIndex).Interface()
-		value.Index(reverseIndex).Set(value.Index(i))
-		value.Index(i).Set(reflect.ValueOf(tmp))
-	}
-}
-
 func CheckErr(e error) {
 	if e != nil {
 		logger.Log.Errorf("%v", e)
@@ -71,3 +61,27 @@ func SetHostname(address, newHostname string) string {
 	return replacedUrl.String()
 
 }
+
+func ReadJsonFile(filePath string, value interface{}) error {
+	if content, err := ioutil.ReadFile(filePath); err != nil {
+		return err
+	} else {
+		if err = json.Unmarshal(content, value); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func SaveJsonFile(filePath string, value interface{}) error {
+	if data, err := json.Marshal(value); err != nil {
+		return err
+	} else {
+		if err = ioutil.WriteFile(filePath, data, 0644); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cli/Makefile

@@ -14,12 +14,17 @@ help: ## This help.
 install:
 	go install mizu.go
 
+build-debug:
+	export GCLFAGS='-gcflags="all=-N -l"'
+	${MAKE} build
+
 build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
-	go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
+	go build ${GCLFAGS} -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
+					   -X 'github.com/up9inc/mizu/cli/mizu.Platform=$(SUFFIX)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.SemVer=$(SEM_VER)'" \
-					   -o bin/mizu_$(SUFFIX) mizu.go 
+					   -o bin/mizu_$(SUFFIX) mizu.go
 	(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
 
 build-all: ## Build for all supported platforms.

cli/README.md.TEMPLATE

@@ -1,6 +1,7 @@
 # Mizu release _SEM_VER_
+Full changelog for stable release see in [docs](https://github.com/up9inc/mizu/blob/main/docs/CHANGELOG.md)
 
-Download Mizu for your platform
+## Download Mizu for your platform
 
 **Mac** (Intel)
 ```

cli/apiserver/provider.go

@@ -3,7 +3,6 @@ package apiserver
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -40,8 +39,8 @@ func NewProvider(url string, retries int, timeout time.Duration) *Provider {
 func (provider *Provider) TestConnection() error {
 	retriesLeft := provider.retries
 	for retriesLeft > 0 {
-		if _, err := provider.GetHealthStatus(); err != nil {
-			logger.Log.Debugf("[ERROR] api server not ready yet %v", err)
+		if isReachable, err := provider.isReachable(); err != nil || !isReachable {
+			logger.Log.Debugf("api server not ready yet %v", err)
 		} else {
 			logger.Log.Debugf("connection test to api server passed successfully")
 			break
@@ -56,20 +55,31 @@ func (provider *Provider) TestConnection() error {
 	return nil
 }
 
-func (provider *Provider) GetHealthStatus() (*shared.HealthResponse, error) {
-	healthUrl := fmt.Sprintf("%s/status/health", provider.url)
-	if response, err := provider.client.Get(healthUrl); err != nil {
-		return nil, err
-	} else if response.StatusCode > 299 {
-		return nil, errors.New(fmt.Sprintf("status code: %d", response.StatusCode))
+func (provider *Provider) isReachable() (bool, error) {
+	echoUrl := fmt.Sprintf("%s/echo", provider.url)
+	if response, err := provider.client.Get(echoUrl); err != nil {
+		return false, err
+	} else if response.StatusCode != 200 {
+		return false, fmt.Errorf("invalid status code %v", response.StatusCode)
 	} else {
-		defer response.Body.Close()
+		return true, nil
+	}
+}
 
-		healthResponse := &shared.HealthResponse{}
-		if err := json.NewDecoder(response.Body).Decode(&healthResponse); err != nil {
-			return nil, err
+func (provider *Provider) ReportTapperStatus(tapperStatus shared.TapperStatus) error {
+	tapperStatusUrl := fmt.Sprintf("%s/status/tapperStatus", provider.url)
+
+	if jsonValue, err := json.Marshal(tapperStatus); err != nil {
+		return fmt.Errorf("failed Marshal the tapper status %w", err)
+	} else {
+		if response, err := provider.client.Post(tapperStatusUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+			return fmt.Errorf("failed sending to API server the tapped pods %w", err)
+		} else if response.StatusCode != 200 {
+			return fmt.Errorf("failed sending to API server the tapper status, response status code %v", response.StatusCode)
+		} else {
+			logger.Log.Debugf("Reported to server API about tapper status: %v", tapperStatus)
+			return nil
 		}
-		return healthResponse, nil
 	}
 }
 
@@ -77,9 +87,8 @@ func (provider *Provider) ReportTappedPods(pods []core.Pod) error {
 	tappedPodsUrl := fmt.Sprintf("%s/status/tappedPods", provider.url)
 
 	podInfos := kubernetes.GetPodInfosForPods(pods)
-	tapStatus := shared.TapStatus{Pods: podInfos}
 
-	if jsonValue, err := json.Marshal(tapStatus); err != nil {
+	if jsonValue, err := json.Marshal(podInfos); err != nil {
 		return fmt.Errorf("failed Marshal the tapped pods %w", err)
 	} else {
 		if response, err := provider.client.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {

cli/cmd/cleanRunner.go

@@ -1,6 +1,9 @@
 package cmd
 
-import "github.com/up9inc/mizu/cli/apiserver"
+import (
+	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/config"
+)
 
 func performCleanCommand() {
 	kubernetesProvider, err := getKubernetesProviderForCli()
@@ -8,5 +11,5 @@ func performCleanCommand() {
 		return
 	}
 
-	finishMizuExecution(kubernetesProvider, apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout))
+	finishMizuExecution(kubernetesProvider, apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout), config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
 }

cli/cmd/common.go

@@ -2,11 +2,17 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
-	"os"
-	"os/signal"
-	"syscall"
+	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/mizu/fsUtils"
+	"github.com/up9inc/mizu/cli/resources"
+	"github.com/up9inc/mizu/cli/telemetry"
+	"github.com/up9inc/mizu/shared"
+	"path"
+	"time"
 
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
@@ -31,22 +37,6 @@ func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel
 	logger.Log.Debugf("proxy ended")
 }
 
-func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
-	logger.Log.Debugf("waiting for finish...")
-	sigChan := make(chan os.Signal, 1)
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
-
-	// block until ctx cancel is called or termination signal is received
-	select {
-	case <-ctx.Done():
-		logger.Log.Debugf("ctx done")
-		break
-	case <-sigChan:
-		logger.Log.Debugf("Got termination signal, canceling execution...")
-		cancel()
-	}
-}
-
 func getKubernetesProviderForCli() (*kubernetes.Provider, error) {
 	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
 	if err != nil {
@@ -64,3 +54,31 @@ func handleKubernetesProviderError(err error) {
 		logger.Log.Error(err)
 	}
 }
+
+func finishMizuExecution(kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string) {
+	telemetry.ReportAPICalls(apiProvider)
+	removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
+	defer cancel()
+	dumpLogsIfNeeded(removalCtx, kubernetesProvider)
+	resources.CleanUpMizuResources(removalCtx, cancel, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace)
+}
+
+func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provider) {
+	if !config.Config.DumpLogs {
+		return
+	}
+	mizuDir := mizu.GetMizuFolderPath()
+	filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
+	if err := fsUtils.DumpLogs(ctx, kubernetesProvider, filePath); err != nil {
+		logger.Log.Errorf("Failed dump logs %v", err)
+	}
+}
+
+func getSerializedMizuAgentConfig(mizuAgentConfig *shared.MizuAgentConfig) (string, error) {
+	serializedConfig, err := json.Marshal(mizuAgentConfig)
+	if err != nil {
+		return "", err
+	}
+
+	return string(serializedConfig), nil
+}

cli/cmd/install.go

@@ -0,0 +1,30 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/telemetry"
+)
+
+var installCmd = &cobra.Command{
+	Use:   "install",
+	Short: "Installs mizu components",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("install", nil)
+		runMizuInstall()
+		return nil
+	},
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		if config.Config.IsNsRestrictedMode() {
+			return fmt.Errorf("install is not supported in restricted namespace mode")
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(installCmd)
+}
+

cli/cmd/installRunner.go

@@ -0,0 +1,153 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/up9inc/mizu/shared/kubernetes"
+	core "k8s.io/api/core/v1"
+	"regexp"
+	"time"
+
+	"github.com/creasty/defaults"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/errormessage"
+	"github.com/up9inc/mizu/cli/resources"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func runMizuInstall() {
+	kubernetesProvider, err := getKubernetesProviderForCli()
+	if err != nil {
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel() // cancel will be called when this function exits
+
+	var serializedValidationRules string
+	var serializedContract string
+
+	var defaultMaxEntriesDBSizeBytes int64 = 200 * 1000 * 1000
+
+	defaultResources := shared.Resources{}
+	defaults.Set(&defaultResources)
+
+	mizuAgentConfig := getInstallMizuAgentConfig(defaultMaxEntriesDBSizeBytes, defaultResources)
+	serializedMizuConfig, err := getSerializedMizuAgentConfig(mizuAgentConfig)
+	if err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error serializing mizu config: %v", errormessage.FormatError(err)))
+		return
+	}
+
+	if err = resources.CreateInstallMizuResources(ctx, kubernetesProvider, serializedValidationRules,
+		serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(),
+		config.Config.MizuResourcesNamespace, config.Config.AgentImage,
+		nil, defaultMaxEntriesDBSizeBytes, defaultResources, config.Config.ImagePullPolicy(),
+		config.Config.LogLevel(), false); err != nil {
+		var statusError *k8serrors.StatusError
+		if errors.As(err, &statusError) {
+			if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists {
+				logger.Log.Info("Mizu is already running in this namespace, run `mizu clean` to remove the currently running Mizu instance")
+			}
+		} else {
+			defer resources.CleanUpMizuResources(ctx, cancel, kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
+		}
+
+		return
+	}
+
+	logger.Log.Infof("Waiting for Mizu server to start...")
+	readyChan := make(chan string)
+	readyErrorChan := make(chan error)
+	go watchApiServerPodReady(ctx, kubernetesProvider, readyChan, readyErrorChan)
+
+	select {
+	case readyMessage := <-readyChan:
+		logger.Log.Infof(readyMessage)
+	case err := <-readyErrorChan:
+		defer resources.CleanUpMizuResources(ctx, cancel, kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("%v", errormessage.FormatError(err)))
+		return
+	}
+
+	logger.Log.Infof(uiUtils.Magenta, "Installation completed, run `mizu view` to connect to the mizu daemon instance")
+}
+
+func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Resources) *shared.MizuAgentConfig {
+	mizuAgentConfig := shared.MizuAgentConfig{
+		MaxDBSizeBytes:         maxDBSizeBytes,
+		AgentImage:             config.Config.AgentImage,
+		PullPolicy:             config.Config.ImagePullPolicyStr,
+		LogLevel:               config.Config.LogLevel(),
+		TapperResources:        tapperResources,
+		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
+		AgentDatabasePath:      shared.DataDirPath,
+		StandaloneMode:         true,
+		ServiceMap:             config.Config.ServiceMap,
+		OAS:                    config.Config.OAS,
+	}
+
+	return &mizuAgentConfig
+}
+
+func watchApiServerPodReady(ctx context.Context, kubernetesProvider *kubernetes.Provider, readyChan chan string, readyErrorChan chan error) {
+	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", kubernetes.ApiServerPodName))
+	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
+	eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper)
+
+	apiServerTimeoutSec := config.GetIntEnvConfig(config.ApiServerTimeoutSec, 120)
+	timeAfter := time.After(time.Duration(apiServerTimeoutSec) * time.Second)
+	for {
+		select {
+		case wEvent, ok := <-eventChan:
+			if !ok {
+				eventChan = nil
+				continue
+			}
+
+			switch wEvent.Type {
+			case kubernetes.EventAdded:
+				logger.Log.Debugf("Watching API Server pod ready loop, added")
+			case kubernetes.EventDeleted:
+				logger.Log.Debugf("Watching API Server pod ready loop, %s removed", kubernetes.ApiServerPodName)
+			case kubernetes.EventModified:
+				modifiedPod, err := wEvent.ToPod()
+				if err != nil {
+					readyErrorChan <- err
+					return
+				}
+
+				logger.Log.Debugf("Watching API Server pod ready loop, modified: %v", modifiedPod.Status.Phase)
+
+				if modifiedPod.Status.Phase == core.PodRunning {
+					readyChan <- fmt.Sprintf("%v pod is running", modifiedPod.Name)
+					return
+				}
+			case kubernetes.EventBookmark:
+				break
+			case kubernetes.EventError:
+				break
+			}
+		case err, ok := <-errorChan:
+			if !ok {
+				errorChan = nil
+				continue
+			}
+
+			readyErrorChan <- fmt.Errorf("[ERROR] Agent creation, watching %v namespace, error: %v", config.Config.MizuResourcesNamespace, err)
+			return
+		case <-timeAfter:
+			readyErrorChan <- fmt.Errorf("mizu API server was not ready in time")
+			return
+		case <-ctx.Done():
+			logger.Log.Debugf("Watching API Server pod ready loop, ctx done")
+			return
+		}
+	}
+}

cli/cmd/root.go

@@ -23,6 +23,7 @@ Further info is available at https://github.com/up9inc/mizu`,
 		if err := config.InitConfig(cmd); err != nil {
 			logger.Log.Fatal(err)
 		}
+
 		return nil
 	},
 }

cli/cmd/tap.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"github.com/up9inc/mizu/cli/up9"
 	"os"
 
 	"github.com/creasty/defaults"
@@ -62,6 +63,12 @@ Supported protocols are HTTP and gRPC.`,
 						logger.Log.Errorf("failed to log in, err: %v", err)
 						return nil
 					}
+				} else if isValidToken := up9.IsTokenValid(config.Config.Auth.Token, config.Config.Auth.EnvName); !isValidToken {
+					logger.Log.Errorf("Token is not valid, please log in again to continue")
+					if err := auth.Login(); err != nil {
+						logger.Log.Errorf("failed to log in, err: %v", err)
+						return nil
+					}
 				}
 			}
 		}
@@ -112,5 +119,5 @@ func init() {
 	tapCmd.Flags().StringP(configStructs.WorkspaceTapName, "w", defaultTapConfig.Workspace, "Uploads traffic to your UP9 workspace for further analysis (requires auth)")
 	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules")
 	tapCmd.Flags().String(configStructs.ContractFile, defaultTapConfig.ContractFile, "OAS/Swagger file to validate to monitor the contracts")
-	tapCmd.Flags().Bool(configStructs.DaemonModeTapName, defaultTapConfig.DaemonMode, "Run mizu in daemon mode, detached from the cli")
+	tapCmd.Flags().Bool(configStructs.ServiceMeshName, defaultTapConfig.ServiceMesh, "Record decrypted traffic if the cluster is configured with a service mesh and with mtls")
 }

cli/cmd/tapRunner.go

@@ -4,27 +4,26 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/cli/cmd/goUtils"
 	"io/ioutil"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
-	"path"
 	"regexp"
 	"strings"
 	"time"
 
+	"github.com/up9inc/mizu/cli/resources"
+	"github.com/up9inc/mizu/cli/utils"
+
 	"github.com/getkin/kin-openapi/openapi3"
+	"gopkg.in/yaml.v3"
+	core "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/cmd/goUtils"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
-	"gopkg.in/yaml.v3"
-	core "k8s.io/api/core/v1"
-
-	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/kubernetes"
@@ -35,8 +34,8 @@ import (
 const cleanupTimeout = time.Minute
 
 type tapState struct {
-	apiServerService         *core.Service
-	tapperSyncer             *kubernetes.MizuTapperSyncer
+	startTime                time.Time
+	targetNamespaces         []string
 	mizuServiceAccountExists bool
 }
 
@@ -44,13 +43,11 @@ var state tapState
 var apiProvider *apiserver.Provider
 
 func RunMizuTap() {
-	mizuApiFilteringOptions, err := getMizuApiFilteringOptions()
-	apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
-	if err != nil {
-		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error parsing regex-masking: %v", errormessage.FormatError(err)))
-		return
-	}
+	state.startTime = time.Now()
 
+	apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
+
+	var err error
 	var serializedValidationRules string
 	if config.Config.Tap.EnforcePolicyFile != "" {
 		serializedValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile)
@@ -92,16 +89,17 @@ func RunMizuTap() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel() // cancel will be called when this function exits
 
-	targetNamespaces := getNamespaces(kubernetesProvider)
+	state.targetNamespaces = getNamespaces(kubernetesProvider)
 
-	serializedMizuConfig, err := config.GetSerializedMizuAgentConfig(targetNamespaces, mizuApiFilteringOptions)
+	mizuAgentConfig := getTapMizuAgentConfig()
+	serializedMizuConfig, err := getSerializedMizuAgentConfig(mizuAgentConfig)
 	if err != nil {
-		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error composing mizu config: %v", errormessage.FormatError(err)))
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error serializing mizu config: %v", errormessage.FormatError(err)))
 		return
 	}
 
 	if config.Config.IsNsRestrictedMode() {
-		if len(targetNamespaces) != 1 || !shared.Contains(targetNamespaces, config.Config.MizuResourcesNamespace) {
+		if len(state.targetNamespaces) != 1 || !shared.Contains(state.targetNamespaces, config.Config.MizuResourcesNamespace) {
 			logger.Log.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+
 				"You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, config.MizuResourcesNamespaceConfigName)
 			return
@@ -109,89 +107,82 @@ func RunMizuTap() {
 	}
 
 	var namespacesStr string
-	if !shared.Contains(targetNamespaces, kubernetes.K8sAllNamespaces) {
-		namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(targetNamespaces, "\", \""))
+	if !shared.Contains(state.targetNamespaces, kubernetes.K8sAllNamespaces) {
+		namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(state.targetNamespaces, "\", \""))
 	} else {
 		namespacesStr = "all namespaces"
 	}
 
 	logger.Log.Infof("Tapping pods in %s", namespacesStr)
 
+	if err := printTappedPodsPreview(ctx, kubernetesProvider, state.targetNamespaces); err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error listing pods: %v", errormessage.FormatError(err)))
+	}
+
 	if config.Config.Tap.DryRun {
 		return
 	}
 
-	if err := createMizuResources(ctx, cancel, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil {
-		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
-
+	logger.Log.Infof("Waiting for Mizu Agent to start...")
+	if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel()); err != nil {
 		var statusError *k8serrors.StatusError
 		if errors.As(err, &statusError) {
 			if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists {
 				logger.Log.Info("Mizu is already running in this namespace, change the `mizu-resources-namespace` configuration or run `mizu clean` to remove the currently running Mizu instance")
 			}
+		} else {
+			defer resources.CleanUpMizuResources(ctx, cancel, kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
 		}
+
 		return
 	}
-	if config.Config.Tap.DaemonMode {
-		if err := handleDaemonModePostCreation(cancel, kubernetesProvider); err != nil {
-			defer finishMizuExecution(kubernetesProvider, apiProvider)
-			cancel()
-		} else {
-			logger.Log.Infof(uiUtils.Magenta, "Mizu is now running in daemon mode, run `mizu view` to connect to the mizu daemon instance")
-		}
-	} else {
-		defer finishMizuExecution(kubernetesProvider, apiProvider)
 
-		if err = startTapperSyncer(ctx, cancel, kubernetesProvider, targetNamespaces, *mizuApiFilteringOptions); err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", err))
-			cancel()
-		}
+	defer finishMizuExecution(kubernetesProvider, apiProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
 
-		go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel)
-		go goUtils.HandleExcWrapper(watchTapperPod, ctx, kubernetesProvider, cancel)
+	go goUtils.HandleExcWrapper(watchApiServerEvents, ctx, kubernetesProvider, cancel)
+	go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel)
 
-		// block until exit signal or error
-		waitForFinish(ctx, cancel)
-	}
+	// block until exit signal or error
+	utils.WaitForFinish(ctx, cancel)
 }
 
-func handleDaemonModePostCreation(cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) error {
-	apiProvider := apiserver.NewProvider(GetApiServerUrl(), 90, 1*time.Second)
-
-	if err := waitForDaemonModeToBeReady(cancel, kubernetesProvider, apiProvider); err != nil {
-		return err
-	}
-	if err := printDaemonModeTappedPods(apiProvider); err != nil {
-		return err
+func getTapMizuAgentConfig() *shared.MizuAgentConfig {
+	mizuAgentConfig := shared.MizuAgentConfig{
+		MaxDBSizeBytes:         config.Config.Tap.MaxEntriesDBSizeBytes(),
+		AgentImage:             config.Config.AgentImage,
+		PullPolicy:             config.Config.ImagePullPolicyStr,
+		LogLevel:               config.Config.LogLevel(),
+		TapperResources:        config.Config.Tap.TapperResources,
+		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
+		AgentDatabasePath:      shared.DataDirPath,
+		ServiceMap:             config.Config.ServiceMap,
+		OAS:                    config.Config.OAS,
 	}
 
-	return nil
+	return &mizuAgentConfig
 }
 
-func printDaemonModeTappedPods(apiProvider *apiserver.Provider) error {
-	if healthStatus, err := apiProvider.GetHealthStatus(); err != nil {
+/*
+this function is a bit problematic as it might be detached from the actual pods the mizu api server will tap.
+The alternative would be to wait for api server to be ready and then query it for the pods it listens to, this has
+the arguably worse drawback of taking a relatively very long time before the user sees which pods are targeted, if any.
+*/
+func printTappedPodsPreview(ctx context.Context, kubernetesProvider *kubernetes.Provider, namespaces []string) error {
+	if matchingPods, err := kubernetesProvider.ListAllRunningPodsMatchingRegex(ctx, config.Config.Tap.PodRegex(), namespaces); err != nil {
 		return err
 	} else {
-		for _, tappedPod := range healthStatus.TapStatus.Pods {
+		if len(matchingPods) == 0 {
+			printNoPodsFoundSuggestion(namespaces)
+		}
+		for _, tappedPod := range matchingPods {
 			logger.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", tappedPod.Name))
 		}
+		return nil
 	}
-	return nil
 }
 
-func waitForDaemonModeToBeReady(cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider) error {
-	logger.Log.Info("Waiting for mizu to be ready... (may take a few minutes)")
-	go startProxyReportErrorIfAny(kubernetesProvider, cancel)
-
-	// TODO: TRA-3903 add a smarter test to see that tapping/pod watching is functioning properly
-	if err := apiProvider.TestConnection(); err != nil {
-		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Mizu was not ready in time, for more info check logs at %s", fsUtils.GetLogFilePath()))
-		return err
-	}
-	return nil
-}
-
-func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, mizuApiFilteringOptions api.TrafficFilteringOptions) error {
+func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, mizuApiFilteringOptions api.TrafficFilteringOptions, startTime time.Time) error {
 	tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
 		TargetNamespaces:         targetNamespaces,
 		PodFilterRegex:           *config.Config.Tap.PodRegex(),
@@ -199,28 +190,17 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider
 		AgentImage:               config.Config.AgentImage,
 		TapperResources:          config.Config.Tap.TapperResources,
 		ImagePullPolicy:          config.Config.ImagePullPolicy(),
-		DumpLogs:                 config.Config.DumpLogs,
+		LogLevel:                 config.Config.LogLevel(),
 		IgnoredUserAgents:        config.Config.Tap.IgnoredUserAgents,
 		MizuApiFilteringOptions:  mizuApiFilteringOptions,
 		MizuServiceAccountExists: state.mizuServiceAccountExists,
-	})
+		ServiceMesh:              config.Config.Tap.ServiceMesh,
+	}, startTime)
 
 	if err != nil {
 		return err
 	}
 
-	for _, tappedPod := range tapperSyncer.CurrentlyTappedPods {
-		logger.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", tappedPod.Name))
-	}
-
-	if len(tapperSyncer.CurrentlyTappedPods) == 0 {
-		var suggestionStr string
-		if !shared.Contains(targetNamespaces, kubernetes.K8sAllNamespaces) {
-			suggestionStr = ". Select a different namespace with -n or tap all namespaces with -A"
-		}
-		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr))
-	}
-
 	go func() {
 		for {
 			select {
@@ -239,6 +219,14 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider
 				if err := apiProvider.ReportTappedPods(tapperSyncer.CurrentlyTappedPods); err != nil {
 					logger.Log.Debugf("[Error] failed update tapped pods %v", err)
 				}
+			case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut:
+				if !ok {
+					logger.Log.Debug("mizuTapperSyncer tapper status changed channel closed, ending listener loop")
+					return
+				}
+				if err := apiProvider.ReportTapperStatus(tapperStatus); err != nil {
+					logger.Log.Debugf("[Error] failed update tapper status %v", err)
+				}
 			case <-ctx.Done():
 				logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")
 				return
@@ -246,11 +234,17 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider
 		}
 	}()
 
-	state.tapperSyncer = tapperSyncer
-
 	return nil
 }
 
+func printNoPodsFoundSuggestion(targetNamespaces []string) {
+	var suggestionStr string
+	if !shared.Contains(targetNamespaces, kubernetes.K8sAllNamespaces) {
+		suggestionStr = ". You can also try selecting a different namespace with -n or tap all namespaces with -A"
+	}
+	logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any currently running pods that match the regex argument, mizu will automatically tap matching pods if any are created later%s", suggestionStr))
+}
+
 func getErrorDisplayTextForK8sTapManagerError(err kubernetes.K8sTapManagerError) string {
 	switch err.TapManagerReason {
 	case kubernetes.TapManagerPodListError:
@@ -273,118 +267,6 @@ func readValidationRules(file string) (string, error) {
 	return string(newContent), nil
 }
 
-func createMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error {
-	if !config.Config.IsNsRestrictedMode() {
-		if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
-			return err
-		}
-	}
-
-	if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil {
-		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err)))
-	}
-
-	var err error
-	state.mizuServiceAccountExists, err = createRBACIfNecessary(ctx, kubernetesProvider)
-	if err != nil {
-		if !config.Config.Tap.DaemonMode {
-			logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err)))
-		}
-	}
-
-	var serviceAccountName string
-	if state.mizuServiceAccountExists {
-		serviceAccountName = kubernetes.ServiceAccountName
-	} else {
-		serviceAccountName = ""
-	}
-
-	opts := &kubernetes.ApiServerOptions{
-		Namespace:             config.Config.MizuResourcesNamespace,
-		PodName:               kubernetes.ApiServerPodName,
-		PodImage:              config.Config.AgentImage,
-		ServiceAccountName:    serviceAccountName,
-		IsNamespaceRestricted: config.Config.IsNsRestrictedMode(),
-		SyncEntriesConfig:     getSyncEntriesConfig(),
-		MaxEntriesDBSizeBytes: config.Config.Tap.MaxEntriesDBSizeBytes(),
-		Resources:             config.Config.Tap.ApiServerResources,
-		ImagePullPolicy:       config.Config.ImagePullPolicy(),
-	}
-
-	if config.Config.Tap.DaemonMode {
-		if !state.mizuServiceAccountExists {
-			defer cleanUpMizuResources(ctx, cancel, kubernetesProvider)
-			logger.Log.Fatalf(uiUtils.Red, fmt.Sprintf("Failed to ensure the resources required for mizu to run in daemon mode. cannot proceed. error: %v", errormessage.FormatError(err)))
-		}
-		if err := createMizuApiServerDeployment(ctx, kubernetesProvider, opts); err != nil {
-			return err
-		}
-	} else {
-		if err := createMizuApiServerPod(ctx, kubernetesProvider, opts); err != nil {
-			return err
-		}
-	}
-
-	state.apiServerService, err = kubernetesProvider.CreateService(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName, kubernetes.ApiServerPodName)
-	if err != nil {
-		return err
-	}
-	logger.Log.Debugf("Successfully created service: %s", kubernetes.ApiServerPodName)
-
-	return nil
-}
-
-func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error {
-	err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, kubernetes.ConfigMapName, serializedValidationRules, serializedContract, serializedMizuConfig)
-	return err
-}
-
-func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider) error {
-	_, err := kubernetesProvider.CreateNamespace(ctx, config.Config.MizuResourcesNamespace)
-	return err
-}
-
-func createMizuApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions) error {
-	pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, false, "")
-	if err != nil {
-		return err
-	}
-	if _, err = kubernetesProvider.CreatePod(ctx, config.Config.MizuResourcesNamespace, pod); err != nil {
-		return err
-	}
-	logger.Log.Debugf("Successfully created API server pod: %s", kubernetes.ApiServerPodName)
-	return nil
-}
-
-func createMizuApiServerDeployment(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions) error {
-	isDefaultStorageClassAvailable, err := kubernetesProvider.IsDefaultStorageProviderAvailable(ctx)
-	volumeClaimCreated := false
-	if err != nil {
-		return err
-	}
-	if isDefaultStorageClassAvailable {
-		if _, err = kubernetesProvider.CreatePersistentVolumeClaim(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName, config.Config.Tap.MaxEntriesDBSizeBytes()+mizu.DaemonModePersistentVolumeSizeBufferBytes); err != nil {
-			logger.Log.Warningf(uiUtils.Yellow, "An error has occured while creating a persistent volume claim for mizu, this will mean that mizu's data will be lost on pod restart")
-			logger.Log.Debugf("error creating persistent volume claim: %v", err)
-		} else {
-			volumeClaimCreated = true
-		}
-	} else {
-		logger.Log.Warningf(uiUtils.Yellow, "Could not find default volume provider in this cluster, this will mean that mizu's data will be lost on pod restart")
-	}
-
-	pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, volumeClaimCreated, kubernetes.PersistentVolumeClaimName)
-	if err != nil {
-		return err
-	}
-
-	if _, err = kubernetesProvider.CreateDeployment(ctx, config.Config.MizuResourcesNamespace, opts.PodName, pod); err != nil {
-		return err
-	}
-	logger.Log.Debugf("Successfully created API server deployment: %s", kubernetes.ApiServerPodName)
-	return nil
-}
-
 func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
 	var compiledRegexSlice []*api.SerializableRegexp
 
@@ -419,216 +301,47 @@ func getSyncEntriesConfig() *shared.SyncEntriesConfig {
 	}
 }
 
-func finishMizuExecution(kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider) {
-	telemetry.ReportAPICalls(apiProvider)
-	removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
-	defer cancel()
-	dumpLogsIfNeeded(removalCtx, kubernetesProvider)
-	cleanUpMizuResources(removalCtx, cancel, kubernetesProvider)
-}
-
-func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provider) {
-	if !config.Config.DumpLogs {
-		return
-	}
-	mizuDir := mizu.GetMizuFolderPath()
-	filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
-	if err := fsUtils.DumpLogs(ctx, kubernetesProvider, filePath); err != nil {
-		logger.Log.Errorf("Failed dump logs %v", err)
-	}
-}
-
-func cleanUpMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
-	logger.Log.Infof("\nRemoving mizu resources\n")
-
-	var leftoverResources []string
-
-	if config.Config.IsNsRestrictedMode() {
-		leftoverResources = cleanUpRestrictedMode(ctx, kubernetesProvider)
-	} else {
-		leftoverResources = cleanUpNonRestrictedMode(ctx, cancel, kubernetesProvider)
-	}
-
-	if len(leftoverResources) > 0 {
-		errMsg := fmt.Sprintf("Failed to remove the following resources, for more info check logs at %s:", fsUtils.GetLogFilePath())
-		for _, resource := range leftoverResources {
-			errMsg += "\n- " + resource
-		}
-		logger.Log.Errorf(uiUtils.Error, errMsg)
-	}
-}
-
-func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.Provider) []string {
-	leftoverResources := make([]string, 0)
-
-	if err := kubernetesProvider.RemoveService(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
-		resourceDesc := fmt.Sprintf("Service %s in namespace %s", kubernetes.ApiServerPodName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveDaemonSet(ctx, config.Config.MizuResourcesNamespace, kubernetes.TapperDaemonSetName); err != nil {
-		resourceDesc := fmt.Sprintf("DaemonSet %s in namespace %s", kubernetes.TapperDaemonSetName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveConfigMap(ctx, config.Config.MizuResourcesNamespace, kubernetes.ConfigMapName); err != nil {
-		resourceDesc := fmt.Sprintf("ConfigMap %s in namespace %s", kubernetes.ConfigMapName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveServicAccount(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName); err != nil {
-		resourceDesc := fmt.Sprintf("Service Account %s in namespace %s", kubernetes.ServiceAccountName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveRole(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleName); err != nil {
-		resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.RoleName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemovePod(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
-		resourceDesc := fmt.Sprintf("Pod %s in namespace %s", kubernetes.ApiServerPodName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	//daemon mode resources
-	if err := kubernetesProvider.RemoveRoleBinding(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleBindingName); err != nil {
-		resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.RoleBindingName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveDeployment(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
-		resourceDesc := fmt.Sprintf("Deployment %s in namespace %s", kubernetes.ApiServerPodName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemovePersistentVolumeClaim(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName); err != nil {
-		resourceDesc := fmt.Sprintf("PersistentVolumeClaim %s in namespace %s", kubernetes.PersistentVolumeClaimName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveRole(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleName); err != nil {
-		resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.DaemonRoleName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveRoleBinding(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleBindingName); err != nil {
-		resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.DaemonRoleBindingName, config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	return leftoverResources
-}
-
-func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) []string {
-	leftoverResources := make([]string, 0)
-
-	if err := kubernetesProvider.RemoveNamespace(ctx, config.Config.MizuResourcesNamespace); err != nil {
-		resourceDesc := fmt.Sprintf("Namespace %s", config.Config.MizuResourcesNamespace)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	} else {
-		defer waitUntilNamespaceDeleted(ctx, cancel, kubernetesProvider)
-	}
-
-	if err := kubernetesProvider.RemoveClusterRole(ctx, kubernetes.ClusterRoleName); err != nil {
-		resourceDesc := fmt.Sprintf("ClusterRole %s", kubernetes.ClusterRoleName)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	if err := kubernetesProvider.RemoveClusterRoleBinding(ctx, kubernetes.ClusterRoleBindingName); err != nil {
-		resourceDesc := fmt.Sprintf("ClusterRoleBinding %s", kubernetes.ClusterRoleBindingName)
-		handleDeletionError(err, resourceDesc, &leftoverResources)
-	}
-
-	return leftoverResources
-}
-
-func handleDeletionError(err error, resourceDesc string, leftoverResources *[]string) {
-	logger.Log.Debugf("Error removing %s: %v", resourceDesc, errormessage.FormatError(err))
-	*leftoverResources = append(*leftoverResources, resourceDesc)
-}
-
-func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
-	// Call cancel if a terminating signal was received. Allows user to skip the wait.
-	go func() {
-		waitForFinish(ctx, cancel)
-	}()
-
-	if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, config.Config.MizuResourcesNamespace); err != nil {
-		switch {
-		case ctx.Err() == context.Canceled:
-			logger.Log.Debugf("Do nothing. User interrupted the wait")
-		case err == wait.ErrWaitTimeout:
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", config.Config.MizuResourcesNamespace))
-		default:
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error while waiting for Namespace %s to be deleted: %v", config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
-		}
-	}
-}
-
 func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", kubernetes.ApiServerPodName))
-	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
+	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
+	eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper)
 	isPodReady := false
-	timeAfter := time.After(25 * time.Second)
+
+	apiServerTimeoutSec := config.GetIntEnvConfig(config.ApiServerTimeoutSec, 120)
+	timeAfter := time.After(time.Duration(apiServerTimeoutSec) * time.Second)
 	for {
 		select {
-		case _, ok := <-added:
+		case wEvent, ok := <-eventChan:
 			if !ok {
-				added = nil
+				eventChan = nil
 				continue
 			}
 
-			logger.Log.Debugf("Watching API Server pod loop, added")
-		case _, ok := <-removed:
-			if !ok {
-				removed = nil
-				continue
-			}
-
-			logger.Log.Infof("%s removed", kubernetes.ApiServerPodName)
-			cancel()
-			return
-		case modifiedPod, ok := <-modified:
-			if !ok {
-				modified = nil
-				continue
-			}
-
-			logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
-
-			if modifiedPod.Status.Phase == core.PodPending {
-				if modifiedPod.Status.Conditions[0].Type == core.PodScheduled && modifiedPod.Status.Conditions[0].Status != core.ConditionTrue {
-					logger.Log.Debugf("Wasn't able to deploy the API server. Reason: \"%s\"", modifiedPod.Status.Conditions[0].Message)
-					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Wasn't able to deploy the API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
+			switch wEvent.Type {
+			case kubernetes.EventAdded:
+				logger.Log.Debugf("Watching API Server pod loop, added")
+			case kubernetes.EventDeleted:
+				logger.Log.Infof("%s removed", kubernetes.ApiServerPodName)
+				cancel()
+				return
+			case kubernetes.EventModified:
+				modifiedPod, err := wEvent.ToPod()
+				if err != nil {
+					logger.Log.Errorf(uiUtils.Error, err)
 					cancel()
-					break
+					continue
 				}
 
-				if len(modifiedPod.Status.ContainerStatuses) > 0 && modifiedPod.Status.ContainerStatuses[0].State.Waiting != nil && modifiedPod.Status.ContainerStatuses[0].State.Waiting.Reason == "ErrImagePull" {
-					logger.Log.Debugf("Wasn't able to deploy the API server. (ErrImagePull) Reason: \"%s\"", modifiedPod.Status.ContainerStatuses[0].State.Waiting.Message)
-					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Wasn't able to deploy the API server: failed to pull the image, for more info check logs at %v", fsUtils.GetLogFilePath()))
-					cancel()
-					break
-				}
-			}
+				logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
 
-			if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
-				isPodReady = true
-				go startProxyReportErrorIfAny(kubernetesProvider, cancel)
-
-				url := GetApiServerUrl()
-				if err := apiProvider.TestConnection(); err != nil {
-					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
-					cancel()
-					break
-				}
-
-				logger.Log.Infof("Mizu is available at %s\n", url)
-				uiUtils.OpenBrowser(url)
-				if err := apiProvider.ReportTappedPods(state.tapperSyncer.CurrentlyTappedPods); err != nil {
-					logger.Log.Debugf("[Error] failed update tapped pods %v", err)
+				if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
+					isPodReady = true
+					postApiServerStarted(ctx, kubernetesProvider, cancel, err)
 				}
+			case kubernetes.EventBookmark:
+				break
+			case kubernetes.EventError:
+				break
 			}
 		case err, ok := <-errorChan:
 			if !ok {
@@ -651,72 +364,78 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 	}
 }
 
-func watchTapperPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
-	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", kubernetes.TapperDaemonSetName))
-	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
-	var prevPodPhase core.PodPhase
+func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s", kubernetes.ApiServerPodName))
+	eventWatchHelper := kubernetes.NewEventWatchHelper(kubernetesProvider, podExactRegex, "pod")
+	eventChan, errorChan := kubernetes.FilteredWatch(ctx, eventWatchHelper, []string{config.Config.MizuResourcesNamespace}, eventWatchHelper)
 	for {
 		select {
-		case addedPod, ok := <-added:
+		case wEvent, ok := <-eventChan:
 			if !ok {
-				added = nil
+				eventChan = nil
 				continue
 			}
 
-			logger.Log.Debugf("Tapper is created [%s]", addedPod.Name)
-		case removedPod, ok := <-removed:
-			if !ok {
-				removed = nil
+			event, err := wEvent.ToEvent()
+			if err != nil {
+				logger.Log.Debugf("[ERROR] parsing Mizu resource event: %+v", err)
 				continue
 			}
 
-			logger.Log.Debugf("Tapper is removed [%s]", removedPod.Name)
-		case modifiedPod, ok := <-modified:
-			if !ok {
-				modified = nil
+			if state.startTime.After(event.CreationTimestamp.Time) {
 				continue
 			}
 
-			if modifiedPod.Status.Phase == core.PodPending && modifiedPod.Status.Conditions[0].Type == core.PodScheduled && modifiedPod.Status.Conditions[0].Status != core.ConditionTrue {
-				logger.Log.Infof(uiUtils.Red, fmt.Sprintf("Wasn't able to deploy the tapper %s. Reason: \"%s\"", modifiedPod.Name, modifiedPod.Status.Conditions[0].Message))
+			logger.Log.Debugf(
+				fmt.Sprintf("Watching API server events loop, event %s, time: %v, resource: %s (%s), reason: %s, note: %s",
+					event.Name,
+					event.CreationTimestamp.Time,
+					event.Regarding.Name,
+					event.Regarding.Kind,
+					event.Reason,
+					event.Note))
+
+			switch event.Reason {
+			case "FailedScheduling", "Failed":
+				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Mizu API Server status: %s - %s", event.Reason, event.Note))
 				cancel()
 				break
 			}
-
-			podStatus := modifiedPod.Status
-			if podStatus.Phase == core.PodPending && prevPodPhase == podStatus.Phase {
-				logger.Log.Debugf("Tapper %s is %s", modifiedPod.Name, strings.ToLower(string(podStatus.Phase)))
-				continue
-			}
-			prevPodPhase = podStatus.Phase
-
-			if podStatus.Phase == core.PodRunning {
-				state := podStatus.ContainerStatuses[0].State
-				if state.Terminated != nil {
-					switch state.Terminated.Reason {
-					case "OOMKilled":
-						logger.Log.Infof(uiUtils.Red, fmt.Sprintf("Tapper %s was terminated (reason: OOMKilled). You should consider increasing machine resources.", modifiedPod.Name))
-					}
-				}
-			}
-
-			logger.Log.Debugf("Tapper %s is %s", modifiedPod.Name, strings.ToLower(string(podStatus.Phase)))
 		case err, ok := <-errorChan:
 			if !ok {
 				errorChan = nil
 				continue
 			}
 
-			logger.Log.Errorf("[Error] Error in mizu tapper watch, err: %v", err)
-			cancel()
-
+			logger.Log.Debugf("[Error] Watching API server events loop, error: %+v", err)
 		case <-ctx.Done():
-			logger.Log.Debugf("Watching tapper pod loop, ctx done")
+			logger.Log.Debugf("Watching API server events loop, ctx done")
 			return
 		}
 	}
 }
 
+func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, err error) {
+	go startProxyReportErrorIfAny(kubernetesProvider, cancel)
+
+	url := GetApiServerUrl()
+	if err := apiProvider.TestConnection(); err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
+		cancel()
+		return
+	}
+	options, _ := getMizuApiFilteringOptions()
+	if err = startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", err))
+		cancel()
+	}
+
+	logger.Log.Infof("Mizu is available at %s", url)
+	if !config.Config.HeadlessMode {
+		uiUtils.OpenBrowser(url)
+	}
+}
+
 func getNamespaces(kubernetesProvider *kubernetes.Provider) []string {
 	if config.Config.Tap.AllNamespaces {
 		return []string{kubernetes.K8sAllNamespaces}
@@ -730,21 +449,3 @@ func getNamespaces(kubernetesProvider *kubernetes.Provider) []string {
 		return []string{currentNamespace}
 	}
 }
-
-func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) (bool, error) {
-	if !config.Config.IsNsRestrictedMode() {
-		if err := kubernetesProvider.CreateMizuRBAC(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.ClusterRoleName, kubernetes.ClusterRoleBindingName, mizu.RBACVersion); err != nil {
-			return false, err
-		}
-	} else {
-		if err := kubernetesProvider.CreateMizuRBACNamespaceRestricted(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.RoleName, kubernetes.RoleBindingName, mizu.RBACVersion); err != nil {
-			return false, err
-		}
-	}
-	if config.Config.Tap.DaemonMode {
-		if err := kubernetesProvider.CreateDaemonsetRBAC(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.DaemonRoleName, kubernetes.DaemonRoleBindingName, mizu.RBACVersion); err != nil {
-			return false, err
-		}
-	}
-	return true, nil
-}

cli/cmd/viewRunner.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"github.com/up9inc/mizu/cli/utils"
 	"net/http"
 
 	"github.com/up9inc/mizu/cli/apiserver"
@@ -56,9 +57,11 @@ func runMizuView() {
 		return
 	}
 
-	logger.Log.Infof("Mizu is available at %s\n", url)
+	logger.Log.Infof("Mizu is available at %s", url)
 
-	uiUtils.OpenBrowser(url)
+	if !config.Config.HeadlessMode {
+		uiUtils.OpenBrowser(url)
+	}
 
 	if isCompatible, err := version.CheckVersionCompatibility(apiServerProvider); err != nil {
 		logger.Log.Errorf("Failed to check versions compatibility %v", err)
@@ -69,5 +72,5 @@ func runMizuView() {
 		return
 	}
 
-	waitForFinish(ctx, cancel)
+	utils.WaitForFinish(ctx, cancel)
 }

cli/config/config.go

@@ -3,9 +3,7 @@ package config
 import (
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/tap/api"
 	"io/ioutil"
-	"k8s.io/apimachinery/pkg/util/json"
 	"os"
 	"reflect"
 	"strconv"
@@ -51,6 +49,10 @@ func InitConfig(cmd *cobra.Command) error {
 
 	cmd.Flags().Visit(initFlag)
 
+	if err := Config.validate(); err != nil {
+		return fmt.Errorf("config validation failed, err: %v", err)
+	}
+
 	finalConfigPrettified, _ := uiUtils.PrettyJson(Config)
 	logger.Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified)
 
@@ -366,37 +368,3 @@ func setZeroForReadonlyFields(currentElem reflect.Value) {
 		}
 	}
 }
-
-func GetSerializedMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api.TrafficFilteringOptions) (string, error) {
-	mizuConfig, err := getMizuAgentConfig(targetNamespaces, mizuApiFilteringOptions)
-	if err != nil {
-		return "", err
-	}
-	serializedConfig, err := json.Marshal(mizuConfig)
-	if err != nil {
-		return "", err
-	}
-	return string(serializedConfig), nil
-}
-
-func getMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api.TrafficFilteringOptions) (*shared.MizuAgentConfig, error) {
-	serializableRegex, err := api.CompileRegexToSerializableRegexp(Config.Tap.PodRegexStr)
-	if err != nil {
-		return nil, err
-	}
-	config := shared.MizuAgentConfig{
-		TapTargetRegex:          *serializableRegex,
-		MaxDBSizeBytes:          Config.Tap.MaxEntriesDBSizeBytes(),
-		DaemonMode:              Config.Tap.DaemonMode,
-		TargetNamespaces:        targetNamespaces,
-		AgentImage:              Config.AgentImage,
-		PullPolicy:              Config.ImagePullPolicyStr,
-		DumpLogs:                Config.DumpLogs,
-		IgnoredUserAgents:       Config.Tap.IgnoredUserAgents,
-		TapperResources:         Config.Tap.TapperResources,
-		MizuResourcesNamespace:  Config.MizuResourcesNamespace,
-		MizuApiFilteringOptions: *mizuApiFilteringOptions,
-		AgentDatabasePath:       fmt.Sprintf("%s%s", shared.DataDirPath, "entries.db"),
-	}
-	return &config, nil
-}

cli/config/configStruct.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/mizu"
 	v1 "k8s.io/api/core/v1"
@@ -31,6 +32,18 @@ type ConfigStruct struct {
 	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`
 	KubeConfigPathStr      string                      `yaml:"kube-config-path"`
 	ConfigFilePath         string                      `yaml:"config-path,omitempty" readonly:""`
+	HeadlessMode           bool                        `yaml:"headless" default:"false"`
+	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
+	ServiceMap             bool                        `yaml:"service-map,omitempty" default:"false" readonly:""`
+	OAS                    bool                        `yaml:"oas,omitempty" default:"false" readonly:""`
+}
+
+func (config *ConfigStruct) validate() error {
+	if _, err := logging.LogLevel(config.LogLevelStr); err != nil {
+		return fmt.Errorf("%s is not a valid log level, err: %v", config.LogLevelStr, err)
+	}
+
+	return nil
 }
 
 func (config *ConfigStruct) SetDefaults() {
@@ -59,3 +72,8 @@ func (config *ConfigStruct) KubeConfigPath() string {
 	home := homedir.HomeDir()
 	return filepath.Join(home, ".kube", "config")
 }
+
+func (config *ConfigStruct) LogLevel() logging.Level {
+	logLevel, _ := logging.LogLevel(config.LogLevelStr)
+	return logLevel
+}

cli/config/configStructs/tapConfig.go

@@ -3,9 +3,10 @@ package configStructs
 import (
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/shared"
 	"regexp"
 
+	"github.com/up9inc/mizu/shared"
+
 	"github.com/up9inc/mizu/shared/units"
 )
 
@@ -21,29 +22,29 @@ const (
 	WorkspaceTapName              = "workspace"
 	EnforcePolicyFile             = "traffic-validation-file"
 	ContractFile                  = "contract"
-	DaemonModeTapName             = "daemon"
+	ServiceMeshName               = "service-mesh"
 )
 
 type TapConfig struct {
-	UploadIntervalSec      int              `yaml:"upload-interval" default:"10"`
-	PodRegexStr            string           `yaml:"regex" default:".*"`
-	GuiPort                uint16           `yaml:"gui-port" default:"8899"`
-	ProxyHost              string           `yaml:"proxy-host" default:"127.0.0.1"`
-	Namespaces             []string         `yaml:"namespaces"`
-	Analysis               bool             `yaml:"analysis" default:"false"`
-	AllNamespaces          bool             `yaml:"all-namespaces" default:"false"`
-	PlainTextFilterRegexes []string         `yaml:"regex-masking"`
-	IgnoredUserAgents      []string         `yaml:"ignored-user-agents"`
-	DisableRedaction       bool             `yaml:"no-redact" default:"false"`
-	HumanMaxEntriesDBSize  string           `yaml:"max-entries-db-size" default:"200MB"`
-	DryRun                 bool             `yaml:"dry-run" default:"false"`
-	Workspace              string           `yaml:"workspace"`
-	EnforcePolicyFile      string           `yaml:"traffic-validation-file"`
-	ContractFile           string           `yaml:"contract"`
-	AskUploadConfirmation  bool             `yaml:"ask-upload-confirmation" default:"true"`
-	ApiServerResources     shared.Resources `yaml:"api-server-resources"`
-	TapperResources        shared.Resources `yaml:"tapper-resources"`
-	DaemonMode             bool             `yaml:"daemon" default:"false"`
+	UploadIntervalSec       int              `yaml:"upload-interval" default:"10"`
+	PodRegexStr             string           `yaml:"regex" default:".*"`
+	GuiPort                 uint16           `yaml:"gui-port" default:"8899"`
+	ProxyHost               string           `yaml:"proxy-host" default:"127.0.0.1"`
+	Namespaces              []string         `yaml:"namespaces"`
+	Analysis                bool             `yaml:"analysis" default:"false"`
+	AllNamespaces           bool             `yaml:"all-namespaces" default:"false"`
+	PlainTextFilterRegexes  []string         `yaml:"regex-masking"`
+	IgnoredUserAgents       []string         `yaml:"ignored-user-agents"`
+	DisableRedaction        bool             `yaml:"no-redact" default:"false"`
+	HumanMaxEntriesDBSize   string           `yaml:"max-entries-db-size" default:"200MB"`
+	DryRun                  bool             `yaml:"dry-run" default:"false"`
+	Workspace               string           `yaml:"workspace"`
+	EnforcePolicyFile       string           `yaml:"traffic-validation-file"`
+	ContractFile            string           `yaml:"contract"`
+	AskUploadConfirmation   bool             `yaml:"ask-upload-confirmation" default:"true"`
+	ApiServerResources      shared.Resources `yaml:"api-server-resources"`
+	TapperResources         shared.Resources `yaml:"tapper-resources"`
+	ServiceMesh             bool             `yaml:"service-mesh" default:"false"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {

cli/config/envConfig.go

@@ -7,6 +7,7 @@ import (
 
 const (
 	ApiServerRetries = "API_SERVER_RETRIES"
+	ApiServerTimeoutSec = "API_SERVER_TIMEOUT_SEC"
 )
 
 func GetIntEnvConfig(key string, defaultValue int) int {

cli/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/getkin/kin-openapi v0.79.0
 	github.com/google/go-github/v37 v37.0.0
 	github.com/google/uuid v1.1.2
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	github.com/up9inc/mizu/shared v0.0.0