TRA-4202 role management (#688)

* WIP

* wip

* Update keto.yml, socket_routes.go, and 12 more files...

* fixes and docs

* Update api.js

* Update auth.go and api.js

* Update user_role_provider.go

* Update config_routes.go and api.js

* Update consts.go

.dockerignore

@@ -2,7 +2,6 @@
 .dockerignore
 .editorconfig
 .gitignore
-**/.env*
 Dockerfile
 Makefile
 LICENSE

.github/workflows/pr_validation.yml

@@ -44,3 +44,18 @@ jobs:
 
       - name: Build Agent
         run: make agent
+
+  build-ui:
+    name: Build UI
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Node 16
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Build UI
+        run: make ui

.github/workflows/publish.yml

@@ -51,12 +51,17 @@ jobs:
         id: meta
         uses: crazy-max/ghaction-docker-meta@v2
         with:
-          images: ${{ steps.base_image_step.outputs.image }}
+          images: |
+            ${{ steps.base_image_step.outputs.image }}
+            up9inc/mizu
           tags: |
-            type=sha
-            type=raw,${{ github.sha }}
             type=raw,${{ steps.versioning.outputs.version }}
       - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+      - name: Login to GCR
         uses: docker/login-action@v1
         with:
           registry: gcr.io

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14-slim AS site-build
+FROM node:16-slim AS site-build
 
 WORKDIR /app/ui-build
 
@@ -7,7 +7,7 @@ COPY ui/package-lock.json .
 RUN npm i
 COPY ui .
 RUN npm run build
-
+RUN npm run build-ent
 
 FROM golang:1.16-alpine AS builder
 # Set necessary environment variables needed for our image.
@@ -54,6 +54,7 @@ WORKDIR /app
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
 COPY --from=builder ["/app/agent/build/extensions", "extensions"]
 COPY --from=site-build ["/app/ui-build/build", "site"]
+COPY --from=site-build ["/app/ui-build/build-ent", "site-standalone"]
 RUN mkdir /app/data/
 
 # gin-gonic runs in debug mode without this

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui agent cli tap docker
+.PHONY: help ui extensions extensions-debug agent agent-debug cli tap docker
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -28,6 +28,9 @@ ui: ## Build UI.
 cli: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build
 
+cli-debug: ## Build CLI.
+	@echo "building cli"; cd cli && $(MAKE) build-debug
+
 build-cli-ci: ## Build CLI for CI.
 	@echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci
 
@@ -37,6 +40,12 @@ agent: ## Build agent.
 	${MAKE} extensions
 	@ls -l agent/build
 
+agent-debug: ## Build agent for debug.
+	@(echo "building mizu agent for debug.." )
+	@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
+	${MAKE} extensions-debug
+	@ls -l agent/build
+
 docker: ## Build and publish agent docker image.
 	$(MAKE) push-docker
 
@@ -62,7 +71,7 @@ push-cli: ## Build and publish CLI.
 	gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
-clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
+clean: clean-ui clean-agent clean-cli clean-docker clean-extensions ## Clean all build artifacts.
 
 clean-ui: ## Clean UI.
 	@(rm -rf ui/build ; echo "UI cleanup done" )
@@ -73,9 +82,15 @@ clean-agent: ## Clean agent.
 clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
+clean-extensions:  ## Clean extensions
+	@(rm -rf tap/extensions/*.so ; echo "Extensions cleanup done" )
+
 clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
+extensions-debug:
+	devops/build_extensions_debug.sh
+
 extensions:
 	devops/build_extensions.sh
 

README.md

@@ -1,5 +1,17 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
 
+<p align="center">
+    <a href="https://github.com/up9inc/mizu/releases/latest">
+        <img alt="GitHub Latest Release" src="https://img.shields.io/github/v/release/up9inc/mizu?logo=GitHub&style=flat-square">
+    </a>
+    <a href="https://github.com/up9inc/mizu/blob/main/LICENSE">
+        <img alt="GitHub License" src="https://img.shields.io/github/license/up9inc/mizu?logo=GitHub&style=flat-square">
+    </a>
+    <a href="https://join.slack.com/t/up9/shared_invite/zt-tfjnduli-QzlR8VV4Z1w3YnPIAJfhlQ">
+      <img alt="Slack" src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social">
+    </a>
+</p>
+
 # The API Traffic Viewer for Kubernetes
 
 A simple-yet-powerful API traffic viewer for Kubernetes enabling you to view all API communication between microservices to help your debug and troubleshoot regressions.
@@ -12,7 +24,7 @@ Think TCPDump and Wireshark re-invented for Kubernetes.
 
 - Simple and powerful CLI
 - Monitoring network traffic in real-time. Supported protocols:
-  - [HTTP/1.1](https://datatracker.ietf.org/doc/html/rfc2616) (REST, etc.)
+  - [HTTP/1.x](https://datatracker.ietf.org/doc/html/rfc2616) (REST, GraphQL, SOAP, etc.)
   - [HTTP/2](https://datatracker.ietf.org/doc/html/rfc7540) (gRPC)
   - [AMQP](https://www.rabbitmq.com/amqp-0-9-1-reference.html) (RabbitMQ, Apache Qpid, etc.)
   - [Apache Kafka](https://kafka.apache.org/protocol)

acceptanceTests/cypress.json

@@ -1,8 +0,0 @@
-{
-  "watchForFileChanges":false,
-  "viewportWidth": 1920,
-  "viewportHeight": 1080,
-  "video": false,
-  "screenshotOnRunFailure": false,
-  "testFiles": ["tests/GuiPort.js"]
-}

acceptanceTests/cypress/integration/configurations/Default.json

@@ -0,0 +1,22 @@
+{
+  "watchForFileChanges":false,
+  "viewportWidth": 1920,
+  "viewportHeight": 1080,
+  "video": false,
+  "screenshotOnRunFailure": false,
+  "testFiles": [
+    "tests/GuiPort.js",
+    "tests/MultipleNamespaces.js",
+    "tests/Redact.js",
+    "tests/NoRedact.js",
+    "tests/Regex.js",
+    "tests/RegexMasking.js"
+  ],
+
+  "env": {
+    "testUrl": "http://localhost:8899/",
+    "redactHeaderContent": "User-Header[REDACTED]",
+    "redactBodyContent": "{ \"User\": \"[REDACTED]\" }",
+    "regexMaskingBodyContent": "[REDACTED]"
+  }
+}

acceptanceTests/cypress/integration/configurations/HugeMizu.json

@@ -0,0 +1,14 @@
+{
+  "watchForFileChanges":false,
+  "viewportWidth": 1920,
+  "viewportHeight": 3500,
+  "video": false,
+  "screenshotOnRunFailure": false,
+  "testFiles": [
+    "tests/IgnoredUserAgents.js"
+  ],
+
+  "env": {
+    "testUrl": "http://localhost:8899/"
+  }
+}

acceptanceTests/cypress/integration/testHelpers/StatusBarHelper.js

@@ -0,0 +1,46 @@
+const columns = {podName : 1, namespace : 2, tapping : 3};
+const greenStatusImageSrc = '/static/media/success.662997eb.svg';
+
+function getDomPathInStatusBar(line, column) {
+    return `.expandedStatusBar > :nth-child(2) > > :nth-child(2) > :nth-child(${line}) > :nth-child(${column})`;
+}
+
+export function checkLine(line, expectedValues) {
+    cy.get(getDomPathInStatusBar(line, columns.podName)).invoke('text').then(podValue => {
+        const podName = getOnlyPodName(podValue);
+        expect(podName).to.equal(expectedValues.podName);
+
+        cy.get(getDomPathInStatusBar(line, columns.namespace)).invoke('text').then(namespaceValue => {
+            expect(namespaceValue).to.equal(expectedValues.namespace);
+            cy.get(getDomPathInStatusBar(line, columns.tapping)).children().should('have.attr', 'src', greenStatusImageSrc);
+        });
+    });
+}
+
+export function findLineAndCheck(expectedValues) {
+    cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(1)').then(pods => {
+        cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(2)').then(namespaces => {
+            // organizing namespaces array
+            const podObjectsArray = Object.values(pods ?? {});
+            const namespacesObjectsArray = Object.values(namespaces ?? {});
+            let lineNumber = -1;
+            namespacesObjectsArray.forEach((namespaceObj, index) => {
+                const currentLine = index + 1;
+                lineNumber = (namespaceObj.getAttribute && namespaceObj.innerHTML === expectedValues.namespace && (getOnlyPodName(podObjectsArray[index].innerHTML)) === expectedValues.podName) ? currentLine : lineNumber;
+            });
+            lineNumber === -1 ? throwError(expectedValues) : checkLine(lineNumber, expectedValues);
+        });
+    });
+}
+
+function throwError(expectedValues) {
+    throw new Error(`The pod named ${expectedValues.podName} doesn't match any namespace named ${expectedValues.namespace}`);
+}
+
+export function getExpectedDetailsDict(podName, namespace) {
+    return {podName : podName, namespace : namespace};
+}
+
+function getOnlyPodName(podElementFullStr) {
+    return podElementFullStr.substring(0, podElementFullStr.indexOf('-'));
+}

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -0,0 +1,9 @@
+export function isValueExistsInElement(shouldInclude, content, domPathToContainer){
+    it(`should ${shouldInclude ? '' : 'not'} include '${content}'`, function () {
+        cy.get(domPathToContainer).then(htmlText => {
+            const allTextString = htmlText.text();
+            if (allTextString.includes(content) !== shouldInclude)
+                throw new Error(`One of the containers part contains ${content}`)
+        });
+    });
+}

acceptanceTests/cypress/integration/tests/GuiPort.js

@@ -1,8 +1,8 @@
 it('check', function () {
-    cy.visit(`http://localhost:${Cypress.env('port')}/`)
+    cy.visit(`http://localhost:${Cypress.env('port')}/`);
 
-    cy.get('.header').should('be.visible')
-    cy.get('.TrafficPageHeader').should('be.visible')
-    cy.get('.TrafficPage-ListContainer').should('be.visible')
-    cy.get('.TrafficPage-Container').should('be.visible')
-})
+    cy.get('.header').should('be.visible');
+    cy.get('.TrafficPageHeader').should('be.visible');
+    cy.get('.TrafficPage-ListContainer').should('be.visible');
+    cy.get('.TrafficPage-Container').should('be.visible');
+});

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -0,0 +1,33 @@
+import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+it('going through each entry', function () {
+    cy.get('#total-entries').then(number => {
+        const getNum = () => {
+            const numOfEntries = number.text();
+            return parseInt(numOfEntries);
+        };
+        cy.wrap({ there: getNum }).invoke('there').should('be.gte', 25);
+
+        checkThatAllEntriesShown();
+
+        const entriesNum = getNum();
+        [...Array(entriesNum).keys()].map(checkEntry);
+    });
+});
+
+function checkThatAllEntriesShown() {
+    cy.get('#entries-length').then(number => {
+        if (number.text() === '1')
+            cy.get('[title="Fetch old records"]').click();
+    });
+}
+
+function checkEntry(entryIndex) {
+    cy.get(`#entry-${entryIndex}`).click();
+    cy.get('#tbody-Headers').should('be.visible');
+    isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
+}

acceptanceTests/cypress/integration/tests/MultipleNamespaces.js

@@ -0,0 +1,17 @@
+import {findLineAndCheck, getExpectedDetailsDict} from '../testHelpers/StatusBarHelper';
+
+it('opening', function () {
+    cy.visit(Cypress.env('testUrl'));
+    cy.get('.podsCount').trigger('mouseover');
+});
+
+[1, 2, 3].map(doItFunc);
+
+function doItFunc(number) {
+    const podName = Cypress.env(`name${number}`);
+    const namespace = Cypress.env(`namespace${number}`);
+
+    it(`verifying the pod (${podName}, ${namespace})`, function () {
+        findLineAndCheck(getExpectedDetailsDict(podName, namespace));
+    });
+}

acceptanceTests/cypress/integration/tests/NoRedact.js

@@ -0,0 +1,8 @@
+import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+})
+
+isValueExistsInElement(false, Cypress.env('redactHeaderContent'), '#tbody-Headers');
+isValueExistsInElement(false, Cypress.env('redactBodyContent'), '.hljs');

acceptanceTests/cypress/integration/tests/Redact.js

@@ -0,0 +1,8 @@
+import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+})
+
+isValueExistsInElement(true, Cypress.env('redactHeaderContent'), '#tbody-Headers');
+isValueExistsInElement(true, Cypress.env('redactBodyContent'), '.hljs');

acceptanceTests/cypress/integration/tests/Regex.js

@@ -0,0 +1,11 @@
+import {getExpectedDetailsDict, checkLine} from '../testHelpers/StatusBarHelper';
+
+
+it('opening', function () {
+    cy.visit(Cypress.env('testUrl'));
+    cy.get('.podsCount').trigger('mouseover');
+
+    cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) >').should('have.length', 1); // one line
+
+    checkLine(1, getExpectedDetailsDict(Cypress.env('name'), Cypress.env('namespace')));
+});

acceptanceTests/cypress/integration/tests/RegexMasking.js

@@ -0,0 +1,7 @@
+import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+})
+
+isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), '.hljs');

acceptanceTests/tap_test.go

@@ -3,7 +3,6 @@ package acceptanceTests
 import (
 	"archive/zip"
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -139,7 +138,7 @@ func TestTapGuiPort(t *testing.T) {
 				return
 			}
 
-			runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env port=%d", guiPort))
+			runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env port=%d --config-file cypress/integration/configurations/Default.json", guiPort))
 		})
 	}
 }
@@ -151,6 +150,7 @@ func TestTapAllNamespaces(t *testing.T) {
 
 	expectedPods := []PodDescriptor{
 		{Name: "httpbin", Namespace: "mizu-tests"},
+		{Name: "httpbin2", Namespace: "mizu-tests"},
 		{Name: "httpbin", Namespace: "mizu-tests2"},
 	}
 
@@ -184,25 +184,8 @@ func TestTapAllNamespaces(t *testing.T) {
 		return
 	}
 
-	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
-	requestResult, requestErr := executeHttpGetRequest(podsUrl)
-	if requestErr != nil {
-		t.Errorf("failed to get tap status, err: %v", requestErr)
-		return
-	}
-
-	pods, err := getPods(requestResult)
-	if err != nil {
-		t.Errorf("failed to get pods, err: %v", err)
-		return
-	}
-
-	for _, expectedPod := range expectedPods {
-		if !isPodDescriptorInPodArray(pods, expectedPod) {
-			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
-			return
-		}
-	}
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v --config-file cypress/integration/configurations/Default.json",
+		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
 }
 
 func TestTapMultipleNamespaces(t *testing.T) {
@@ -250,30 +233,8 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		return
 	}
 
-	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
-	requestResult, requestErr := executeHttpGetRequest(podsUrl)
-	if requestErr != nil {
-		t.Errorf("failed to get tap status, err: %v", requestErr)
-		return
-	}
-
-	pods, err := getPods(requestResult)
-	if err != nil {
-		t.Errorf("failed to get pods, err: %v", err)
-		return
-	}
-
-	if len(expectedPods) != len(pods) {
-		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
-		return
-	}
-
-	for _, expectedPod := range expectedPods {
-		if !isPodDescriptorInPodArray(pods, expectedPod) {
-			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
-			return
-		}
-	}
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v --config-file cypress/integration/configurations/Default.json",
+		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
 }
 
 func TestTapRegex(t *testing.T) {
@@ -318,30 +279,8 @@ func TestTapRegex(t *testing.T) {
 		return
 	}
 
-	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
-	requestResult, requestErr := executeHttpGetRequest(podsUrl)
-	if requestErr != nil {
-		t.Errorf("failed to get tap status, err: %v", requestErr)
-		return
-	}
-
-	pods, err := getPods(requestResult)
-	if err != nil {
-		t.Errorf("failed to get pods, err: %v", err)
-		return
-	}
-
-	if len(expectedPods) != len(pods) {
-		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
-		return
-	}
-
-	for _, expectedPod := range expectedPods {
-		if !isPodDescriptorInPodArray(pods, expectedPod) {
-			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
-			return
-		}
-	}
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Regex.js\" --env name=%v,namespace=%v --config-file cypress/integration/configurations/Default.json",
+		expectedPods[0].Name, expectedPods[0].Namespace))
 }
 
 func TestTapDryRun(t *testing.T) {
@@ -438,59 +377,7 @@ func TestTapRedact(t *testing.T) {
 		}
 	}
 
-	redactCheckFunc := func() error {
-		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
-
-		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
-		if err != nil {
-			return err
-		}
-		err = checkEntriesAtLeast(entries, 1)
-		if err != nil {
-			return err
-		}
-		firstEntry := entries[0]
-
-		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr := executeHttpGetRequest(entryUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entry, err: %v", requestErr)
-		}
-
-		entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		request := entry["request"].(map[string]interface{})
-
-		headers := request["_headers"].([]interface{})
-		for _, headerInterface := range headers {
-			header := headerInterface.(map[string]interface{})
-			if header["name"].(string) != "User-Header" {
-				continue
-			}
-
-			userHeader := header["value"].(string)
-			if userHeader != "[REDACTED]" {
-				return fmt.Errorf("unexpected result - user agent is not redacted")
-			}
-		}
-
-		postData := request["postData"].(map[string]interface{})
-		textDataStr := postData["text"].(string)
-
-		var textData map[string]string
-		if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
-			return fmt.Errorf("failed to parse text data, err: %v", parseErr)
-		}
-
-		if textData["User"] != "[REDACTED]" {
-			return fmt.Errorf("unexpected result - user in body is not redacted")
-		}
-
-		return nil
-	}
-	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Redact.js\" --config-file cypress/integration/configurations/Default.json"))
 }
 
 func TestTapNoRedact(t *testing.T) {
@@ -542,59 +429,7 @@ func TestTapNoRedact(t *testing.T) {
 		}
 	}
 
-	redactCheckFunc := func() error {
-		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
-
-		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
-		if err != nil {
-			return err
-		}
-		err = checkEntriesAtLeast(entries, 1)
-		if err != nil {
-			return err
-		}
-		firstEntry := entries[0]
-
-		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr := executeHttpGetRequest(entryUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entry, err: %v", requestErr)
-		}
-
-		entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		request := entry["request"].(map[string]interface{})
-
-		headers := request["_headers"].([]interface{})
-		for _, headerInterface := range headers {
-			header := headerInterface.(map[string]interface{})
-			if header["name"].(string) != "User-Header" {
-				continue
-			}
-
-			userHeader := header["value"].(string)
-			if userHeader == "[REDACTED]" {
-				return fmt.Errorf("unexpected result - user agent is redacted")
-			}
-		}
-
-		postData := request["postData"].(map[string]interface{})
-		textDataStr := postData["text"].(string)
-
-		var textData map[string]string
-		if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
-			return fmt.Errorf("failed to parse text data, err: %v", parseErr)
-		}
-
-		if textData["User"] == "[REDACTED]" {
-			return fmt.Errorf("unexpected result - user in body is redacted")
-		}
-
-		return nil
-	}
-	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/NoRedact.js\" --config-file cypress/integration/configurations/Default.json")
 }
 
 func TestTapRegexMasking(t *testing.T) {
@@ -645,41 +480,8 @@ func TestTapRegexMasking(t *testing.T) {
 		}
 	}
 
-	redactCheckFunc := func() error {
-		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+	runCypressTests(t, "npx cypress run --spec \"cypress/integration/tests/RegexMasking.js\" --config-file cypress/integration/configurations/Default.json")
 
-		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
-		if err != nil {
-			return err
-		}
-		err = checkEntriesAtLeast(entries, 1)
-		if err != nil {
-			return err
-		}
-		firstEntry := entries[0]
-
-		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr := executeHttpGetRequest(entryUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entry, err: %v", requestErr)
-		}
-
-		entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		request := entry["request"].(map[string]interface{})
-
-		postData := request["postData"].(map[string]interface{})
-		textData := postData["text"].(string)
-
-		if textData != "[REDACTED]" {
-			return fmt.Errorf("unexpected result - body is not redacted")
-		}
-
-		return nil
-	}
-	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
-		t.Errorf("%v", err)
-		return
-	}
 }
 
 func TestTapIgnoredUserAgents(t *testing.T) {
@@ -740,45 +542,7 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 		}
 	}
 
-	ignoredUserAgentsCheckFunc := func() error {
-		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
-
-		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
-		if err != nil {
-			return err
-		}
-		err = checkEntriesAtLeast(entries, 1)
-		if err != nil {
-			return err
-		}
-
-		for _, entryInterface := range entries {
-			entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface["id"])
-			requestResult, requestErr := executeHttpGetRequest(entryUrl)
-			if requestErr != nil {
-				return fmt.Errorf("failed to get entry, err: %v", requestErr)
-			}
-
-			entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-			request := entry["request"].(map[string]interface{})
-
-			headers := request["_headers"].([]interface{})
-			for _, headerInterface := range headers {
-				header := headerInterface.(map[string]interface{})
-				if header["name"].(string) != ignoredUserAgentCustomHeader {
-					continue
-				}
-
-				return fmt.Errorf("unexpected result - user agent is not ignored")
-			}
-		}
-
-		return nil
-	}
-	if err := retriesExecute(shortRetriesCount, ignoredUserAgentsCheckFunc); err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/IgnoredUserAgents.js\" --config-file cypress/integration/configurations/HugeMizu.json")
 }
 
 func TestTapDumpLogs(t *testing.T) {

acceptanceTests/testsUtils.go

@@ -183,16 +183,16 @@ func tryExecuteFunc(executeFunc func() error) (err interface{}) {
 }
 
 func waitTapPodsReady(apiServerUrl string) error {
-	resolvingUrl := fmt.Sprintf("%v/status/tappersCount", apiServerUrl)
+	resolvingUrl := fmt.Sprintf("%v/status/connectedTappersCount", apiServerUrl)
 	tapPodsReadyFunc := func() error {
 		requestResult, requestErr := executeHttpGetRequest(resolvingUrl)
 		if requestErr != nil {
 			return requestErr
 		}
 
-		tappersCount := requestResult.(float64)
-		if tappersCount == 0 {
-			return fmt.Errorf("no tappers running")
+		connectedTappersCount := requestResult.(float64)
+		if connectedTappersCount == 0 {
+			return fmt.Errorf("no connected tappers running")
 		}
 		time.Sleep(waitAfterTapPodsReady)
 		return nil

agent/go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
+	github.com/chanced/openapi v0.0.6
 	github.com/djherbis/atime v1.0.0
 	github.com/getkin/kin-openapi v0.76.0
 	github.com/gin-contrib/static v0.0.1
@@ -12,17 +13,19 @@ require (
 	github.com/go-playground/universal-translator v0.17.0
 	github.com/go-playground/validator/v10 v10.5.0
 	github.com/google/martian v2.1.0+incompatible
+	github.com/google/uuid v1.1.2
 	github.com/gorilla/websocket v1.4.2
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
+	github.com/ory/keto-client-go v0.7.0-alpha.1
 	github.com/ory/kratos-client-go v0.8.2-alpha.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/up9inc/basenine/client/go v0.0.0-20220107003657-7c0578359920
+	github.com/stretchr/testify v1.7.0
+	github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
-	golang.org/x/text v0.3.5 // indirect
 	k8s.io/api v0.21.2
 	k8s.io/apimachinery v0.21.2
 	k8s.io/client-go v0.21.2

agent/go.sum

@@ -54,7 +54,9 @@ github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YH
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -70,6 +72,11 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
+github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
+github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg=
+github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
+github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@@ -82,6 +89,12 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
+github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a h1:zG6t+4krPXcCKtLbjFvAh+fKN1d0qfD+RaCj+680OU8=
+github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a/go.mod h1:yhcmlFk1hxuZ+5XZbupzT/cEm/eE4ZvWbmsW1+Q/aZE=
+github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44 h1:4NOJMtvZaOA6cI2gkIuXk/2b5KTOvm/R4zyPy/yLCM4=
+github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44/go.mod h1:XVNfXN5kgZST4PQ0W/oBAHJku2OteCeHxjAbvfd0ARM=
+github.com/chanced/openapi v0.0.6 h1:2giGS47+T8/7MN2hfRHSIUPx86Qksk+J/ciIWcAM7hY=
+github.com/chanced/openapi v0.0.6/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -109,6 +122,7 @@ github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -119,6 +133,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
+github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -156,10 +172,22 @@ github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70t
 github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
 github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
 github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
+github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
 github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU=
+github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ=
+github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk=
+github.com/go-openapi/analysis v0.20.0 h1:UN09o0kNhleunxW7LR+KnltD0YrJ8FF03pSqvAN3Vro=
+github.com/go-openapi/analysis v0.20.0/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og=
 github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
 github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
 github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
+github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
+github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
+github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
+github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
+github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
+github.com/go-openapi/errors v0.20.1 h1:j23mMDtRxMwIobkpId7sWh7Ddcx4ivaoqUbfXx5P+a8=
+github.com/go-openapi/errors v0.20.1/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
 github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
@@ -170,32 +198,73 @@ github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3Hfo
 github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM=
+github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
 github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
 github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
 github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
 github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
+github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI=
 github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk=
+github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY=
+github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc=
+github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc=
+github.com/go-openapi/loads v0.20.0/go.mod h1:2LhKquiE513rN5xC6Aan6lYOSddlL8Mp20AW9kpviM4=
+github.com/go-openapi/loads v0.20.2 h1:z5p5Xf5wujMxS1y8aP+vxwW5qYT2zdJBbXKmQUG3lcc=
+github.com/go-openapi/loads v0.20.2/go.mod h1:hTVUotJ+UonAMMZsvakEgmWKgtulweO9vYP2bQYKA/o=
 github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA=
 github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
 github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4=
+github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo=
+github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98=
+github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk=
+github.com/go-openapi/runtime v0.20.0 h1:DEV4oYH28MqakaabtbxH0cjvlzFegi/15kfUVCfiZW0=
+github.com/go-openapi/runtime v0.20.0/go.mod h1:2WnLRxMiOUWNN0UZskSkxW0+WXdfB1KmqRKCFH+ZWYk=
 github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
+github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
+github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
+github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU=
+github.com/go-openapi/spec v0.20.0/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU=
+github.com/go-openapi/spec v0.20.1/go.mod h1:93x7oh+d+FQsmsieroS4cmR3u0p/ywH649a3qwC9OsQ=
+github.com/go-openapi/spec v0.20.3 h1:uH9RQ6vdyPSs2pSy9fL8QPspDF2AMIMPtmK5coSSjtQ=
+github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg=
 github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
 github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
 github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
+github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
 github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
+github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
 github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
+github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc=
+github.com/go-openapi/strfmt v0.20.0/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc=
+github.com/go-openapi/strfmt v0.20.2/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk=
+github.com/go-openapi/strfmt v0.20.3 h1:YVG4ZgPZ00km/lRHrIf7c6cKL5/4FAUtG2T9RxWAgDY=
+github.com/go-openapi/strfmt v0.20.3/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk=
 github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY=
+github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY=
+github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M=
+github.com/go-openapi/swag v0.19.13/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
+github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
 github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
+github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo=
 github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
+github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8=
+github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4=
+github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI=
+github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0=
+github.com/go-openapi/validate v0.20.3 h1:GZPPhhKSZrE8HjB4eEkoYAZmoWA4+tCemSgINH1/vKw=
+github.com/go-openapi/validate v0.20.3/go.mod h1:goDdqVGiigM3jChcrYJxD2joalke3ZXeftD16byIjA4=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
@@ -206,8 +275,34 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E=
 github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
+github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
+github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
+github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
+github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
+github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
+github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
+github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
+github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
+github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
+github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
+github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
 github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
+github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
+github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
+github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
+github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -242,6 +337,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -253,8 +349,10 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -316,7 +414,13 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
@@ -325,10 +429,15 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
+github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -347,9 +456,13 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
+github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
@@ -367,6 +480,12 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -376,12 +495,14 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -392,8 +513,12 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU=
 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
+github.com/ory/keto-client-go v0.7.0-alpha.1 h1:8iZz37j7YXFvFBxO/lAqFkGyQK7KysToQhAE4aNFzqE=
+github.com/ory/keto-client-go v0.7.0-alpha.1/go.mod h1:xtlyH3xz9lgzvYnuzFXeGwaui0Swc/hxnmqzY2g8AYs=
 github.com/ory/kratos-client-go v0.8.2-alpha.1 h1:YlKhGOSZjounlB9iY4xSWlqHbyLYkeLzlLk8ZL7/nEM=
 github.com/ory/kratos-client-go v0.8.2-alpha.1/go.mod h1:dOQIsar76K07wMPJD/6aMhrWyY+sFGEagLDLso1CpsA=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -401,6 +526,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -428,15 +555,21 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE=
+github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
@@ -445,6 +578,7 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
@@ -465,23 +599,40 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.12.0 h1:61wEp/qfvFnqKH/WCI3M8HuRut+mHT6Mr82QrFmM2SY=
+github.com/tidwall/gjson v1.12.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.3 h1:5+deguEhHSEjmuICXZ21uSSsXotWMA0orU783+Z7Cp8=
+github.com/tidwall/sjson v1.2.3/go.mod h1:5WdjKx3AQMvCJ4RG6/2UYT7dLrGvJUV1x4jdTAyGvZs=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/up9inc/basenine/client/go v0.0.0-20220107003657-7c0578359920 h1:QQpgRleNNpxxAG/rKmk4dwJh0jHyRaQz4QOVlPmqv1c=
-github.com/up9inc/basenine/client/go v0.0.0-20220107003657-7c0578359920/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d h1:WTz53dcfqCIWZpZLQoHbIcNc21s0ZHEZH7EqMPp99qQ=
+github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=
+github.com/wI2L/jsondiff v0.1.1/go.mod h1:bAbJSAJXZtfOCZ5y3v7Mfb6UQa3DGdGFjQj1cNv8EcM=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
+github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -490,6 +641,13 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
+go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
+go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
+go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
+go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
+go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
+go.mongodb.org/mongo-driver v1.5.1 h1:9nOVLGDfOaZ9R0tBumx/BcuqkbFpyTCU2r/Po7A2azI=
+go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -503,11 +661,14 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
@@ -576,11 +737,15 @@ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -594,6 +759,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -609,10 +775,13 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -665,6 +834,7 @@ golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiT
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -672,9 +842,13 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
@@ -819,6 +993,8 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
@@ -863,5 +1039,6 @@ sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

agent/keto/Dockerfile

@@ -0,0 +1,13 @@
+FROM oryd/keto:v0.7.0-alpha.1-sqlite
+
+USER root
+
+RUN apk add sqlite
+
+RUN mkdir -p /etc/config/keto
+
+COPY ./keto.yml /etc/config/keto/keto.yml
+COPY ./start.sh /opt/start.sh
+RUN chmod +x /opt/start.sh
+
+ENTRYPOINT ["/opt/start.sh"]

agent/keto/build-push-featurebranch.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+GCP_PROJECT=up9-docker-hub
+REPOSITORY=gcr.io/$GCP_PROJECT
+SERVER_NAME=mizu-keto
+GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
+
+DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
+SEM_VER=${SEM_VER=0.0.0}
+
+DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
+
+if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
+then
+  echo "Pushing to $GIT_BRANCH is allowed only via CI"
+  exit 1
+fi
+
+echo "building ${DOCKER_TAGGED_BUILDS[@]}"
+DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
+docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
+
+for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
+do
+  echo pushing "$DOCKER_TAG"
+  docker push "$DOCKER_TAG"
+done

agent/keto/keto.yml

@@ -0,0 +1,22 @@
+version: v0.7.0-alpha.1
+
+dsn: sqlite:///app/data/kratos.sqlite?_fk=true
+
+log:
+  level: info
+  format: text
+  leak_sensitive_values: false
+
+serve:
+  read:
+    host: 0.0.0.0
+    port: 4466
+  write:
+    host: 0.0.0.0
+    port: 4467
+
+namespaces:
+  - id: 0
+    name: system
+  - id: 1
+    name: workspaces

agent/keto/start.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+keto migrate up -c /etc/config/keto/keto.yml --yes # this initializes the db
+keto serve -c /etc/config/keto/keto.yml # start keto

agent/kratos/kratos.yml

@@ -57,7 +57,7 @@ selfservice:
 log:
   level: info
   format: text
-  leak_sensitive_values: true
+  leak_sensitive_values: false
 
 secrets:
   cookie:

agent/main.go

@@ -11,7 +11,9 @@ import (
 	"mizuserver/pkg/controllers"
 	"mizuserver/pkg/middlewares"
 	"mizuserver/pkg/models"
+	"mizuserver/pkg/oas"
 	"mizuserver/pkg/routes"
+	"mizuserver/pkg/servicemap"
 	"mizuserver/pkg/up9"
 	"mizuserver/pkg/utils"
 	"net/http"
@@ -120,14 +122,14 @@ func main() {
 
 		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-
+		enableExpFeatureIfNeeded()
 		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
 		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
 
 		syncEntriesConfig := getSyncEntriesConfig()
 		if syncEntriesConfig != nil {
 			if err := up9.SyncEntries(syncEntriesConfig); err != nil {
-				panic(fmt.Sprintf("Error syncing entries, err: %v", err))
+				logger.Log.Error("Error syncing entries, err: %v", err)
 			}
 		}
 
@@ -148,6 +150,15 @@ func main() {
 	logger.Log.Info("Exiting")
 }
 
+func enableExpFeatureIfNeeded() {
+	if config.Config.OAS {
+		oas.GetOasGeneratorInstance().Start()
+	}
+	if config.Config.ServiceMap {
+		servicemap.GetInstance().SetConfig(config.Config)
+	}
+}
+
 func configureBasenineServer(host string, port string) {
 	if !wait.New(
 		wait.WithProto("tcp"),
@@ -233,10 +244,15 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 
 	app.Use(DisableRootStaticCache())
 
-	if err := setUIMode(); err != nil {
+	if err := setUIFlags(); err != nil {
 		logger.Log.Errorf("Error setting ui mode, err: %v", err)
 	}
-	app.Use(static.ServeRoot("/", "./site"))
+
+	if config.Config.StandaloneMode {
+		app.Use(static.ServeRoot("/", "./site-standalone"))
+	} else {
+		app.Use(static.ServeRoot("/", "./site"))
+	}
 
 	app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
 
@@ -247,13 +263,18 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 		routes.UserRoutes(app)
 		routes.InstallRoutes(app)
 	}
+	if config.Config.OAS {
+		routes.OASRoutes(app)
+	}
+	if config.Config.ServiceMap {
+		routes.ServiceMapRoutes(app)
+	}
 
 	routes.QueryRoutes(app)
 	routes.EntriesRoutes(app)
 	routes.MetadataRoutes(app)
 	routes.StatusRoutes(app)
 	routes.NotFoundRoute(app)
-
 	utils.StartServer(app)
 }
 
@@ -268,13 +289,14 @@ func DisableRootStaticCache() gin.HandlerFunc {
 	}
 }
 
-func setUIMode() error {
+func setUIFlags() error {
 	read, err := ioutil.ReadFile(uiIndexPath)
 	if err != nil {
 		return err
 	}
 
-	replacedContent := strings.Replace(string(read), "__IS_STANDALONE__", strconv.FormatBool(config.Config.StandaloneMode), 1)
+	replacedContent := strings.Replace(string(read), "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS), 1)
+	replacedContent = strings.Replace(replacedContent, "__IS_SERVICE_MAP_ENABLED__", strconv.FormatBool(config.Config.ServiceMap), 1)
 
 	err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
 	if err != nil {

agent/pkg/api/main.go

@@ -13,12 +13,15 @@ import (
 	"strings"
 	"time"
 
+	"mizuserver/pkg/servicemap"
+
 	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	"mizuserver/pkg/models"
+	"mizuserver/pkg/oas"
 	"mizuserver/pkg/resolver"
 	"mizuserver/pkg/utils"
 
@@ -119,15 +122,12 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
 		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
-		baseEntry := extension.Dissector.Summarize(mizuEntry)
-		mizuEntry.Base = baseEntry
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
 				var httpPair tapApi.HTTPRequestResponsePair
 				json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair)
 
 				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
-				baseEntry.ContractStatus = contract.Status
 				mizuEntry.ContractStatus = contract.Status
 				mizuEntry.ContractRequestReason = contract.RequestReason
 				mizuEntry.ContractResponseReason = contract.ResponseReason
@@ -137,8 +137,10 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 			harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
 			if err == nil {
 				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
-				baseEntry.Rules = rules
+				mizuEntry.Rules = rules
 			}
+
+			oas.GetOasGeneratorInstance().PushEntry(harEntry)
 		}
 
 		data, err := json.Marshal(mizuEntry)
@@ -146,6 +148,8 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 			panic(err)
 		}
 		connection.SendText(string(data))
+
+		servicemap.GetInstance().NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
 	}
 }
 

agent/pkg/api/socket_routes.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"mizuserver/pkg/middlewares"
 	"mizuserver/pkg/models"
 	"net/http"
 	"sync"
@@ -16,6 +15,7 @@ import (
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 type EventHandlers interface {
@@ -48,7 +48,7 @@ func init() {
 func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
 	app.GET("/ws", func(c *gin.Context) {
 		websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime)
-	}, middlewares.RequiresAuth())
+	})
 
 	app.GET("/wsTapper", func(c *gin.Context) { // TODO: add m2m authentication to this route
 		websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime)
@@ -131,18 +131,10 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 						return
 					}
 
-					var dataMap map[string]interface{}
-					err = json.Unmarshal(bytes, &dataMap)
+					var entry *tapApi.Entry
+					err = json.Unmarshal(bytes, &entry)
 
-					var base map[string]interface{}
-					switch dataMap["base"].(type) {
-					case map[string]interface{}:
-						base = dataMap["base"].(map[string]interface{})
-						base["id"] = uint(dataMap["id"].(float64))
-					default:
-						logger.Log.Debugf("Base field has an unrecognized type: %+v", dataMap)
-						continue
-					}
+					base := tapApi.Summarize(entry)
 
 					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
 					SendToSocket(socketId, baseEntryBytes)

agent/pkg/api/socket_server_handlers.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tappers"
 	"mizuserver/pkg/up9"
 	"sync"
 
@@ -29,7 +30,7 @@ func init() {
 func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
-		providers.TapperAdded()
+		tappers.Connected()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
 		socketListLock.Lock()
@@ -41,7 +42,7 @@ func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
-		providers.TapperRemoved()
+		tappers.Disconnected()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
 		socketListLock.Lock()

agent/pkg/controllers/config_controller.go

@@ -11,18 +11,20 @@ import (
 	"mizuserver/pkg/config"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tapConfig"
+	"mizuserver/pkg/providers/tappedPods"
+	"mizuserver/pkg/providers/tappers"
 	"net/http"
 	"regexp"
 	"time"
 )
 
-var globalTapConfig = &models.TapConfig{TappedNamespaces: make(map[string]bool)}
 var cancelTapperSyncer context.CancelFunc
 
 func PostTapConfig(c *gin.Context) {
-	tapConfig := &models.TapConfig{}
+	requestTapConfig := &models.TapConfig{}
 
-	if err := c.Bind(tapConfig); err != nil {
+	if err := c.Bind(requestTapConfig); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
@@ -30,14 +32,14 @@ func PostTapConfig(c *gin.Context) {
 	if cancelTapperSyncer != nil {
 		cancelTapperSyncer()
 
-		providers.TapStatus = shared.TapStatus{}
-		providers.TappersStatus = make(map[string]shared.TapperStatus)
+		tappedPods.Set([]*shared.PodInfo{})
+		tappers.ResetStatus()
 
 		broadcastTappedPodsStatus()
 	}
 
 	var tappedNamespaces []string
-	for namespace, tapped := range tapConfig.TappedNamespaces {
+	for namespace, tapped := range requestTapConfig.TappedNamespaces {
 		if tapped {
 			tappedNamespaces = append(tappedNamespaces, namespace)
 		}
@@ -60,7 +62,7 @@ func PostTapConfig(c *gin.Context) {
 	}
 
 	cancelTapperSyncer = cancel
-	globalTapConfig = tapConfig
+	tapConfig.Save(requestTapConfig)
 
 	c.JSON(http.StatusOK, "OK")
 }
@@ -81,17 +83,19 @@ func GetTapConfig(c *gin.Context) {
 		return
 	}
 
+	savedTapConfig := tapConfig.Get()
+
 	tappedNamespaces := make(map[string]bool)
 	for _, namespace := range namespaces {
 		if namespace.Name == config.Config.MizuResourcesNamespace {
 			continue
 		}
 
-		tappedNamespaces[namespace.Name] = globalTapConfig.TappedNamespaces[namespace.Name]
+		tappedNamespaces[namespace.Name] = savedTapConfig.TappedNamespaces[namespace.Name]
 	}
 
-	tapConfig := models.TapConfig{TappedNamespaces: tappedNamespaces}
-	c.JSON(http.StatusOK, tapConfig)
+	tapConfigToReturn := models.TapConfig{TappedNamespaces: tappedNamespaces}
+	c.JSON(http.StatusOK, tapConfigToReturn)
 }
 
 func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, serviceMesh bool) (*kubernetes.MizuTapperSyncer, error) {
@@ -129,7 +133,7 @@ func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, t
 					return
 				}
 
-				providers.TapStatus = shared.TapStatus{Pods: kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods)}
+				tappedPods.Set(kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods))
 				broadcastTappedPodsStatus()
 			case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut:
 				if !ok {
@@ -137,7 +141,7 @@ func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, t
 					return
 				}
 
-				addTapperStatus(tapperStatus)
+				tappers.SetStatus(&tapperStatus)
 				broadcastTappedPodsStatus()
 			case <-ctx.Done():
 				logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")

agent/pkg/controllers/entries_controller.go

@@ -64,8 +64,8 @@ func GetEntries(c *gin.Context) {
 	var dataSlice []interface{}
 
 	for _, row := range data {
-		var dataMap map[string]interface{}
-		err = json.Unmarshal(row, &dataMap)
+		var entry *tapApi.Entry
+		err = json.Unmarshal(row, &entry)
 		if err != nil {
 			c.JSON(http.StatusBadRequest, gin.H{
 				"error":     true,
@@ -76,8 +76,7 @@ func GetEntries(c *gin.Context) {
 			return // exit
 		}
 
-		base := dataMap["base"].(map[string]interface{})
-		base["id"] = uint(dataMap["id"].(float64))
+		base := tapApi.Summarize(entry)
 
 		dataSlice = append(dataSlice, base)
 	}
@@ -95,9 +94,19 @@ func GetEntries(c *gin.Context) {
 }
 
 func GetEntry(c *gin.Context) {
+	singleEntryRequest := &models.SingleEntryRequest{}
+
+	if err := c.BindQuery(singleEntryRequest); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+	validationError := validation.Validate(singleEntryRequest)
+	if validationError != nil {
+		c.JSON(http.StatusBadRequest, validationError)
+	}
+
 	id, _ := strconv.Atoi(c.Param("id"))
-	var entry tapApi.MizuEntry
-	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id)
+	var entry *tapApi.Entry
+	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id, singleEntryRequest.Query)
 	if Error(c, err) {
 		return // exit
 	}
@@ -125,7 +134,7 @@ func GetEntry(c *gin.Context) {
 		json.Unmarshal(inrec, &rules)
 	}
 
-	c.JSON(http.StatusOK, tapApi.MizuEntryWrapper{
+	c.JSON(http.StatusOK, tapApi.EntryWrapper{
 		Protocol:       entry.Protocol,
 		Representation: string(representation),
 		BodySize:       bodySize,

agent/pkg/controllers/install_controller.go

@@ -16,3 +16,8 @@ func IsSetupNecessary(c *gin.Context) {
 		c.JSON(http.StatusOK, IsInstallNeeded)
 	}
 }
+
+func SetupAdminUser(c *gin.Context) {
+	token, err, formErrorMessages := providers.CreateAdminUser(c.PostForm("password"), c.Request.Context())
+	handleRegistration(token, err, formErrorMessages, c)
+}

agent/pkg/controllers/oas_controller.go

@@ -0,0 +1,62 @@
+package controllers
+
+import (
+	"github.com/chanced/openapi"
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/oas"
+	"net/http"
+)
+
+func GetOASServers(c *gin.Context) {
+	m := make([]string, 0)
+	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
+		m = append(m, key.(string))
+		return true
+	})
+
+	c.JSON(http.StatusOK, m)
+}
+
+func GetOASSpec(c *gin.Context) {
+	res, ok := oas.GetOasGeneratorInstance().ServiceSpecs.Load(c.Param("id"))
+	if !ok {
+		c.JSON(http.StatusNotFound, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       "Service not found among specs",
+		})
+		return // exit
+	}
+
+	gen := res.(*oas.SpecGen)
+	spec, err := gen.GetSpec()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       err,
+		})
+		return // exit
+	}
+
+	c.JSON(http.StatusOK, spec)
+}
+
+func GetOASAllSpecs(c *gin.Context) {
+	res := map[string]*openapi.OpenAPI{}
+	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
+		svc := key.(string)
+		gen := value.(*oas.SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			logger.Log.Warningf("Failed to obtain spec for service %s: %s", svc, err)
+			return true
+		}
+		res[svc] = spec
+		return true
+	})
+	c.JSON(http.StatusOK, res)
+}

agent/pkg/controllers/oas_controller_test.go

@@ -0,0 +1,43 @@
+package controllers
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/oas"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestGetOASServers(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+
+	GetOASServers(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+	return
+}
+
+func TestGetOASAllSpecs(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+
+	GetOASAllSpecs(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+	return
+}
+
+func TestGetOASSpec(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+
+	c.Params = []gin.Param{{Key: "id", Value: "some"}}
+
+	GetOASSpec(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+	return
+}

agent/pkg/controllers/service_map_controller.go

@@ -0,0 +1,36 @@
+package controllers
+
+import (
+	"mizuserver/pkg/servicemap"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+)
+
+type ServiceMapController struct {
+	service servicemap.ServiceMap
+}
+
+func NewServiceMapController() *ServiceMapController {
+	return &ServiceMapController{
+		service: servicemap.GetInstance(),
+	}
+}
+
+func (s *ServiceMapController) Status(c *gin.Context) {
+	c.JSON(http.StatusOK, s.service.GetStatus())
+}
+
+func (s *ServiceMapController) Get(c *gin.Context) {
+	response := &servicemap.ServiceMapResponse{
+		Status: s.service.GetStatus(),
+		Nodes:  s.service.GetNodes(),
+		Edges:  s.service.GetEdges(),
+	}
+	c.JSON(http.StatusOK, response)
+}
+
+func (s *ServiceMapController) Reset(c *gin.Context) {
+	s.service.Reset()
+	s.Status(c)
+}

agent/pkg/controllers/service_map_controller_test.go

@@ -0,0 +1,147 @@
+package controllers
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"mizuserver/pkg/servicemap"
+
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/suite"
+	"github.com/up9inc/mizu/shared"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+const (
+	a    = "aService"
+	b    = "bService"
+	Ip   = "127.0.0.1"
+	Port = "80"
+)
+
+var (
+	TCPEntryA = &tapApi.TCP{
+		Name: a,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, a),
+	}
+	TCPEntryB = &tapApi.TCP{
+		Name: b,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, b),
+	}
+)
+
+var ProtocolHttp = &tapApi.Protocol{
+	Name:            "http",
+	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
+	Abbreviation:    "HTTP",
+	Macro:           "http",
+	Version:         "1.1",
+	BackgroundColor: "#205cf5",
+	ForegroundColor: "#ffffff",
+	FontSize:        12,
+	ReferenceLink:   "https://datatracker.ietf.org/doc/html/rfc2616",
+	Ports:           []string{"80", "443", "8080"},
+	Priority:        0,
+}
+
+type ServiceMapControllerSuite struct {
+	suite.Suite
+
+	c *ServiceMapController
+	w *httptest.ResponseRecorder
+	g *gin.Context
+}
+
+func (s *ServiceMapControllerSuite) SetupTest() {
+	s.c = NewServiceMapController()
+	s.c.service.SetConfig(&shared.MizuAgentConfig{
+		ServiceMap: true,
+	})
+	s.c.service.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
+
+	s.w = httptest.NewRecorder()
+	s.g, _ = gin.CreateTestContext(s.w)
+}
+
+func (s *ServiceMapControllerSuite) TestGetStatus() {
+	assert := s.Assert()
+
+	s.c.Status(s.g)
+	assert.Equal(http.StatusOK, s.w.Code)
+
+	var status servicemap.ServiceMapStatus
+	err := json.Unmarshal(s.w.Body.Bytes(), &status)
+	assert.NoError(err)
+	assert.Equal("enabled", status.Status)
+	assert.Equal(1, status.EntriesProcessedCount)
+	assert.Equal(2, status.NodeCount)
+	assert.Equal(1, status.EdgeCount)
+}
+
+func (s *ServiceMapControllerSuite) TestGet() {
+	assert := s.Assert()
+
+	s.c.Get(s.g)
+	assert.Equal(http.StatusOK, s.w.Code)
+
+	var response servicemap.ServiceMapResponse
+	err := json.Unmarshal(s.w.Body.Bytes(), &response)
+	assert.NoError(err)
+
+	// response status
+	assert.Equal("enabled", response.Status.Status)
+	assert.Equal(1, response.Status.EntriesProcessedCount)
+	assert.Equal(2, response.Status.NodeCount)
+	assert.Equal(1, response.Status.EdgeCount)
+
+	// response nodes
+	aNode := servicemap.ServiceMapNode{
+		Id:    1,
+		Name:  TCPEntryA.IP,
+		Entry: TCPEntryA,
+		Count: 1,
+	}
+	bNode := servicemap.ServiceMapNode{
+		Id:    2,
+		Name:  TCPEntryB.IP,
+		Entry: TCPEntryB,
+		Count: 1,
+	}
+	assert.Contains(response.Nodes, aNode)
+	assert.Contains(response.Nodes, bNode)
+	assert.Len(response.Nodes, 2)
+
+	// response edges
+	assert.Equal([]servicemap.ServiceMapEdge{
+		{
+			Source:      aNode,
+			Destination: bNode,
+			Protocol:    ProtocolHttp,
+			Count:       1,
+		},
+	}, response.Edges)
+}
+
+func (s *ServiceMapControllerSuite) TestGetReset() {
+	assert := s.Assert()
+
+	s.c.Reset(s.g)
+	assert.Equal(http.StatusOK, s.w.Code)
+
+	var status servicemap.ServiceMapStatus
+	err := json.Unmarshal(s.w.Body.Bytes(), &status)
+	assert.NoError(err)
+	assert.Equal("enabled", status.Status)
+	assert.Equal(0, status.EntriesProcessedCount)
+	assert.Equal(0, status.NodeCount)
+	assert.Equal(0, status.EdgeCount)
+}
+
+func TestServiceMapControllerSuite(t *testing.T) {
+	suite.Run(t, new(ServiceMapControllerSuite))
+}

agent/pkg/controllers/status_controller.go

@@ -8,43 +8,41 @@ import (
 	"mizuserver/pkg/api"
 	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tappedPods"
+	"mizuserver/pkg/providers/tappers"
 	"mizuserver/pkg/up9"
-	"mizuserver/pkg/utils"
 	"mizuserver/pkg/validation"
 	"net/http"
 )
 
 func HealthCheck(c *gin.Context) {
-	tappers := make([]shared.TapperStatus, 0)
-	for _, value := range providers.TappersStatus {
-		tappers = append(tappers, value)
+	tappersStatus := make([]*shared.TapperStatus, 0)
+	for _, value := range tappers.GetStatus() {
+		tappersStatus = append(tappersStatus, value)
 	}
 
 	response := shared.HealthResponse{
-		TapStatus:     providers.TapStatus,
-		TappersCount:  providers.TappersCount,
-		TappersStatus: tappers,
+		TappedPods:            tappedPods.Get(),
+		ConnectedTappersCount: tappers.GetConnectedCount(),
+		TappersStatus:         tappersStatus,
 	}
 	c.JSON(http.StatusOK, response)
 }
 
 func PostTappedPods(c *gin.Context) {
-	tapStatus := &shared.TapStatus{}
-	if err := c.Bind(tapStatus); err != nil {
+	var requestTappedPods []*shared.PodInfo
+	if err := c.Bind(&requestTappedPods); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
-	if err := validation.Validate(tapStatus); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-		return
-	}
-	logger.Log.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods))
-	providers.TapStatus.Pods = tapStatus.Pods
+
+	logger.Log.Infof("[Status] POST request: %d tapped pods", len(requestTappedPods))
+	tappedPods.Set(requestTappedPods)
 	broadcastTappedPodsStatus()
 }
 
 func broadcastTappedPodsStatus() {
-	tappedPodsStatus := utils.GetTappedPodsStatus()
+	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
 
 	message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
 	if jsonBytes, err := json.Marshal(message); err != nil {
@@ -54,14 +52,6 @@ func broadcastTappedPodsStatus() {
 	}
 }
 
-func addTapperStatus(tapperStatus shared.TapperStatus) {
-	if providers.TappersStatus == nil {
-		providers.TappersStatus = make(map[string]shared.TapperStatus)
-	}
-
-	providers.TappersStatus[tapperStatus.NodeName] = tapperStatus
-}
-
 func PostTapperStatus(c *gin.Context) {
 	tapperStatus := &shared.TapperStatus{}
 	if err := c.Bind(tapperStatus); err != nil {
@@ -75,12 +65,12 @@ func PostTapperStatus(c *gin.Context) {
 	}
 
 	logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus)
-	addTapperStatus(*tapperStatus)
+	tappers.SetStatus(tapperStatus)
 	broadcastTappedPodsStatus()
 }
 
-func GetTappersCount(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.TappersCount)
+func GetConnectedTappersCount(c *gin.Context) {
+	c.JSON(http.StatusOK, tappers.GetConnectedCount())
 }
 
 func GetAuthStatus(c *gin.Context) {
@@ -94,7 +84,7 @@ func GetAuthStatus(c *gin.Context) {
 }
 
 func GetTappingStatus(c *gin.Context) {
-	tappedPodsStatus := utils.GetTappedPodsStatus()
+	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
 	c.JSON(http.StatusOK, tappedPodsStatus)
 }
 

agent/pkg/controllers/user_controller.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/up9inc/mizu/shared/logger"
+
+	ory "github.com/ory/kratos-client-go"
 )
 
 func Login(c *gin.Context) {
@@ -25,7 +27,12 @@ func Logout(c *gin.Context) {
 }
 
 func Register(c *gin.Context) {
-	if token, _, err, formErrorMessages := providers.RegisterUser(c.PostForm("username"), c.PostForm("password"), c.Request.Context()); err != nil {
+	token, _, err, formErrorMessages := providers.RegisterUser(c.PostForm("username"), c.PostForm("password"), c.Request.Context())
+	handleRegistration(token, err, formErrorMessages, c)
+}
+
+func handleRegistration(token *string, err error, formErrorMessages map[string][]ory.UiText, c *gin.Context) {
+	if err != nil {
 		if formErrorMessages != nil {
 			logger.Log.Infof("user attempted to register but had form errors %v %v", formErrorMessages, err)
 			c.AbortWithStatusJSON(400, formErrorMessages)
@@ -34,6 +41,6 @@ func Register(c *gin.Context) {
 			c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while registering"})
 		}
 	} else {
-		c.JSON(200, gin.H{"token": token})
+		c.JSON(201, gin.H{"token": token})
 	}
 }

agent/pkg/middlewares/auth.go

@@ -0,0 +1,73 @@
+package middlewares
+
+import (
+	"mizuserver/pkg/config"
+	"mizuserver/pkg/providers"
+
+	"github.com/gin-gonic/gin"
+	ory "github.com/ory/kratos-client-go"
+	"github.com/up9inc/mizu/shared/logger"
+)
+
+func RequiresAuth() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// auth is irrelevant for ephermeral mizu
+		if !config.Config.StandaloneMode {
+			c.Next()
+			return
+		}
+
+		verifyKratosSessionForRequest(c)
+		if !c.IsAborted() {
+			c.Next()
+		}
+	}
+}
+
+func RequiresAdmin() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// auth is irrelevant for ephermeral mizu
+		if !config.Config.StandaloneMode {
+			c.Next()
+			return
+		}
+
+		session := verifyKratosSessionForRequest(c)
+		if c.IsAborted() {
+			return
+		}
+
+		traits := session.Identity.Traits.(map[string]interface{})
+		username := traits["username"].(string)
+
+		isAdmin, err := providers.CheckIfUserHasSystemRole(username, providers.AdminRole)
+		if err != nil {
+			logger.Log.Errorf("error checking user role %v", err)
+			c.AbortWithStatusJSON(403, gin.H{"error": "unknown auth error occured"})
+		} else if !isAdmin {
+			logger.Log.Warningf("user %s attempted to call an admin only endpoint with insufficient privileges", username)
+			c.AbortWithStatusJSON(403, gin.H{"error": "unauthorized"})
+		} else {
+			c.Next()
+		}
+	}
+}
+
+func verifyKratosSessionForRequest(c *gin.Context) *ory.Session {
+	token := c.GetHeader("x-session-token")
+	if token == "" {
+		c.AbortWithStatusJSON(401, gin.H{"error": "token header is empty"})
+		return nil
+	}
+
+	if session, err := providers.VerifyToken(token, c.Request.Context()); err != nil {
+		logger.Log.Errorf("error verifying token %v", err)
+		c.AbortWithStatusJSON(401, gin.H{"error": "unknown auth error occured"})
+		return nil
+	} else if session == nil {
+		c.AbortWithStatusJSON(401, gin.H{"error": "invalid token"})
+		return nil
+	} else {
+		return session
+	}
+}

agent/pkg/middlewares/requiresAuth.go

@@ -1,49 +0,0 @@
-package middlewares
-
-import (
-	"mizuserver/pkg/config"
-	"mizuserver/pkg/providers"
-	"time"
-
-	"github.com/gin-gonic/gin"
-	"github.com/patrickmn/go-cache"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-const cachedValidTokensRetainmentTime = time.Minute * 1
-
-var cachedValidTokens = cache.New(cachedValidTokensRetainmentTime, cachedValidTokensRetainmentTime)
-
-func RequiresAuth() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		// auth is irrelevant for ephermeral mizu
-		if !config.Config.StandaloneMode {
-			c.Next()
-			return
-		}
-
-		token := c.GetHeader("x-session-token")
-		if token == "" {
-			c.AbortWithStatusJSON(401, gin.H{"error": "token header is empty"})
-			return
-		}
-
-		if _, isTokenCached := cachedValidTokens.Get(token); isTokenCached {
-			c.Next()
-			return
-		}
-
-		if isTokenValid, err := providers.VerifyToken(token, c.Request.Context()); err != nil {
-			logger.Log.Errorf("error verifying token %s", err)
-			c.AbortWithStatusJSON(401, gin.H{"error": "unknown auth error occured"})
-			return
-		} else if !isTokenValid {
-			c.AbortWithStatusJSON(401, gin.H{"error": "invalid token"})
-			return
-		}
-
-		cachedValidTokens.Set(token, true, cachedValidTokensRetainmentTime)
-
-		c.Next()
-	}
-}

agent/pkg/models/models.go

@@ -12,7 +12,7 @@ import (
 	"github.com/up9inc/mizu/tap"
 )
 
-func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error {
+func GetEntry(r *tapApi.Entry, v tapApi.DataUnmarshaler) error {
 	return v.UnmarshalData(r)
 }
 
@@ -28,6 +28,10 @@ type EntriesRequest struct {
 	TimeoutMs int    `form:"timeoutMs" validate:"min=1"`
 }
 
+type SingleEntryRequest struct {
+	Query string `form:"query"`
+}
+
 type EntriesResponse struct {
 	Data []interface{}      `json:"data"`
 	Meta *basenine.Metadata `json:"meta"`
@@ -35,7 +39,7 @@ type EntriesResponse struct {
 
 type WebSocketEntryMessage struct {
 	*shared.WebSocketMessageMetadata
-	Data map[string]interface{} `json:"data,omitempty"`
+	Data *tapApi.BaseEntry `json:"data,omitempty"`
 }
 
 type WebSocketTappedEntryMessage struct {
@@ -74,7 +78,7 @@ type WebSocketStartTimeMessage struct {
 	Data int64 `json:"data"`
 }
 
-func CreateBaseEntryWebSocketMessage(base map[string]interface{}) ([]byte, error) {
+func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) {
 	message := &WebSocketEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
 			MessageType: shared.WebSocketMessageTypeEntry,

agent/pkg/oas/feeder_test.go

@@ -0,0 +1,158 @@
+package oas
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func getFiles(baseDir string) (result []string, err error) {
+	result = make([]string, 0, 0)
+	logger.Log.Infof("Reading files from tree: %s", baseDir)
+
+	// https://yourbasic.org/golang/list-files-in-directory/
+	err = filepath.Walk(baseDir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			ext := strings.ToLower(filepath.Ext(path))
+			if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
+				result = append(result, path)
+			}
+
+			return nil
+		})
+
+	sort.SliceStable(result, func(i, j int) bool {
+		return fileSize(result[i]) < fileSize(result[j])
+	})
+
+	logger.Log.Infof("Got files: %d", len(result))
+	return result, err
+}
+
+func fileSize(fname string) int64 {
+	fi, err := os.Stat(fname)
+	if err != nil {
+		panic(err)
+	}
+
+	return fi.Size()
+}
+
+func feedEntries(fromFiles []string) (err error) {
+	for _, file := range fromFiles {
+		logger.Log.Info("Processing file: " + file)
+		ext := strings.ToLower(filepath.Ext(file))
+		switch ext {
+		case ".har":
+			err = feedFromHAR(file)
+			if err != nil {
+				logger.Log.Warning("Failed processing file: " + err.Error())
+				continue
+			}
+		case ".ldjson":
+			err = feedFromLDJSON(file)
+			if err != nil {
+				logger.Log.Warning("Failed processing file: " + err.Error())
+				continue
+			}
+		default:
+			return errors.New("Unsupported file extension: " + ext)
+		}
+	}
+
+	return nil
+}
+
+func feedFromHAR(file string) error {
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		return err
+	}
+
+	var harDoc har.HAR
+	err = json.Unmarshal(data, &harDoc)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range harDoc.Log.Entries {
+		GetOasGeneratorInstance().PushEntry(entry)
+	}
+
+	return nil
+}
+
+func feedFromLDJSON(file string) error {
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	reader := bufio.NewReader(fd)
+
+	var meta map[string]interface{}
+
+	buf := strings.Builder{}
+	for {
+		substr, isPrefix, err := reader.ReadLine()
+		if err == io.EOF {
+			break
+		}
+
+		buf.WriteString(string(substr))
+		if isPrefix {
+			continue
+		}
+
+		line := buf.String()
+		buf.Reset()
+
+		if meta == nil {
+			err := json.Unmarshal([]byte(line), &meta)
+			if err != nil {
+				return err
+			}
+		} else {
+			var entry har.Entry
+			err := json.Unmarshal([]byte(line), &entry)
+			if err != nil {
+				logger.Log.Warningf("Failed decoding entry: %s", line)
+			}
+			GetOasGeneratorInstance().PushEntry(&entry)
+		}
+	}
+
+	return nil
+}
+
+func TestFilesList(t *testing.T) {
+	res, err := getFiles("./test_artifacts/")
+	t.Log(len(res))
+	t.Log(res)
+	if err != nil || len(res) != 2 {
+		t.Logf("Should return 2 files but returned %d", len(res))
+		t.FailNow()
+	}
+}

agent/pkg/oas/gibberish.go

@@ -0,0 +1,108 @@
+package oas
+
+import (
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+var (
+	patBase64   = regexp.MustCompile(`^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$`)
+	patUuid4    = regexp.MustCompile(`(?i)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
+	patEmail    = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
+	patHexLower = regexp.MustCompile(`(0x)?[0-9a-f]{6,}`)
+	patHexUpper = regexp.MustCompile(`(0x)?[0-9A-F]{6,}`)
+)
+
+func IsGibberish(str string) bool {
+	if patBase64.MatchString(str) && len(str) > 32 {
+		return true
+	}
+
+	if patUuid4.MatchString(str) {
+		return true
+	}
+
+	if patEmail.MatchString(str) {
+		return true
+	}
+
+	if patHexLower.MatchString(str) || patHexUpper.MatchString(str) {
+		return true
+	}
+
+	noise := noiseLevel(str)
+	if noise >= 0.2 {
+		return true
+	}
+
+	return false
+}
+
+func noiseLevel(str string) (score float64) {
+	// opinionated algo of certain char pairs marking the non-human strings
+	prev := *new(rune)
+	cnt := 0.0
+	for _, char := range str {
+		cnt += 1
+		if prev > 0 {
+			switch {
+			// continued class of upper/lower/digit adds no noise
+			case unicode.IsUpper(prev) && unicode.IsUpper(char):
+			case unicode.IsLower(prev) && unicode.IsLower(char):
+			case unicode.IsDigit(prev) && unicode.IsDigit(char):
+
+			// upper =>
+			case unicode.IsUpper(prev) && unicode.IsLower(char):
+				score += 0.25
+			case unicode.IsUpper(prev) && unicode.IsDigit(char):
+				score += 0.25
+
+			// lower =>
+			case unicode.IsLower(prev) && unicode.IsUpper(char):
+				score += 0.75
+			case unicode.IsLower(prev) && unicode.IsDigit(char):
+				score += 0.25
+
+			// digit =>
+			case unicode.IsDigit(prev) && unicode.IsUpper(char):
+				score += 0.75
+			case unicode.IsDigit(prev) && unicode.IsLower(char):
+				score += 0.75
+
+			// the rest is 100% noise
+			default:
+				score += 1
+			}
+		}
+		prev = char
+	}
+
+	score /= cnt // weigh it
+
+	return score
+}
+
+func IsVersionString(component string) bool {
+	if component == "" {
+		return false
+	}
+
+	hasV := false
+	if strings.HasPrefix(component, "v") {
+		component = component[1:]
+		hasV = true
+	}
+
+	for _, c := range component {
+		if string(c) != "." && !unicode.IsDigit(c) {
+			return false
+		}
+	}
+
+	if !hasV && strings.Contains(component, ".") {
+		return false
+	}
+
+	return true
+}

agent/pkg/oas/gibberish_test.go

@@ -0,0 +1,71 @@
+package oas
+
+import "testing"
+
+func TestNegative(t *testing.T) {
+	cases := []string{
+		"",
+		"b", // can be valid hexadecimal
+		"GetUniversalVariableUser",
+		"callback",
+		"runs",
+		"tcfv2",
+		"StartUpCheckout",
+		"GetCart",
+	}
+
+	for _, str := range cases {
+		if IsGibberish(str) {
+			t.Errorf("Mistakenly true: %s", str)
+		}
+	}
+}
+
+func TestPositive(t *testing.T) {
+	cases := []string{
+		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
+		"952bea17-3776-11ea-9341-42010a84012a",
+		"456795af-b48f-4a8d-9b37-3e932622c2f0",
+		"0a0d0174-b338-4520-a1c3-24f7e3d5ec50.html",
+		"6120c057c7a97b03f6986f1b",
+		"610bc3fd5a77a7fa25033fb0",
+		"610bd0315a77a7fa25034368",
+		"610bd0315a77a7fa25034368zh",
+		"710a462e",
+		"1554507871",
+		"qwerqwerasdfqwer@protonmai.com",
+		"john.dow.1981@protonmail.com",
+		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
+		"11ca096cbc224a67360493d44a9903",
+		"c738338322370b47a79251f7510dd", // prefixed hex
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
+		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
+		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
+		"730970532670-compute@developer.gserviceaccount.com",
+		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
+		"AAAA028295945",
+		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
+		"n63nd45qsj",
+		"n9z9QGNiz",
+		"proxy.3d2100fd7107262ecb55ce6847f01fa5.html",
+		"r-ext-5579e00a95c90",
+		"r-ext-5579e8b12f11e",
+		"r-v4-5c92513c9e04a",
+		"r-v4-5c92513c9e04a-0-c.monitoring.json",
+		"segments-1563566437171.639994",
+		"t_52d94268-8810-4a7e-ba87-ffd657a6752f",
+		"timeouts-1563566437171.639994",
+
+		// TODO
+		// "fb6cjraf9cejut2a",
+		// "Fxvd1timk", // questionable
+		// "JEHJW4BKVFDRTMTUQLHKK5WVAU",
+	}
+
+	for _, str := range cases {
+		if !IsGibberish(str) {
+			t.Errorf("Mistakenly false: %s", str)
+		}
+	}
+}

agent/pkg/oas/ignores.go

@@ -0,0 +1,75 @@
+package oas
+
+import "strings"
+
+var ignoredExtensions = []string{"gif", "svg", "css", "png", "ico", "js", "woff2", "woff", "jpg", "jpeg", "swf", "ttf", "map", "webp", "otf", "mp3"}
+
+var ignoredCtypePrefixes = []string{"image/", "font/", "video/", "audio/", "text/javascript"}
+var ignoredCtypes = []string{"application/javascript", "application/x-javascript", "text/css", "application/font-woff2", "application/font-woff", "application/x-font-woff"}
+
+var ignoredHeaders = []string{
+	"a-im", "accept",
+	"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-type", "cookie",
+	"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
+	"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
+	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests",
+	"server", "user-agent", "via", "warning", "strict-transport-security",
+	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data",
+	"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
+	"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
+	"x-wap-profile", "x-scheme",
+	"newrelic", "x-cloud-trace-context", "sentry-trace",
+	"expires", "set-cookie", "p3p", "location", "content-security-policy", "content-security-policy-report-only",
+	"last-modified", "content-language",
+	"keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by",
+	"age", "allow", "www-authenticate",
+}
+
+var ignoredHeaderPrefixes = []string{
+	":", "accept-", "access-control-", "if-", "sec-", "grpc-",
+	"x-forwarded-", "x-original-",
+	"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
+	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-",
+}
+
+func isCtypeIgnored(ctype string) bool {
+	for _, prefix := range ignoredCtypePrefixes {
+		if strings.HasPrefix(ctype, prefix) {
+			return true
+		}
+	}
+
+	for _, toIgnore := range ignoredCtypes {
+		if ctype == toIgnore {
+			return true
+		}
+	}
+	return false
+}
+
+func isExtIgnored(path string) bool {
+	for _, extIgn := range ignoredExtensions {
+		if strings.HasSuffix(path, "."+extIgn) {
+			return true
+		}
+	}
+	return false
+}
+
+func isHeaderIgnored(name string) bool {
+	name = strings.ToLower(name)
+
+	for _, ignore := range ignoredHeaders {
+		if name == ignore {
+			return true
+		}
+	}
+
+	for _, prefix := range ignoredHeaderPrefixes {
+		if strings.HasPrefix(name, prefix) {
+			return true
+		}
+	}
+
+	return false
+}

agent/pkg/oas/oas_generator.go

@@ -0,0 +1,107 @@
+package oas
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"net/url"
+	"sync"
+)
+
+var (
+	syncOnce sync.Once
+	instance *oasGenerator
+)
+
+func GetOasGeneratorInstance() *oasGenerator {
+	syncOnce.Do(func() {
+		instance = newOasGenerator()
+		logger.Log.Debug("Oas Generator Initialized")
+	})
+	return instance
+}
+
+func (g *oasGenerator) Start() {
+	if g.started {
+		return
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	g.cancel = cancel
+	g.ctx = ctx
+	g.entriesChan = make(chan har.Entry, 100) // buffer up to 100 entries for OAS processing
+	g.ServiceSpecs = &sync.Map{}
+	g.started = true
+	go instance.runGeneretor()
+}
+
+func (g *oasGenerator) runGeneretor() {
+	for {
+		select {
+		case <-g.ctx.Done():
+			logger.Log.Infof("OAS Generator was canceled")
+			return
+
+		case entry, ok := <-g.entriesChan:
+			if !ok {
+				logger.Log.Infof("OAS Generator - entries channel closed")
+				break
+			}
+			u, err := url.Parse(entry.Request.URL)
+			if err != nil {
+				logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
+			}
+
+			val, found := g.ServiceSpecs.Load(u.Host)
+			var gen *SpecGen
+			if !found {
+				gen = NewGen(u.Scheme + "://" + u.Host)
+				g.ServiceSpecs.Store(u.Host, gen)
+			} else {
+				gen = val.(*SpecGen)
+			}
+
+			opId, err := gen.feedEntry(entry)
+			if err != nil {
+				txt, suberr := json.Marshal(entry)
+				if suberr == nil {
+					logger.Log.Debugf("Problematic entry: %s", txt)
+				}
+
+				logger.Log.Warningf("Failed processing entry: %s", err)
+				continue
+			}
+
+			logger.Log.Debugf("Handled entry %s as opId: %s", entry.Request.URL, opId) // TODO: set opId back to entry?
+		}
+	}
+}
+
+func (g *oasGenerator) PushEntry(entry *har.Entry) {
+	if !g.started {
+		return
+	}
+	select {
+	case g.entriesChan <- *entry:
+	default:
+		logger.Log.Warningf("OAS Generator - entry wasn't sent to channel because the channel has no buffer or there is no receiver")
+	}
+}
+
+func newOasGenerator() *oasGenerator {
+	return &oasGenerator{
+		started:      false,
+		ctx:          nil,
+		cancel:       nil,
+		ServiceSpecs: nil,
+		entriesChan:  nil,
+	}
+}
+
+type oasGenerator struct {
+	started      bool
+	ctx          context.Context
+	cancel       context.CancelFunc
+	ServiceSpecs *sync.Map
+	entriesChan  chan har.Entry
+}

agent/pkg/oas/specgen.go

@@ -0,0 +1,497 @@
+package oas
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/google/uuid"
+	"github.com/up9inc/mizu/shared/logger"
+	"mime"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+type reqResp struct { // hello, generics in Go
+	Req  *har.Request
+	Resp *har.Response
+}
+
+type SpecGen struct {
+	oas  *openapi.OpenAPI
+	tree *Node
+	lock sync.Mutex
+}
+
+func NewGen(server string) *SpecGen {
+	spec := new(openapi.OpenAPI)
+	spec.Version = "3.1.0"
+
+	info := openapi.Info{Title: server}
+	info.Version = "0.0"
+	spec.Info = &info
+	spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
+
+	spec.Servers = make([]*openapi.Server, 0)
+	spec.Servers = append(spec.Servers, &openapi.Server{URL: server})
+
+	gen := SpecGen{oas: spec, tree: new(Node)}
+	return &gen
+}
+
+func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
+	g.oas = oas
+	for pathStr, pathObj := range oas.Paths.Items {
+		pathSplit := strings.Split(string(pathStr), "/")
+		g.tree.getOrSet(pathSplit, pathObj)
+	}
+}
+
+func (g *SpecGen) feedEntry(entry har.Entry) (string, error) {
+	g.lock.Lock()
+	defer g.lock.Unlock()
+
+	opId, err := g.handlePathObj(&entry)
+	if err != nil {
+		return "", err
+	}
+
+	// NOTE: opId can be empty for some failed entries
+	return opId, err
+}
+
+func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
+	g.lock.Lock()
+	defer g.lock.Unlock()
+
+	g.tree.compact()
+
+	for _, pathop := range g.tree.listOps() {
+		if pathop.op.Summary == "" {
+			pathop.op.Summary = pathop.path
+		}
+	}
+
+	// put paths back from tree into OAS
+	g.oas.Paths = g.tree.listPaths()
+
+	suggestTags(g.oas)
+
+	// to make a deep copy, no better idea than marshal+unmarshal
+	specText, err := json.MarshalIndent(g.oas, "", "\t")
+	if err != nil {
+		return nil, err
+	}
+
+	spec := new(openapi.OpenAPI)
+	err = json.Unmarshal(specText, spec)
+	if err != nil {
+		return nil, err
+	}
+
+	return spec, err
+}
+
+func suggestTags(oas *openapi.OpenAPI) {
+	paths := getPathsKeys(oas.Paths.Items)
+	for len(paths) > 0 {
+		group := make([]string, 0)
+		group = append(group, paths[0])
+		paths = paths[1:]
+
+		pathsClone := append(paths[:0:0], paths...)
+		for _, path := range pathsClone {
+			if getSimilarPrefix([]string{group[0], path}) != "" {
+				group = append(group, path)
+				paths = deleteFromSlice(paths, path)
+			}
+		}
+
+		common := getSimilarPrefix(group)
+
+		if len(group) > 1 {
+			for _, path := range group {
+				pathObj := oas.Paths.Items[openapi.PathValue(path)]
+				for _, op := range getOps(pathObj) {
+					if op.Tags == nil {
+						op.Tags = make([]string, 0)
+					}
+					// only add tags if not present
+					if len(op.Tags) == 0 {
+						op.Tags = append(op.Tags, common)
+					}
+				}
+			}
+		}
+
+		//groups[common] = group
+	}
+}
+
+func getSimilarPrefix(strs []string) string {
+	chunked := make([][]string, 0)
+	for _, item := range strs {
+		chunked = append(chunked, strings.Split(item, "/"))
+	}
+
+	cmn := longestCommonXfix(chunked, true)
+	res := make([]string, 0)
+	for _, chunk := range cmn {
+		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
+			res = append(res, chunk)
+		}
+	}
+	return strings.Join(res[1:], ".")
+}
+
+func deleteFromSlice(s []string, val string) []string {
+	temp := s[:0]
+	for _, x := range s {
+		if x != val {
+			temp = append(temp, x)
+		}
+	}
+	return temp
+}
+
+func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
+	keys := make([]string, len(mymap))
+
+	i := 0
+	for k := range mymap {
+		keys[i] = string(k)
+		i++
+	}
+	return keys
+}
+
+func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
+	urlParsed, err := url.Parse(entry.Request.URL)
+	if err != nil {
+		return "", err
+	}
+
+	if isExtIgnored(urlParsed.Path) {
+		logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
+	}
+
+	ctype := getRespCtype(entry.Response)
+	if isCtypeIgnored(ctype) {
+		logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
+	}
+
+	if entry.Response.Status < 100 {
+		logger.Log.Debugf("Dropped traffic entry due to status<100: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	if entry.Response.Status == 301 || entry.Response.Status == 308 {
+		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	split := strings.Split(urlParsed.Path, "/")
+	node := g.tree.getOrSet(split, new(openapi.PathObj))
+	opObj, err := handleOpObj(entry, node.ops)
+
+	if opObj != nil {
+		return opObj.OperationID, err
+	}
+
+	return "", err
+}
+
+func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation, error) {
+	isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
+	opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
+	if err != nil {
+		return nil, err
+	}
+
+	if !isSuccess && wasMissing {
+		logger.Log.Debugf("Dropped traffic entry due to failed status and no known endpoint at: %s", entry.StartedDateTime)
+		return nil, nil
+	}
+
+	err = handleRequest(entry.Request, opObj, isSuccess)
+	if err != nil {
+		return nil, err
+	}
+
+	err = handleResponse(entry.Response, opObj, isSuccess)
+	if err != nil {
+		return nil, err
+	}
+
+	return opObj, nil
+}
+
+func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error {
+	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
+
+	qstrGW := nvParams{
+		In: openapi.InQuery,
+		Pairs: func() []NVPair {
+			return qstrToNVP(req.QueryString)
+		},
+		IsIgnored:      func(name string) bool { return false },
+		GeneralizeName: func(name string) string { return name },
+	}
+	handleNameVals(qstrGW, &opObj.Parameters)
+
+	hdrGW := nvParams{
+		In: openapi.InHeader,
+		Pairs: func() []NVPair {
+			return hdrToNVP(req.Headers)
+		},
+		IsIgnored:      isHeaderIgnored,
+		GeneralizeName: strings.ToLower,
+	}
+	handleNameVals(hdrGW, &opObj.Parameters)
+
+	if req.PostData != nil && req.PostData.Text != "" && isSuccess {
+		reqBody, err := getRequestBody(req, opObj, isSuccess)
+		if err != nil {
+			return err
+		}
+
+		if reqBody != nil {
+			reqCtype := getReqCtype(req)
+			reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, err)
+			if err != nil {
+				return err
+			}
+
+			_ = reqMedia
+		}
+	}
+	return nil
+}
+
+func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool) error {
+	// TODO: we don't support "default" response
+	respObj, err := getResponseObj(resp, opObj, isSuccess)
+	if err != nil {
+		return err
+	}
+
+	handleRespHeaders(resp.Headers, respObj)
+
+	respCtype := getRespCtype(resp)
+	respContent := respObj.Content
+	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, err)
+	if err != nil {
+		return err
+	}
+	_ = respMedia
+	return nil
+}
+
+func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj) {
+	visited := map[string]*openapi.HeaderObj{}
+	for _, pair := range reqHeaders {
+		if isHeaderIgnored(pair.Name) {
+			continue
+		}
+
+		nameGeneral := strings.ToLower(pair.Name)
+
+		initHeaders(respObj)
+		objHeaders := respObj.Headers
+		param := findHeaderByName(&respObj.Headers, pair.Name)
+		if param == nil {
+			param = createHeader(openapi.TypeString)
+			objHeaders[nameGeneral] = param
+		}
+		exmp := &param.Examples
+		err := fillParamExample(&exmp, pair.Value)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+		visited[nameGeneral] = param
+	}
+
+	// maintain "required" flag
+	if respObj.Headers != nil {
+		for name, param := range respObj.Headers {
+			paramObj, err := param.ResolveHeader(headerResolver)
+			if err != nil {
+				logger.Log.Warningf("Failed to resolve param: %s", err)
+				continue
+			}
+
+			_, ok := visited[strings.ToLower(name)]
+			if !ok {
+				flag := false
+				paramObj.Required = &flag
+			}
+		}
+	}
+
+	return
+}
+
+func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err error) (*openapi.MediaType, error) {
+	content, found := respContent[ctype]
+	if !found {
+		respContent[ctype] = &openapi.MediaType{}
+		content = respContent[ctype]
+	}
+
+	var text string
+	if reqResp.Req != nil {
+		text = reqResp.Req.PostData.Text
+	} else {
+		text = decRespText(reqResp.Resp.Content)
+	}
+
+	var exampleMsg []byte
+	// try treating it as json
+	any, isJSON := anyJSON(text)
+	if isJSON {
+		// re-marshal with forced indent
+		exampleMsg, err = json.MarshalIndent(any, "", "\t")
+		if err != nil {
+			panic("Failed to re-marshal value, super-strange")
+		}
+	} else {
+		exampleMsg, err = json.Marshal(text)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	content.Example = exampleMsg
+	return respContent[ctype], nil
+}
+
+func decRespText(content *har.Content) (res string) {
+	res = string(content.Text)
+	if content.Encoding == "base64" {
+		data, err := base64.StdEncoding.DecodeString(res)
+		if err != nil {
+			logger.Log.Warningf("error decoding response text as base64: %s", err)
+		} else {
+			res = string(data)
+		}
+	}
+	return
+}
+
+func getRespCtype(resp *har.Response) string {
+	var ctype string
+	ctype = resp.Content.MimeType
+	for _, hdr := range resp.Headers {
+		if strings.ToLower(hdr.Name) == "content-type" {
+			ctype = hdr.Value
+		}
+	}
+
+	mediaType, _, err := mime.ParseMediaType(ctype)
+	if err != nil {
+		return ""
+	}
+	return mediaType
+}
+
+func getReqCtype(req *har.Request) string {
+	var ctype string
+	ctype = req.PostData.MimeType
+	for _, hdr := range req.Headers {
+		if strings.ToLower(hdr.Name) == "content-type" {
+			ctype = hdr.Value
+		}
+	}
+
+	mediaType, _, err := mime.ParseMediaType(ctype)
+	if err != nil {
+		return ""
+	}
+	return mediaType
+}
+
+func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool) (*openapi.ResponseObj, error) {
+	statusStr := strconv.Itoa(resp.Status)
+
+	var response openapi.Response
+	response, found := opObj.Responses[statusStr]
+	if !found {
+		if opObj.Responses == nil {
+			opObj.Responses = map[string]openapi.Response{}
+		}
+
+		opObj.Responses[statusStr] = &openapi.ResponseObj{Content: map[string]*openapi.MediaType{}}
+		response = opObj.Responses[statusStr]
+	}
+
+	resResponse, err := response.ResolveResponse(responseResolver)
+	if err != nil {
+		return nil, err
+	}
+
+	if isSuccess {
+		resResponse.Description = "Successful call with status " + statusStr
+	} else {
+		resResponse.Description = "Failed call with status " + statusStr
+	}
+	return resResponse, nil
+}
+
+func getRequestBody(req *har.Request, opObj *openapi.Operation, isSuccess bool) (*openapi.RequestBodyObj, error) {
+	if opObj.RequestBody == nil {
+		opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
+	}
+
+	reqBody, err := opObj.RequestBody.ResolveRequestBody(reqBodyResolver)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: maintain required flag for it, but only consider successful responses
+	//reqBody.Content[]
+
+	return reqBody, nil
+}
+
+func getOpObj(pathObj *openapi.PathObj, method string, createIfNone bool) (*openapi.Operation, bool, error) {
+	method = strings.ToLower(method)
+	var op **openapi.Operation
+
+	switch method {
+	case "get":
+		op = &pathObj.Get
+	case "put":
+		op = &pathObj.Put
+	case "post":
+		op = &pathObj.Post
+	case "delete":
+		op = &pathObj.Delete
+	case "options":
+		op = &pathObj.Options
+	case "head":
+		op = &pathObj.Head
+	case "patch":
+		op = &pathObj.Patch
+	case "trace":
+		op = &pathObj.Trace
+	default:
+		return nil, false, errors.New("unsupported HTTP method: " + method)
+	}
+
+	isMissing := false
+	if *op == nil {
+		isMissing = true
+		if createIfNone {
+			*op = &openapi.Operation{Responses: map[string]openapi.Response{}}
+			newUUID := uuid.New().String()
+			(**op).OperationID = newUUID
+		} else {
+			return nil, isMissing, nil
+		}
+	}
+
+	return *op, isMissing, nil
+}

agent/pkg/oas/specgen_test.go

@@ -0,0 +1,197 @@
+package oas
+
+import (
+	"encoding/json"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+)
+
+// if started via env, write file into subdir
+func writeFiles(label string, spec *openapi.OpenAPI) {
+	if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
+		path := "./oas-samples"
+		err := os.MkdirAll(path, 0o755)
+		if err != nil {
+			panic(err)
+		}
+		content, err := json.MarshalIndent(spec, "", "\t")
+		if err != nil {
+			panic(err)
+		}
+		err = ioutil.WriteFile(path+"/"+label+".json", content, 0644)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+func TestEntries(t *testing.T) {
+	files, err := getFiles("./test_artifacts/")
+	// files, err = getFiles("/media/bigdisk/UP9")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	GetOasGeneratorInstance().Start()
+
+	if err := feedEntries(files); err != nil {
+		t.Log(err)
+		t.Fail()
+	}
+
+	loadStartingOAS()
+
+	svcs := strings.Builder{}
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+		gen := val.(*SpecGen)
+		svc := key.(string)
+		svcs.WriteString(svc + ",")
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+			return false
+		}
+
+		err = spec.Validate()
+		if err != nil {
+			specText, _ := json.MarshalIndent(spec, "", "\t")
+			t.Log(string(specText))
+			t.Log(err)
+			t.FailNow()
+		}
+
+		return true
+	})
+
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+		svc := key.(string)
+		gen := val.(*SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		specText, _ := json.MarshalIndent(spec, "", "\t")
+		t.Logf("%s", string(specText))
+
+		err = spec.Validate()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+		writeFiles(svc, spec)
+
+		return true
+	})
+
+}
+
+func TestFileLDJSON(t *testing.T) {
+	GetOasGeneratorInstance().Start()
+	file := "test_artifacts/output_rdwtyeoyrj.har.ldjson"
+	err := feedFromLDJSON(file)
+	if err != nil {
+		logger.Log.Warning("Failed processing file: " + err.Error())
+		t.Fail()
+	}
+
+	loadStartingOAS()
+
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(_, val interface{}) bool {
+		gen := val.(*SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		specText, _ := json.MarshalIndent(spec, "", "\t")
+		t.Logf("%s", string(specText))
+
+		err = spec.Validate()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		return true
+	})
+}
+
+func loadStartingOAS() {
+	file := "test_artifacts/catalogue.json"
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		panic(err)
+	}
+
+	var doc *openapi.OpenAPI
+	err = json.Unmarshal(data, &doc)
+	if err != nil {
+		panic(err)
+	}
+
+	gen := NewGen("catalogue")
+	gen.StartFromSpec(doc)
+
+	GetOasGeneratorInstance().ServiceSpecs.Store("catalogue", gen)
+
+	return
+}
+
+func TestEntriesNegative(t *testing.T) {
+	files := []string{"invalid"}
+	err := feedEntries(files)
+	if err == nil {
+		t.Logf("Should have failed")
+		t.Fail()
+	}
+}
+
+func TestLoadValidHAR(t *testing.T) {
+	inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}`
+	var entry *har.Entry
+	var err = json.Unmarshal([]byte(inp), &entry)
+	if err != nil {
+		t.Logf("Failed to decode entry: %s", err)
+		// t.FailNow() demonstrates the problem of library
+	}
+}
+
+func TestLoadValid3_1(t *testing.T) {
+	fd, err := os.Open("test_artifacts/catalogue.json")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	var oas openapi.OpenAPI
+	err = json.Unmarshal(data, &oas)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	return
+}

agent/pkg/oas/test_artifacts/catalogue.json

@@ -0,0 +1,51 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "Preloaded",
+    "version": "0.1",
+    "description": "Test file for loading pre-existing OAS"
+  },
+  "paths": {
+    "/catalogue/{id}": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ],
+      "get": {
+        "parameters": [        {
+          "name": "non-required-header",
+          "in": "header",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+        ]
+      }
+    },
+    "/catalogue/{id}/details": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "style": "simple",
+          "required": true,
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ]
+    }
+  }
+}

agent/pkg/oas/test_artifacts/output_rdwtyeoyrj.har.ldjson

@@ -0,0 +1,13 @@
+{"messageType": "http", "_source": null, "firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78415179Z", "time": 13, "request": {"method": "GET", "url": "http://catalogue/catalogue/size?tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "x-some", "value": "demo val"},{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [{"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "22"}], "content": {"size": 22, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInNpemUiOjl9", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 22}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 13}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.784918698Z", "time": 19, "request": {"method": "GET", "url": "http://catalogue/catalogue?page=1&size=6&tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "page", "value": "1"}, {"name": "size", "value": "6"}, {"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "1927"}], "content": {"size": 1927, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19LHsiaWQiOiI4MzdhYjE0MS0zOTllLTRjMWYtOWFiYy1iYWNlNDAyOTZiYWMiLCJuYW1lIjoiQ2F0IHNvY2tzIiwiZGVzY3JpcHRpb24iOiJjb25zZXF1YXQgYW1ldCBjdXBpZGF0YXQgbWluaW0gbGFib3J1bSB0ZW1wb3IgZWxpdCBleCBjb25zZXF1YXQgaW4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2NrczIuanBnIl0sInByaWNlIjoxNSwiY291bnQiOjE3NSwidGFnIjpbImJyb3duIiwiZm9ybWFsIiwiZ3JlZW4iXX1dCg==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1927}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 19}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78418182Z", "time": 7, "request": {"method": "GET", "url": "http://catalogue/tags", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "107"}], "content": {"size": 107, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInRhZ3MiOlsiYnJvd24iLCJnZWVrIiwiZm9ybWFsIiwiYmx1ZSIsInNraW4iLCJyZWQiLCJhY3Rpb24iLCJzcG9ydCIsImJsYWNrIiwibWFnaWMiLCJncmVlbiJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 107}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 7}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:18.131501482Z", "time": 5, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}, {"name": "x-some", "value": "demoval"}], ",queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 5}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:18.379836908Z", "time": 14, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 14}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.920540124Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/808a2de1-1aaa-4c25-a9b9-6612e8f29a38", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "275"}], "content": {"size": 275, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NzM4LCJkZXNjcmlwdGlvbiI6IkEgbWF0dXJlIHNvY2ssIGNyb3NzZWQsIHdpdGggYW4gYWlyIG9mIG5vbmNoYWxhbmNlLiIsImlkIjoiODA4YTJkZTEtMWFhYS00YzI1LWE5YjktNjYxMmU4ZjI5YTM4IiwiaW1hZ2VVcmwiOlsiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMS5qcGVnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMi5qcGVnIl0sIm5hbWUiOiJDcm9zc2VkIiwicHJpY2UiOjE3LjMyLCJ0YWciOlsiYmx1ZSIsInJlZCIsImFjdGlvbiIsImZvcm1hbCJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 275}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.921609501Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?sort=id&size=3&tags=blue", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "3"}, {"name": "tags", "value": "blue"}, {"name": "sort", "value": "id"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "789"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}], "content": {"size": 789, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJuYW1lIjoiQ29sb3VyZnVsIiwiZGVzY3JpcHRpb24iOiJwcm9pZGVudCBvY2NhZWNhdCBpcnVyZSBldCBleGNlcHRldXIgbGFib3JlIG1pbmltIG5pc2kgYW1ldCBpcnVyZSIsImltYWdlVXJsIjpbIi9jYXRhbG9ndWUvaW1hZ2VzL2NvbG91cmZ1bF9zb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIl0sInByaWNlIjoxOCwiY291bnQiOjQzOCwidGFnIjpbImJsdWUiXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiYmx1ZSJdfV0K", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 789}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.923197848Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:23.175549218Z", "time": 26, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 26}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.239777333Z", "time": 10, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Transfer-Encoding", "value": "chunked"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "X-Application-Context", "value": "carts:80"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 10}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.725866772Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?size=5", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "5"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "1643"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}], "content": {"size": 1643, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19XQo=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1643}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.729303217Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}

agent/pkg/oas/tree.go

@@ -0,0 +1,205 @@
+package oas
+
+import (
+	"github.com/chanced/openapi"
+	"github.com/up9inc/mizu/shared/logger"
+	"strconv"
+	"strings"
+)
+
+type NodePath = []string
+
+type Node struct {
+	constant *string
+	param    *openapi.ParameterObj
+	ops      *openapi.PathObj
+	parent   *Node
+	children []*Node
+}
+
+func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Node) {
+	if pathObjToSet == nil {
+		panic("Invalid function call")
+	}
+
+	pathChunk := path[0]
+	chunkIsParam := strings.HasPrefix(pathChunk, "{") && strings.HasSuffix(pathChunk, "}")
+	chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
+
+	var paramObj *openapi.ParameterObj
+	if chunkIsParam && pathObjToSet != nil {
+		paramObj = findParamByName(pathObjToSet.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
+	}
+
+	if paramObj == nil {
+		node = n.searchInConstants(pathChunk)
+	}
+
+	if node == nil {
+		node = n.searchInParams(paramObj, chunkIsGibberish)
+	}
+
+	// still no node found, should create it
+	if node == nil {
+		node = new(Node)
+		node.parent = n
+		n.children = append(n.children, node)
+
+		if paramObj != nil {
+			node.param = paramObj
+		} else if chunkIsGibberish {
+			initParams(&pathObjToSet.Parameters)
+
+			newParam := n.createParam()
+			node.param = newParam
+
+			appended := append(*pathObjToSet.Parameters, newParam)
+			pathObjToSet.Parameters = &appended
+		} else {
+			node.constant = &pathChunk
+		}
+	}
+
+	// add example if it's a param
+	if node.param != nil && !chunkIsParam {
+		exmp := &node.param.Examples
+		err := fillParamExample(&exmp, pathChunk)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+	}
+
+	// TODO: eat up trailing slash, in a smart way: node.ops!=nil && path[1]==""
+	if len(path) > 1 {
+		return node.getOrSet(path[1:], pathObjToSet)
+	} else if node.ops == nil {
+		node.ops = pathObjToSet
+	}
+
+	return node
+}
+
+func (n *Node) createParam() *openapi.ParameterObj {
+	name := "param"
+
+	// REST assumption, not always correct
+	if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
+		name = *n.constant
+		name = name[:len(name)-2] + "Id"
+	} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
+		name = *n.constant
+		name = name[:len(name)-1] + "Id"
+	}
+
+	newParam := createSimpleParam(name, "path", "string")
+	x := n.countParentParams()
+	if x > 1 {
+		newParam.Name = newParam.Name + strconv.Itoa(x)
+	}
+
+	return newParam
+}
+
+func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunkIsGibberish bool) *Node {
+	// look among params
+	if paramObj != nil || chunkIsGibberish {
+		for _, subnode := range n.children {
+			if subnode.constant != nil {
+				continue
+			}
+
+			// TODO: check the regex pattern of param? for exceptions etc
+
+			if paramObj != nil {
+				// TODO: mergeParam(subnode.param, paramObj)
+				return subnode
+			} else {
+				return subnode
+			}
+		}
+	}
+	return nil
+}
+
+func (n *Node) searchInConstants(pathChunk string) *Node {
+	// look among constants
+	for _, subnode := range n.children {
+		if subnode.constant == nil {
+			continue
+		}
+
+		if *subnode.constant == pathChunk {
+			return subnode
+		}
+	}
+	return nil
+}
+
+func (n *Node) compact() {
+	// TODO
+}
+
+func (n *Node) listPaths() *openapi.Paths {
+	paths := &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
+
+	var strChunk string
+	if n.constant != nil {
+		strChunk = *n.constant
+	} else if n.param != nil {
+		strChunk = "{" + n.param.Name + "}"
+	} else {
+		// this is the root node
+	}
+
+	// add self
+	if n.ops != nil {
+		paths.Items[openapi.PathValue(strChunk)] = n.ops
+	}
+
+	// recurse into children
+	for _, child := range n.children {
+		subPaths := child.listPaths()
+		for path, pathObj := range subPaths.Items {
+			var concat string
+			if n.parent == nil {
+				concat = string(path)
+			} else {
+				concat = strChunk + "/" + string(path)
+			}
+			paths.Items[openapi.PathValue(concat)] = pathObj
+		}
+	}
+
+	return paths
+}
+
+type PathAndOp struct {
+	path string
+	op   *openapi.Operation
+}
+
+func (n *Node) listOps() []PathAndOp {
+	res := make([]PathAndOp, 0)
+	for path, pathObj := range n.listPaths().Items {
+		for _, op := range getOps(pathObj) {
+			res = append(res, PathAndOp{path: string(path), op: op})
+		}
+	}
+	return res
+}
+
+func (n *Node) countParentParams() int {
+	res := 0
+	node := n
+	for {
+		if node.param != nil {
+			res++
+		}
+
+		if node.parent == nil {
+			break
+		}
+		node = node.parent
+	}
+	return res
+}

agent/pkg/oas/utils.go

@@ -0,0 +1,344 @@
+package oas
+
+import (
+	"encoding/json"
+	"errors"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"strconv"
+	"strings"
+)
+
+func exampleResolver(ref string) (*openapi.ExampleObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func responseResolver(ref string) (*openapi.ResponseObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func reqBodyResolver(ref string) (*openapi.RequestBodyObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func paramResolver(ref string) (*openapi.ParameterObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func headerResolver(ref string) (*openapi.HeaderObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func initParams(obj **openapi.ParameterList) {
+	if *obj == nil {
+		var params openapi.ParameterList
+		params = make([]openapi.Parameter, 0)
+		*obj = &params
+	}
+}
+
+func initHeaders(respObj *openapi.ResponseObj) {
+	if respObj.Headers == nil {
+		var created openapi.Headers
+		created = map[string]openapi.Header{}
+		respObj.Headers = created
+	}
+}
+
+func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *openapi.ParameterObj {
+	if name == "" {
+		panic("Cannot create parameter with empty name")
+	}
+	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
+	schema := new(openapi.SchemaObj)
+	schema.Type = make(openapi.Types, 0)
+	schema.Type = append(schema.Type, ptype)
+
+	style := openapi.StyleSimple
+	if in == openapi.InQuery {
+		style = openapi.StyleForm
+	}
+
+	newParam := openapi.ParameterObj{
+		Name:     name,
+		In:       in,
+		Style:    string(style),
+		Examples: map[string]openapi.Example{},
+		Schema:   schema,
+		Required: &required,
+	}
+	return &newParam
+}
+
+func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (pathParam *openapi.ParameterObj) {
+	caseInsensitive := in == openapi.InHeader
+	for _, param := range *params {
+		paramObj, err := param.ResolveParameter(paramResolver)
+		if err != nil {
+			logger.Log.Warningf("Failed to resolve reference: %s", err)
+			continue
+		}
+
+		if paramObj.In != in {
+			continue
+		}
+
+		if paramObj.Name == name || (caseInsensitive && strings.ToLower(paramObj.Name) == strings.ToLower(name)) {
+			pathParam = paramObj
+			break
+		}
+	}
+	return pathParam
+}
+
+func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj {
+	for hname, param := range *headers {
+		hdrObj, err := param.ResolveHeader(headerResolver)
+		if err != nil {
+			logger.Log.Warningf("Failed to resolve reference: %s", err)
+			continue
+		}
+
+		if strings.ToLower(hname) == strings.ToLower(name) {
+			return hdrObj
+		}
+	}
+	return nil
+}
+
+type NVPair struct {
+	Name  string
+	Value string
+}
+
+type nvParams struct {
+	In             openapi.In
+	Pairs          func() []NVPair
+	IsIgnored      func(name string) bool
+	GeneralizeName func(name string) string
+}
+
+func qstrToNVP(list []har.QueryString) []NVPair {
+	res := make([]NVPair, len(list))
+	for idx, val := range list {
+		res[idx] = NVPair{Name: val.Name, Value: val.Value}
+	}
+	return res
+}
+
+func hdrToNVP(list []har.Header) []NVPair {
+	res := make([]NVPair, len(list))
+	for idx, val := range list {
+		res[idx] = NVPair{Name: val.Name, Value: val.Value}
+	}
+	return res
+}
+
+func handleNameVals(gw nvParams, params **openapi.ParameterList) {
+	visited := map[string]*openapi.ParameterObj{}
+	for _, pair := range gw.Pairs() {
+		if gw.IsIgnored(pair.Name) {
+			continue
+		}
+
+		nameGeneral := gw.GeneralizeName(pair.Name)
+
+		initParams(params)
+		param := findParamByName(*params, gw.In, pair.Name)
+		if param == nil {
+			param = createSimpleParam(nameGeneral, gw.In, openapi.TypeString)
+			appended := append(**params, param)
+			*params = &appended
+		}
+		exmp := &param.Examples
+		err := fillParamExample(&exmp, pair.Value)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+		visited[nameGeneral] = param
+	}
+
+	// maintain "required" flag
+	if *params != nil {
+		for _, param := range **params {
+			paramObj, err := param.ResolveParameter(paramResolver)
+			if err != nil {
+				logger.Log.Warningf("Failed to resolve param: %s", err)
+				continue
+			}
+			if paramObj.In != gw.In {
+				continue
+			}
+
+			_, ok := visited[strings.ToLower(paramObj.Name)]
+			if !ok {
+				flag := false
+				paramObj.Required = &flag
+			}
+		}
+	}
+}
+
+func createHeader(ptype openapi.SchemaType) *openapi.HeaderObj {
+	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
+	schema := new(openapi.SchemaObj)
+	schema.Type = make(openapi.Types, 0)
+	schema.Type = append(schema.Type, ptype)
+
+	style := openapi.StyleSimple
+	newParam := openapi.HeaderObj{
+		Style:    string(style),
+		Examples: map[string]openapi.Example{},
+		Schema:   schema,
+		Required: &required,
+	}
+	return &newParam
+}
+
+func fillParamExample(param **openapi.Examples, exampleValue string) error {
+	if **param == nil {
+		**param = map[string]openapi.Example{}
+	}
+
+	cnt := 0
+	for _, example := range **param {
+		cnt++
+		exampleObj, err := example.ResolveExample(exampleResolver)
+		if err != nil {
+			continue
+		}
+
+		var value string
+		err = json.Unmarshal(exampleObj.Value, &value)
+		if err != nil {
+			logger.Log.Warningf("Failed decoding parameter example into string: %s", err)
+			continue
+		}
+
+		if value == exampleValue || cnt > 5 { // 5 examples is enough
+			return nil
+		}
+	}
+
+	valMsg, err := json.Marshal(exampleValue)
+	if err != nil {
+		return err
+	}
+
+	themap := **param
+	themap["example #"+strconv.Itoa(cnt)] = &openapi.ExampleObj{Value: valMsg}
+
+	return nil
+}
+
+func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.com/jpillora/longestcommon
+	empty := make([]string, 0)
+	//short-circuit empty list
+	if len(strs) == 0 {
+		return empty
+	}
+	xfix := strs[0]
+	//short-circuit single-element list
+	if len(strs) == 1 {
+		return xfix
+	}
+	//compare first to rest
+	for _, str := range strs[1:] {
+		xfixl := len(xfix)
+		strl := len(str)
+		//short-circuit empty strings
+		if xfixl == 0 || strl == 0 {
+			return empty
+		}
+		//maximum possible length
+		maxl := xfixl
+		if strl < maxl {
+			maxl = strl
+		}
+		//compare letters
+		if pre {
+			//prefix, iterate left to right
+			for i := 0; i < maxl; i++ {
+				if xfix[i] != str[i] {
+					xfix = xfix[:i]
+					break
+				}
+			}
+		} else {
+			//suffix, iternate right to left
+			for i := 0; i < maxl; i++ {
+				xi := xfixl - i - 1
+				si := strl - i - 1
+				if xfix[xi] != str[si] {
+					xfix = xfix[xi+1:]
+					break
+				}
+			}
+		}
+	}
+	return xfix
+}
+
+// returns all non-nil ops in PathObj
+func getOps(pathObj *openapi.PathObj) []*openapi.Operation {
+	ops := []**openapi.Operation{&pathObj.Get, &pathObj.Patch, &pathObj.Put, &pathObj.Options, &pathObj.Post, &pathObj.Trace, &pathObj.Head, &pathObj.Delete}
+	res := make([]*openapi.Operation, 0)
+	for _, opp := range ops {
+		if *opp == nil {
+			continue
+		}
+		res = append(res, *opp)
+	}
+	return res
+}
+
+// parses JSON into any possible value
+func anyJSON(text string) (anyVal interface{}, isJSON bool) {
+	isJSON = true
+	asMap := map[string]interface{}{}
+	err := json.Unmarshal([]byte(text), &asMap)
+	if err == nil && asMap != nil {
+		return asMap, isJSON
+	}
+
+	asArray := make([]interface{}, 0)
+	err = json.Unmarshal([]byte(text), &asArray)
+	if err == nil && asArray != nil {
+		return asArray, isJSON
+	}
+
+	asString := ""
+	sPtr := &asString
+	err = json.Unmarshal([]byte(text), &sPtr)
+	if err == nil && sPtr != nil {
+		return asString, isJSON
+	}
+
+	asInt := 0
+	intPtr := &asInt
+	err = json.Unmarshal([]byte(text), &intPtr)
+	if err == nil && intPtr != nil {
+		return asInt, isJSON
+	}
+
+	asFloat := 0.0
+	floatPtr := &asFloat
+	err = json.Unmarshal([]byte(text), &floatPtr)
+	if err == nil && floatPtr != nil {
+		return asFloat, isJSON
+	}
+
+	asBool := false
+	boolPtr := &asBool
+	err = json.Unmarshal([]byte(text), &boolPtr)
+	if err == nil && boolPtr != nil {
+		return asBool, isJSON
+	}
+
+	if text == "null" {
+		return nil, isJSON
+	}
+
+	return nil, false
+}

agent/pkg/oas/utils_test.go

@@ -0,0 +1,35 @@
+package oas
+
+import (
+	"testing"
+)
+
+func TestAnyJSON(t *testing.T) {
+	testCases := []struct {
+		inp    string
+		isJSON bool
+		out    interface{}
+	}{
+		{`{"key": 1, "keyNull": null}`, true, nil},
+		{`[{"key": "val"}, ["subarray"], "string", 1, 2.2, true, null]`, true, nil},
+		{`"somestring"`, true, "somestring"},
+		{"0", true, 0},
+		{"0.5", true, 0.5},
+		{"true", true, true},
+		{"null", true, nil},
+		{"sabbra cadabra", false, nil},
+		{"0.1.2.3", false, nil},
+	}
+	for _, tc := range testCases {
+		any, isJSON := anyJSON(tc.inp)
+		if isJSON != tc.isJSON {
+			t.Errorf("Parse flag mismatch: %t != %t", tc.isJSON, isJSON)
+		} else if isJSON && tc.out != nil && tc.out != any {
+			t.Errorf("%s != %s", any, tc.out)
+		} else if tc.inp == "null" && any != nil {
+			t.Errorf("null has to parse as nil (but got %s)", any)
+		} else {
+			t.Logf("%s => %s", tc.inp, any)
+		}
+	}
+}

agent/pkg/providers/install_provider.go

@@ -2,9 +2,14 @@ package providers
 
 import (
 	"context"
+	"errors"
 	"mizuserver/pkg/config"
+
+	ory "github.com/ory/kratos-client-go"
 )
 
+const AdminUsername = "admin"
+
 func IsInstallNeeded() (bool, error) {
 	if !config.Config.StandaloneMode { // install not needed in ephermeral mizu
 		return false, nil
@@ -16,3 +21,27 @@ func IsInstallNeeded() (bool, error) {
 		return !anyUserExists, nil
 	}
 }
+
+func CreateAdminUser(password string, ctx context.Context) (token *string, err error, formErrorMessages map[string][]ory.UiText) {
+	if isInstallNeeded, err := IsInstallNeeded(); err != nil {
+		return nil, err, nil
+	} else if !isInstallNeeded {
+		return nil, errors.New("The admin user has already been created"), nil
+	}
+
+	token, identityId, err, formErrors := RegisterUser(AdminUsername, password, ctx)
+	if err != nil {
+		return nil, err, formErrors
+	}
+
+	err = SetUserSystemRole(AdminUsername, AdminRole)
+
+	if err != nil {
+		//Delete the user to prevent a half-setup situation where admin user is created without admin privileges
+		DeleteUser(identityId, ctx)
+
+		return nil, err, nil
+	}
+
+	return token, nil, nil
+}

agent/pkg/providers/status_provider.go

@@ -8,19 +8,14 @@ import (
 	"github.com/up9inc/mizu/tap"
 	"mizuserver/pkg/models"
 	"os"
-	"sync"
 	"time"
 )
 
 const tlsLinkRetainmentTime = time.Minute * 15
 
 var (
-	TappersCount         int
-	TapStatus            shared.TapStatus
-	TappersStatus        map[string]shared.TapperStatus
-	authStatus           *models.AuthStatus
-	RecentTLSLinks       = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
-	tappersCountLock     = sync.Mutex{}
+	authStatus     *models.AuthStatus
+	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
 )
 
 func GetAuthStatus() (*models.AuthStatus, error) {
@@ -68,15 +63,3 @@ func GetAllRecentTLSAddresses() []string {
 
 	return recentTLSLinks
 }
-
-func TapperAdded() {
-	tappersCountLock.Lock()
-	TappersCount++
-	tappersCountLock.Unlock()
-}
-
-func TapperRemoved() {
-	tappersCountLock.Lock()
-	TappersCount--
-	tappersCountLock.Unlock()
-}

agent/pkg/providers/tapConfig/tap_config_provider.go

@@ -0,0 +1,42 @@
+package tapConfig
+
+import (
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/utils"
+	"os"
+	"sync"
+)
+
+const FilePath = shared.DataDirPath + "tap-config.json"
+
+var (
+	lock     = &sync.Mutex{}
+	syncOnce sync.Once
+	config   *models.TapConfig
+)
+
+func Get() *models.TapConfig {
+	syncOnce.Do(func() {
+		if err := utils.ReadJsonFile(FilePath, &config); err != nil {
+			config = &models.TapConfig{TappedNamespaces: make(map[string]bool)}
+
+			if !os.IsNotExist(err) {
+				logger.Log.Errorf("Error reading tap config from file, err: %v", err)
+			}
+		}
+	})
+
+	return config
+}
+
+func Save(tapConfigToSave *models.TapConfig) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	config = tapConfigToSave
+	if err := utils.SaveJsonFile(FilePath, config); err != nil {
+		logger.Log.Errorf("Error saving tap config, err: %v", err)
+	}
+}

agent/pkg/providers/tappedPods/tapped_pods_provider.go

@@ -0,0 +1,56 @@
+package tappedPods
+
+import (
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/providers/tappers"
+	"mizuserver/pkg/utils"
+	"os"
+	"strings"
+	"sync"
+)
+
+const FilePath = shared.DataDirPath + "tapped-pods.json"
+
+var (
+	lock       = &sync.Mutex{}
+	syncOnce   sync.Once
+	tappedPods []*shared.PodInfo
+)
+
+func Get() []*shared.PodInfo {
+	syncOnce.Do(func() {
+		if err := utils.ReadJsonFile(FilePath, &tappedPods); err != nil {
+			if !os.IsNotExist(err) {
+				logger.Log.Errorf("Error reading tapped pods from file, err: %v", err)
+			}
+		}
+	})
+
+	return tappedPods
+}
+
+func Set(tappedPodsToSet []*shared.PodInfo) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	tappedPods = tappedPodsToSet
+	if err := utils.SaveJsonFile(FilePath, tappedPods); err != nil {
+		logger.Log.Errorf("Error saving tapped pods, err: %v", err)
+	}
+}
+
+func GetTappedPodsStatus() []shared.TappedPodStatus {
+	tappedPodsStatus := make([]shared.TappedPodStatus, 0)
+	for _, pod := range Get() {
+		var status string
+		if tapperStatus, ok := tappers.GetStatus()[pod.NodeName]; ok {
+			status = strings.ToLower(tapperStatus.Status)
+		}
+
+		isTapped := status == "running"
+		tappedPodsStatus = append(tappedPodsStatus, shared.TappedPodStatus{Name: pod.Name, Namespace: pod.Namespace, IsTapped: isTapped})
+	}
+
+	return tappedPodsStatus
+}

agent/pkg/providers/tappers/tappers_provider.go

@@ -0,0 +1,82 @@
+package tappers
+
+import (
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/utils"
+	"os"
+	"sync"
+)
+
+const FilePath = shared.DataDirPath + "tappers-status.json"
+
+var (
+	lockStatus = &sync.Mutex{}
+	syncOnce   sync.Once
+	status     map[string]*shared.TapperStatus
+
+	lockConnectedCount = &sync.Mutex{}
+	connectedCount     int
+)
+
+func GetStatus() map[string]*shared.TapperStatus {
+	initStatus()
+
+	return status
+}
+
+func SetStatus(tapperStatus *shared.TapperStatus) {
+	initStatus()
+
+	lockStatus.Lock()
+	defer lockStatus.Unlock()
+
+	status[tapperStatus.NodeName] = tapperStatus
+
+	saveStatus()
+}
+
+func ResetStatus() {
+	lockStatus.Lock()
+	defer lockStatus.Unlock()
+
+	status = make(map[string]*shared.TapperStatus)
+
+	saveStatus()
+}
+
+func GetConnectedCount() int {
+	return connectedCount
+}
+
+func Connected() {
+	lockConnectedCount.Lock()
+	defer lockConnectedCount.Unlock()
+
+	connectedCount++
+}
+
+func Disconnected() {
+	lockConnectedCount.Lock()
+	defer lockConnectedCount.Unlock()
+
+	connectedCount--
+}
+
+func initStatus() {
+	syncOnce.Do(func() {
+		if err := utils.ReadJsonFile(FilePath, &status); err != nil {
+			status = make(map[string]*shared.TapperStatus)
+
+			if !os.IsNotExist(err) {
+				logger.Log.Errorf("Error reading tappers status from file, err: %v", err)
+			}
+		}
+	})
+}
+
+func saveStatus() {
+	if err := utils.SaveJsonFile(FilePath, status); err != nil {
+		logger.Log.Errorf("Error saving tappers status, err: %v", err)
+	}
+}

agent/pkg/providers/user_provider.go

@@ -66,17 +66,17 @@ func PerformLogin(username string, password string, ctx context.Context) (*strin
 	return result.SessionToken, nil
 }
 
-func VerifyToken(token string, ctx context.Context) (bool, error) {
+func VerifyToken(token string, ctx context.Context) (*ory.Session, error) {
 	flow, _, err := client.V0alpha2Api.ToSession(ctx).XSessionToken(token).Execute()
 	if err != nil {
-		return false, err
+		return nil, err
 	}
 
 	if flow == nil {
-		return false, nil
+		return nil, nil
 	}
 
-	return true, nil
+	return flow, nil
 }
 
 func DeleteUser(identityId string, ctx context.Context) error {

agent/pkg/providers/user_role_provider.go

@@ -0,0 +1,180 @@
+package providers
+
+/*
+This provider abstracts keto role management down to what we need for mizu
+
+Keto, in the configuration we use it, is basically a tuple database. Each tuple consists of 4 strings (namespace, object, relation, subjectID) - for example ("workspaces", "sock-shop-workspace", "viewer", "ramiberman")
+
+namespace - used to organize tuples into groups - we currently use "system" for defining admins and "workspaces" for defining workspace permissions
+objects - represents something one can have permissions to (files, mizu workspaces etc)
+relation - represents the permission (viewer, editor, owner etc) - we currently use only viewer and admin
+subject - represents the user or group that has the permission - we currently use usernames
+
+more on keto here: https://www.ory.sh/keto/docs/
+*/
+
+import (
+	"fmt"
+	"mizuserver/pkg/utils"
+
+	ketoClient "github.com/ory/keto-client-go/client"
+	ketoRead "github.com/ory/keto-client-go/client/read"
+	ketoWrite "github.com/ory/keto-client-go/client/write"
+	ketoModels "github.com/ory/keto-client-go/models"
+)
+
+const (
+	ketoHost      = "localhost"
+	ketoReadPort  = 4466
+	ketoWritePort = 4467
+)
+
+var (
+	readClient              = getKetoClient(fmt.Sprintf("%s:%d", ketoHost, ketoReadPort))
+	writeClient             = getKetoClient(fmt.Sprintf("%s:%d", ketoHost, ketoWritePort))
+	systemRoleNamespace     = "system"
+	workspacesRoleNamespace = "workspaces"
+
+	systemObject = "system"
+
+	AdminRole  = "admin"
+	ViewerRole = "viewer"
+)
+
+func GetUserSystemRoles(username string) ([]string, error) {
+	return getObjectRelationsForSubjectID(systemRoleNamespace, systemObject, username)
+}
+
+func CheckIfUserHasSystemRole(username string, role string) (bool, error) {
+	systemRoles, err := GetUserSystemRoles(username)
+	if err != nil {
+		return false, err
+	}
+
+	for _, systemRole := range systemRoles {
+		if systemRole == role {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func GetUserWorkspaceRole(username string, workspace string) ([]string, error) {
+	return getObjectRelationsForSubjectID(workspacesRoleNamespace, workspace, username)
+}
+
+func SetUserWorkspaceRole(username string, workspace string, role string) error {
+	return createObjectRelationForSubjectID(workspacesRoleNamespace, workspace, username, role)
+}
+
+func SetUserSystemRole(username string, role string) error {
+	return createObjectRelationForSubjectID(systemRoleNamespace, systemObject, username, role)
+}
+
+func DeleteAllUserWorkspaceRoles(username string) error {
+	return deleteAllNamespacedRelationsForSubjectID(workspacesRoleNamespace, username)
+}
+
+func createObjectRelationForSubjectID(namespace string, object string, subjectID string, relation string) error {
+	tuple := ketoModels.RelationQuery{
+		Namespace: &namespace,
+		Object:    object,
+		Relation:  relation,
+		SubjectID: subjectID,
+	}
+
+	_, err := writeClient.Write.CreateRelationTuple(ketoWrite.
+		NewCreateRelationTupleParams().
+		WithPayload(&tuple))
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getObjectRelationsForSubjectID(namespace string, object string, subjectID string) ([]string, error) {
+	relationTuples, err := queryRelationTuples(&namespace, &object, &subjectID, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	relations := make([]string, 0)
+
+	for _, clientRelation := range relationTuples {
+		relations = append(relations, *clientRelation.Relation)
+	}
+
+	return utils.UniqueStringSlice(relations), nil
+}
+
+func deleteAllNamespacedRelationsForSubjectID(namespace string, subjectID string) error {
+	relationTuples, err := queryRelationTuples(&namespace, nil, &subjectID, nil)
+	if err != nil {
+		return err
+	}
+
+	for _, clientRelation := range relationTuples {
+		_, err := writeClient.Write.DeleteRelationTuple(ketoWrite.
+			NewDeleteRelationTupleParams().
+			WithNamespace(*clientRelation.Namespace).
+			WithObject(*clientRelation.Object).
+			WithRelation(*clientRelation.Relation).
+			WithSubjectID(&clientRelation.SubjectID))
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func queryRelationTuples(namespace *string, object *string, subjectID *string, role *string) ([]*ketoModels.InternalRelationTuple, error) {
+	relationTuplesQuery := ketoRead.NewGetRelationTuplesParams()
+	if namespace != nil {
+		relationTuplesQuery = relationTuplesQuery.WithNamespace(*namespace)
+	}
+	if object != nil {
+		relationTuplesQuery = relationTuplesQuery.WithObject(object)
+	}
+	if subjectID != nil {
+		relationTuplesQuery = relationTuplesQuery.WithSubjectID(subjectID)
+	}
+	if role != nil {
+		relationTuplesQuery = relationTuplesQuery.WithRelation(role)
+	}
+
+	return recursiveKetoPagingTraverse(relationTuplesQuery, make([]*ketoModels.InternalRelationTuple, 0), "")
+}
+
+func recursiveKetoPagingTraverse(queryParams *ketoRead.GetRelationTuplesParams, tuples []*ketoModels.InternalRelationTuple, pagingToken string) ([]*ketoModels.InternalRelationTuple, error) {
+	params := queryParams
+	if pagingToken != "" {
+		params = queryParams.WithPageToken(&pagingToken)
+	}
+
+	clientRelationsResponse, err := readClient.Read.GetRelationTuples(params)
+
+	if err != nil {
+		return nil, err
+	}
+
+	tuples = append(tuples, clientRelationsResponse.Payload.RelationTuples...)
+
+	if clientRelationsResponse.Payload.NextPageToken != "" {
+		return recursiveKetoPagingTraverse(queryParams, tuples, clientRelationsResponse.Payload.NextPageToken)
+	}
+
+	return tuples, nil
+}
+
+func getKetoClient(url string) *ketoClient.OryKeto {
+	return ketoClient.NewHTTPClientWithConfig(nil,
+		ketoClient.
+			DefaultTransportConfig().
+			WithSchemes([]string{"http"}).
+			WithHost(url))
+}

agent/pkg/routes/config_routes.go

@@ -1,15 +1,16 @@
 package routes
 
 import (
-	"github.com/gin-gonic/gin"
 	"mizuserver/pkg/controllers"
 	"mizuserver/pkg/middlewares"
+
+	"github.com/gin-gonic/gin"
 )
 
 func ConfigRoutes(ginApp *gin.Engine) {
 	routeGroup := ginApp.Group("/config")
 	routeGroup.Use(middlewares.RequiresAuth())
 
-	routeGroup.POST("/tapConfig", controllers.PostTapConfig)
-	routeGroup.GET("/tapConfig", controllers.GetTapConfig)
+	routeGroup.POST("/tap", middlewares.RequiresAdmin(), controllers.PostTapConfig)
+	routeGroup.GET("/tap", controllers.GetTapConfig)
 }

agent/pkg/routes/install_routes.go

@@ -10,4 +10,5 @@ func InstallRoutes(ginApp *gin.Engine) {
 	routeGroup := ginApp.Group("/install")
 
 	routeGroup.GET("/isNeeded", controllers.IsSetupNecessary)
+	routeGroup.POST("/admin", controllers.SetupAdminUser)
 }

agent/pkg/routes/oas_routes.go

@@ -0,0 +1,18 @@
+package routes
+
+import (
+	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/middlewares"
+
+	"github.com/gin-gonic/gin"
+)
+
+// OASRoutes methods to access OAS spec
+func OASRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/oas")
+	routeGroup.Use(middlewares.RequiresAuth())
+
+	routeGroup.GET("/", controllers.GetOASServers)     // list of servers in OAS map
+	routeGroup.GET("/all", controllers.GetOASAllSpecs) // list of servers in OAS map
+	routeGroup.GET("/:id", controllers.GetOASSpec)     // get OAS spec for given server
+}

agent/pkg/routes/service_map_routes.go

@@ -0,0 +1,19 @@
+package routes
+
+import (
+	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/middlewares"
+
+	"github.com/gin-gonic/gin"
+)
+
+func ServiceMapRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/servicemap")
+	routeGroup.Use(middlewares.RequiresAuth())
+
+	controller := controllers.NewServiceMapController()
+
+	routeGroup.GET("/status", controller.Status)
+	routeGroup.GET("/get", controller.Get)
+	routeGroup.GET("/reset", controller.Reset)
+}

agent/pkg/routes/status_routes.go

@@ -15,7 +15,7 @@ func StatusRoutes(ginApp *gin.Engine) {
 
 	routeGroup.POST("/tappedPods", controllers.PostTappedPods)
 	routeGroup.POST("/tapperStatus", controllers.PostTapperStatus)
-	routeGroup.GET("/tappersCount", controllers.GetTappersCount)
+	routeGroup.GET("/connectedTappersCount", controllers.GetConnectedTappersCount)
 	routeGroup.GET("/tap", controllers.GetTappingStatus)
 
 	routeGroup.GET("/auth", controllers.GetAuthStatus)

agent/pkg/servicemap/models.go

@@ -0,0 +1,32 @@
+package servicemap
+
+import (
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+type ServiceMapStatus struct {
+	Status                string `json:"status"`
+	EntriesProcessedCount int    `json:"entriesProcessedCount"`
+	NodeCount             int    `json:"nodeCount"`
+	EdgeCount             int    `json:"edgeCount"`
+}
+
+type ServiceMapResponse struct {
+	Status ServiceMapStatus `json:"status"`
+	Nodes  []ServiceMapNode `json:"nodes"`
+	Edges  []ServiceMapEdge `json:"edges"`
+}
+
+type ServiceMapNode struct {
+	Id    int         `json:"id"`
+	Name  string      `json:"name"`
+	Entry *tapApi.TCP `json:"entry"`
+	Count int         `json:"count"`
+}
+
+type ServiceMapEdge struct {
+	Source      ServiceMapNode   `json:"source"`
+	Destination ServiceMapNode   `json:"destination"`
+	Count       int              `json:"count"`
+	Protocol    *tapApi.Protocol `json:"protocol"`
+}

agent/pkg/servicemap/servicemap.go

@@ -0,0 +1,271 @@
+package servicemap
+
+import (
+	"sync"
+
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+const (
+	ServiceMapEnabled  = "enabled"
+	ServiceMapDisabled = "disabled"
+	UnresolvedNodeName = "unresolved"
+)
+
+var instance *serviceMap
+var once sync.Once
+
+func GetInstance() ServiceMap {
+	once.Do(func() {
+		instance = newServiceMap()
+		logger.Log.Debug("Service Map Initialized")
+	})
+	return instance
+}
+
+type serviceMap struct {
+	config           *shared.MizuAgentConfig
+	graph            *graph
+	entriesProcessed int
+}
+
+type ServiceMap interface {
+	SetConfig(config *shared.MizuAgentConfig)
+	IsEnabled() bool
+	NewTCPEntry(source *tapApi.TCP, destination *tapApi.TCP, protocol *tapApi.Protocol)
+	GetStatus() ServiceMapStatus
+	GetNodes() []ServiceMapNode
+	GetEdges() []ServiceMapEdge
+	GetEntriesProcessedCount() int
+	GetNodesCount() int
+	GetEdgesCount() int
+	Reset()
+}
+
+func newServiceMap() *serviceMap {
+	return &serviceMap{
+		config:           nil,
+		entriesProcessed: 0,
+		graph:            newDirectedGraph(),
+	}
+}
+
+type key string
+
+type entryData struct {
+	key   key
+	entry *tapApi.TCP
+}
+
+type nodeData struct {
+	id    int
+	entry *tapApi.TCP
+	count int
+}
+
+type edgeProtocol struct {
+	protocol *tapApi.Protocol
+	count    int
+}
+
+type edgeData struct {
+	data map[key]*edgeProtocol
+}
+
+type graph struct {
+	Nodes map[key]*nodeData
+	Edges map[key]map[key]*edgeData
+}
+
+func newDirectedGraph() *graph {
+	return &graph{
+		Nodes: make(map[key]*nodeData),
+		Edges: make(map[key]map[key]*edgeData),
+	}
+}
+
+func newNodeData(id int, e *tapApi.TCP) *nodeData {
+	return &nodeData{
+		id:    id,
+		entry: e,
+		count: 1,
+	}
+}
+
+func newEdgeData(p *tapApi.Protocol) *edgeData {
+	return &edgeData{
+		data: map[key]*edgeProtocol{
+			key(p.Name): {
+				protocol: p,
+				count:    1,
+			},
+		},
+	}
+}
+
+func (s *serviceMap) nodeExists(k key) (*nodeData, bool) {
+	n, ok := s.graph.Nodes[k]
+	return n, ok
+}
+
+func (s *serviceMap) addNode(k key, e *tapApi.TCP) (*nodeData, bool) {
+	nd, exists := s.nodeExists(k)
+	if !exists {
+		s.graph.Nodes[k] = newNodeData(len(s.graph.Nodes)+1, e)
+		return s.graph.Nodes[k], true
+	}
+	return nd, false
+}
+
+func (s *serviceMap) addEdge(u, v *entryData, p *tapApi.Protocol) {
+	if n, ok := s.addNode(u.key, u.entry); !ok {
+		n.count++
+	}
+	if n, ok := s.addNode(v.key, v.entry); !ok {
+		n.count++
+	}
+
+	if _, ok := s.graph.Edges[u.key]; !ok {
+		s.graph.Edges[u.key] = make(map[key]*edgeData)
+	}
+
+	// new edge u -> v pair
+	// protocol is the same for u and v
+	if e, ok := s.graph.Edges[u.key][v.key]; ok {
+		// edge data already exists for u -> v pair
+		// we have a new protocol for this u -> v pair
+
+		k := key(p.Name)
+		if pd, pOk := e.data[k]; pOk {
+			// protocol key already exists, just increment the count
+			pd.count++
+		} else {
+			// new protocol key
+			e.data[k] = &edgeProtocol{
+				protocol: p,
+				count:    1,
+			}
+		}
+	} else {
+		// new edge data for u -> v pair
+		s.graph.Edges[u.key][v.key] = newEdgeData(p)
+	}
+
+	s.entriesProcessed++
+}
+
+func (s *serviceMap) SetConfig(config *shared.MizuAgentConfig) {
+	s.config = config
+}
+
+func (s *serviceMap) IsEnabled() bool {
+	if s.config != nil && s.config.ServiceMap {
+		return true
+	}
+	return false
+}
+
+func (s *serviceMap) NewTCPEntry(src *tapApi.TCP, dst *tapApi.TCP, p *tapApi.Protocol) {
+	if !s.IsEnabled() {
+		return
+	}
+
+	srcEntry := &entryData{
+		key:   key(src.IP),
+		entry: src,
+	}
+	if len(srcEntry.entry.Name) == 0 {
+		srcEntry.entry.Name = UnresolvedNodeName
+	}
+
+	dstEntry := &entryData{
+		key:   key(dst.IP),
+		entry: dst,
+	}
+	if len(dstEntry.entry.Name) == 0 {
+		dstEntry.entry.Name = UnresolvedNodeName
+	}
+
+	s.addEdge(srcEntry, dstEntry, p)
+}
+
+func (s *serviceMap) GetStatus() ServiceMapStatus {
+	status := ServiceMapDisabled
+	if s.IsEnabled() {
+		status = ServiceMapEnabled
+	}
+
+	return ServiceMapStatus{
+		Status:                status,
+		EntriesProcessedCount: s.entriesProcessed,
+		NodeCount:             s.GetNodesCount(),
+		EdgeCount:             s.GetEdgesCount(),
+	}
+}
+
+func (s *serviceMap) GetNodes() []ServiceMapNode {
+	var nodes []ServiceMapNode
+	for i, n := range s.graph.Nodes {
+		nodes = append(nodes, ServiceMapNode{
+			Id:    n.id,
+			Name:  string(i),
+			Entry: n.entry,
+			Count: n.count,
+		})
+	}
+	return nodes
+}
+
+func (s *serviceMap) GetEdges() []ServiceMapEdge {
+	var edges []ServiceMapEdge
+	for u, m := range s.graph.Edges {
+		for v := range m {
+			for _, p := range s.graph.Edges[u][v].data {
+				edges = append(edges, ServiceMapEdge{
+					Source: ServiceMapNode{
+						Id:    s.graph.Nodes[u].id,
+						Name:  string(u),
+						Entry: s.graph.Nodes[u].entry,
+						Count: s.graph.Nodes[u].count,
+					},
+					Destination: ServiceMapNode{
+						Id:    s.graph.Nodes[v].id,
+						Name:  string(v),
+						Entry: s.graph.Nodes[v].entry,
+						Count: s.graph.Nodes[v].count,
+					},
+					Count:    p.count,
+					Protocol: p.protocol,
+				})
+			}
+		}
+	}
+	return edges
+}
+
+func (s *serviceMap) GetEntriesProcessedCount() int {
+	return s.entriesProcessed
+}
+
+func (s *serviceMap) GetNodesCount() int {
+	return len(s.graph.Nodes)
+}
+
+func (s *serviceMap) GetEdgesCount() int {
+	var count int
+	for u, m := range s.graph.Edges {
+		for v := range m {
+			for range s.graph.Edges[u][v].data {
+				count++
+			}
+		}
+	}
+	return count
+}
+
+func (s *serviceMap) Reset() {
+	s.entriesProcessed = 0
+	s.graph = newDirectedGraph()
+}

agent/pkg/servicemap/servicemap_test.go

@@ -0,0 +1,405 @@
+package servicemap
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/up9inc/mizu/shared"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+const (
+	a    = "aService"
+	b    = "bService"
+	c    = "cService"
+	d    = "dService"
+	Ip   = "127.0.0.1"
+	Port = "80"
+)
+
+var (
+	TCPEntryA = &tapApi.TCP{
+		Name: a,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, a),
+	}
+	TCPEntryB = &tapApi.TCP{
+		Name: b,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, b),
+	}
+	TCPEntryC = &tapApi.TCP{
+		Name: c,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, c),
+	}
+	TCPEntryD = &tapApi.TCP{
+		Name: d,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, d),
+	}
+	TCPEntryUnresolved = &tapApi.TCP{
+		Name: "",
+		Port: Port,
+		IP:   Ip,
+	}
+	TCPEntryUnresolved2 = &tapApi.TCP{
+		Name: "",
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, UnresolvedNodeName),
+	}
+	ProtocolHttp = &tapApi.Protocol{
+		Name:            "http",
+		LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
+		Abbreviation:    "HTTP",
+		Macro:           "http",
+		Version:         "1.1",
+		BackgroundColor: "#205cf5",
+		ForegroundColor: "#ffffff",
+		FontSize:        12,
+		ReferenceLink:   "https://datatracker.ietf.org/doc/html/rfc2616",
+		Ports:           []string{"80", "443", "8080"},
+		Priority:        0,
+	}
+	ProtocolRedis = &tapApi.Protocol{
+		Name:            "redis",
+		LongName:        "Redis Serialization Protocol",
+		Abbreviation:    "REDIS",
+		Macro:           "redis",
+		Version:         "3.x",
+		BackgroundColor: "#a41e11",
+		ForegroundColor: "#ffffff",
+		FontSize:        11,
+		ReferenceLink:   "https://redis.io/topics/protocol",
+		Ports:           []string{"6379"},
+		Priority:        3,
+	}
+)
+
+type ServiceMapDisabledSuite struct {
+	suite.Suite
+
+	instance ServiceMap
+}
+
+type ServiceMapEnabledSuite struct {
+	suite.Suite
+
+	instance ServiceMap
+}
+
+func (s *ServiceMapDisabledSuite) SetupTest() {
+	s.instance = GetInstance()
+}
+
+func (s *ServiceMapEnabledSuite) SetupTest() {
+	s.instance = GetInstance()
+	s.instance.SetConfig(&shared.MizuAgentConfig{
+		ServiceMap: true,
+	})
+}
+
+func (s *ServiceMapDisabledSuite) TestServiceMapInstance() {
+	assert := s.Assert()
+
+	assert.NotNil(s.instance)
+}
+
+func (s *ServiceMapDisabledSuite) TestServiceMapSingletonInstance() {
+	assert := s.Assert()
+
+	instance2 := GetInstance()
+
+	assert.NotNil(s.instance)
+	assert.NotNil(instance2)
+	assert.Equal(s.instance, instance2)
+}
+
+func (s *ServiceMapDisabledSuite) TestServiceMapIsEnabledShouldReturnFalseByDefault() {
+	assert := s.Assert()
+
+	enabled := s.instance.IsEnabled()
+
+	assert.False(enabled)
+}
+
+func (s *ServiceMapDisabledSuite) TestGetStatusShouldReturnDisabledByDefault() {
+	assert := s.Assert()
+
+	status := s.instance.GetStatus()
+
+	assert.Equal("disabled", status.Status)
+	assert.Equal(0, status.EntriesProcessedCount)
+	assert.Equal(0, status.NodeCount)
+	assert.Equal(0, status.EdgeCount)
+}
+
+func (s *ServiceMapDisabledSuite) TestNewTCPEntryShouldDoNothingWhenDisabled() {
+	assert := s.Assert()
+
+	s.instance.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
+	s.instance.NewTCPEntry(TCPEntryC, TCPEntryD, ProtocolHttp)
+	status := s.instance.GetStatus()
+
+	assert.Equal("disabled", status.Status)
+	assert.Equal(0, status.EntriesProcessedCount)
+	assert.Equal(0, status.NodeCount)
+	assert.Equal(0, status.EdgeCount)
+}
+
+// Enabled
+
+func (s *ServiceMapEnabledSuite) TestServiceMapIsEnabled() {
+	assert := s.Assert()
+
+	enabled := s.instance.IsEnabled()
+
+	assert.True(enabled)
+}
+
+func (s *ServiceMapEnabledSuite) TestServiceMap() {
+	assert := s.Assert()
+
+	// A -> B - HTTP
+	s.instance.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
+
+	nodes := s.instance.GetNodes()
+	edges := s.instance.GetEdges()
+
+	// Counts for the first entry
+	assert.Equal(1, s.instance.GetEntriesProcessedCount())
+	assert.Equal(2, s.instance.GetNodesCount())
+	assert.Equal(2, len(nodes))
+	assert.Equal(1, s.instance.GetEdgesCount())
+	assert.Equal(1, len(edges))
+	//http protocol
+	assert.Equal(1, edges[0].Count)
+	assert.Equal(ProtocolHttp.Name, edges[0].Protocol.Name)
+
+	// same A -> B - HTTP, http protocol count should be 2, edges count should be 1
+	s.instance.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
+
+	nodes = s.instance.GetNodes()
+	edges = s.instance.GetEdges()
+
+	// Counts for a second entry
+	assert.Equal(2, s.instance.GetEntriesProcessedCount())
+	assert.Equal(2, s.instance.GetNodesCount())
+	assert.Equal(2, len(nodes))
+	// edges count should still be 1, but http protocol count should be 2
+	assert.Equal(1, s.instance.GetEdgesCount())
+	assert.Equal(1, len(edges))
+	// http protocol
+	assert.Equal(2, edges[0].Count) //http
+	assert.Equal(ProtocolHttp.Name, edges[0].Protocol.Name)
+
+	// same A -> B - REDIS, http protocol count should be 2 and redis protocol count should 1, edges count should be 2
+	s.instance.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolRedis)
+
+	nodes = s.instance.GetNodes()
+	edges = s.instance.GetEdges()
+
+	// Counts after second entry
+	assert.Equal(3, s.instance.GetEntriesProcessedCount())
+	assert.Equal(2, s.instance.GetNodesCount())
+	assert.Equal(2, len(nodes))
+	// edges count should be 2, http protocol count should be 2 and redis protocol should be 1
+	assert.Equal(2, s.instance.GetEdgesCount())
+	assert.Equal(2, len(edges))
+	// http and redis protocols
+	httpIndex := -1
+	redisIndex := -1
+	for i, e := range edges {
+		if e.Protocol.Name == ProtocolHttp.Name {
+			httpIndex = i
+			continue
+		}
+		if e.Protocol.Name == ProtocolRedis.Name {
+			redisIndex = i
+		}
+	}
+	assert.NotEqual(-1, httpIndex)
+	assert.NotEqual(-1, redisIndex)
+	// http protocol
+	assert.Equal(2, edges[httpIndex].Count)
+	assert.Equal(ProtocolHttp.Name, edges[httpIndex].Protocol.Name)
+	// redis protocol
+	assert.Equal(1, edges[redisIndex].Count)
+	assert.Equal(ProtocolRedis.Name, edges[redisIndex].Protocol.Name)
+
+	// other entries
+	s.instance.NewTCPEntry(TCPEntryUnresolved, TCPEntryA, ProtocolHttp)
+	s.instance.NewTCPEntry(TCPEntryB, TCPEntryUnresolved2, ProtocolHttp)
+	s.instance.NewTCPEntry(TCPEntryC, TCPEntryD, ProtocolHttp)
+	s.instance.NewTCPEntry(TCPEntryA, TCPEntryC, ProtocolHttp)
+
+	status := s.instance.GetStatus()
+	nodes = s.instance.GetNodes()
+	edges = s.instance.GetEdges()
+	expectedEntriesProcessedCount := 7
+	expectedNodeCount := 6
+	expectedEdgeCount := 6
+
+	// Counts after all entries
+	assert.Equal(expectedEntriesProcessedCount, s.instance.GetEntriesProcessedCount())
+	assert.Equal(expectedNodeCount, s.instance.GetNodesCount())
+	assert.Equal(expectedNodeCount, len(nodes))
+	assert.Equal(expectedEdgeCount, s.instance.GetEdgesCount())
+	assert.Equal(expectedEdgeCount, len(edges))
+
+	// Status
+	assert.Equal("enabled", status.Status)
+	assert.Equal(expectedEntriesProcessedCount, status.EntriesProcessedCount)
+	assert.Equal(expectedNodeCount, status.NodeCount)
+	assert.Equal(expectedEdgeCount, status.EdgeCount)
+
+	// Nodes
+	aNode := -1
+	bNode := -1
+	cNode := -1
+	dNode := -1
+	unresolvedNode := -1
+	unresolvedNode2 := -1
+	var validateNode = func(node ServiceMapNode, entryName string, count int) int {
+		// id
+		assert.GreaterOrEqual(node.Id, 1)
+		assert.LessOrEqual(node.Id, expectedNodeCount)
+
+		// entry
+		// node.Name is the key of the node, key = entry.IP
+		// entry.Name is the name of the service and could be unresolved
+		assert.Equal(node.Name, node.Entry.IP)
+		assert.Equal(Port, node.Entry.Port)
+		assert.Equal(entryName, node.Entry.Name)
+
+		// count
+		assert.Equal(count, node.Count)
+
+		return node.Id
+	}
+
+	for _, v := range nodes {
+		if strings.HasSuffix(v.Name, a) {
+			aNode = validateNode(v, a, 5)
+			continue
+		}
+		if strings.HasSuffix(v.Name, b) {
+			bNode = validateNode(v, b, 4)
+			continue
+		}
+		if strings.HasSuffix(v.Name, c) {
+			cNode = validateNode(v, c, 2)
+			continue
+		}
+		if strings.HasSuffix(v.Name, d) {
+			dNode = validateNode(v, d, 1)
+			continue
+		}
+		if v.Name == Ip {
+			unresolvedNode = validateNode(v, UnresolvedNodeName, 1)
+			continue
+		}
+		if strings.HasSuffix(v.Name, UnresolvedNodeName) {
+			unresolvedNode2 = validateNode(v, UnresolvedNodeName, 1)
+			continue
+		}
+	}
+
+	// Make sure we found all the nodes
+	nodeIds := [...]int{aNode, bNode, cNode, dNode, unresolvedNode, unresolvedNode2}
+	for _, v := range nodeIds {
+		assert.NotEqual(-1, v)
+	}
+
+	// Edges
+	abEdge := -1
+	uaEdge := -1
+	buEdge := -1
+	cdEdge := -1
+	acEdge := -1
+	var validateEdge = func(edge ServiceMapEdge, sourceEntryName string, destEntryName string, protocolName string, protocolCount int) {
+		// source
+		assert.Contains(nodeIds, edge.Source.Id)
+		assert.LessOrEqual(edge.Source.Id, expectedNodeCount)
+		assert.Equal(edge.Source.Name, edge.Source.Entry.IP)
+		assert.Equal(sourceEntryName, edge.Source.Entry.Name)
+
+		// destination
+		assert.Contains(nodeIds, edge.Destination.Id)
+		assert.LessOrEqual(edge.Destination.Id, expectedNodeCount)
+		assert.Equal(edge.Destination.Name, edge.Destination.Entry.IP)
+		assert.Equal(destEntryName, edge.Destination.Entry.Name)
+
+		// protocol
+		assert.Equal(protocolName, edge.Protocol.Name)
+		assert.Equal(protocolCount, edge.Count)
+	}
+
+	for i, v := range edges {
+		if v.Source.Entry.Name == a && v.Destination.Entry.Name == b && v.Protocol.Name == "http" {
+			validateEdge(v, a, b, ProtocolHttp.Name, 2)
+			abEdge = i
+			continue
+		}
+		if v.Source.Entry.Name == a && v.Destination.Entry.Name == b && v.Protocol.Name == "redis" {
+			validateEdge(v, a, b, ProtocolRedis.Name, 1)
+			abEdge = i
+			continue
+		}
+		if v.Source.Entry.Name == UnresolvedNodeName && v.Destination.Entry.Name == a {
+			validateEdge(v, UnresolvedNodeName, a, ProtocolHttp.Name, 1)
+			uaEdge = i
+			continue
+		}
+		if v.Source.Entry.Name == b && v.Destination.Entry.Name == UnresolvedNodeName {
+			validateEdge(v, b, UnresolvedNodeName, ProtocolHttp.Name, 1)
+			buEdge = i
+			continue
+		}
+		if v.Source.Entry.Name == c && v.Destination.Entry.Name == d {
+			validateEdge(v, c, d, ProtocolHttp.Name, 1)
+			cdEdge = i
+			continue
+		}
+		if v.Source.Entry.Name == a && v.Destination.Entry.Name == c {
+			validateEdge(v, a, c, ProtocolHttp.Name, 1)
+			acEdge = i
+			continue
+		}
+	}
+
+	// Make sure we found all the edges
+	for _, v := range [...]int{abEdge, uaEdge, buEdge, cdEdge, acEdge} {
+		assert.NotEqual(-1, v)
+	}
+
+	// Reset
+	s.instance.Reset()
+	status = s.instance.GetStatus()
+	nodes = s.instance.GetNodes()
+	edges = s.instance.GetEdges()
+
+	// Counts after reset
+	assert.Equal(0, s.instance.GetEntriesProcessedCount())
+	assert.Equal(0, s.instance.GetNodesCount())
+	assert.Equal(0, s.instance.GetEdgesCount())
+
+	// Status after reset
+	assert.Equal("enabled", status.Status)
+	assert.Equal(0, status.EntriesProcessedCount)
+	assert.Equal(0, status.NodeCount)
+	assert.Equal(0, status.EdgeCount)
+
+	// Nodes after reset
+	assert.Equal([]ServiceMapNode(nil), nodes)
+
+	// Edges after reset
+	assert.Equal([]ServiceMapEdge(nil), edges)
+}
+
+func TestServiceMapSuite(t *testing.T) {
+	suite.Run(t, new(ServiceMapDisabledSuite))
+	suite.Run(t, new(ServiceMapEnabledSuite))
+}

agent/pkg/up9/main.go

@@ -243,7 +243,7 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 			var dataMap map[string]interface{}
 			err = json.Unmarshal(dataBytes, &dataMap)
 
-			var entry tapApi.MizuEntry
+			var entry tapApi.Entry
 			if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil {
 				continue
 			}

agent/pkg/utils/utils.go

@@ -2,13 +2,13 @@ package utils
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/providers"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
 	"os/signal"
-	"strings"
 	"syscall"
 	"time"
 
@@ -45,16 +45,6 @@ func StartServer(app *gin.Engine) {
 	}
 }
 
-func GetTappedPodsStatus() []shared.TappedPodStatus {
-	tappedPodsStatus := make([]shared.TappedPodStatus, 0)
-	for _, pod := range providers.TapStatus.Pods {
-		status := strings.ToLower(providers.TappersStatus[pod.NodeName].Status)
-		isTapped := status == "running"
-		tappedPodsStatus = append(tappedPodsStatus, shared.TappedPodStatus{Name: pod.Name, Namespace: pod.Namespace, IsTapped: isTapped})
-	}
-	return tappedPodsStatus
-}
-
 func CheckErr(e error) {
 	if e != nil {
 		logger.Log.Errorf("%v", e)
@@ -71,3 +61,42 @@ func SetHostname(address, newHostname string) string {
 	return replacedUrl.String()
 
 }
+
+func ReadJsonFile(filePath string, value interface{}) error {
+	if content, err := ioutil.ReadFile(filePath); err != nil {
+		return err
+	} else {
+		if err = json.Unmarshal(content, value); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func SaveJsonFile(filePath string, value interface{}) error {
+	if data, err := json.Marshal(value); err != nil {
+		return err
+	} else {
+		if err = ioutil.WriteFile(filePath, data, 0644); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func UniqueStringSlice(s []string) []string {
+	uniqueSlice := make([]string, 0)
+	uniqueMap := map[string]bool{}
+
+	for _, val := range s {
+		if uniqueMap[val] == true {
+			continue
+		}
+		uniqueMap[val] = true
+		uniqueSlice = append(uniqueSlice, val)
+	}
+
+	return uniqueSlice
+}

cli/Makefile

@@ -14,8 +14,12 @@ help: ## This help.
 install:
 	go install mizu.go
 
+build-debug:
+	export GCLFAGS='-gcflags="all=-N -l"'
+	${MAKE} build
+
 build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
-	go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
+	go build ${GCLFAGS} -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.Platform=$(SUFFIX)' \

cli/README.md.TEMPLATE

@@ -1,6 +1,7 @@
 # Mizu release _SEM_VER_
+Full changelog for stable release see in [docs](https://github.com/up9inc/mizu/blob/main/docs/CHANGELOG.md)
 
-Download Mizu for your platform
+## Download Mizu for your platform
 
 **Mac** (Intel)
 ```

cli/apiserver/provider.go

@@ -36,6 +36,16 @@ func NewProvider(url string, retries int, timeout time.Duration) *Provider {
 	}
 }
 
+func NewProviderWithoutRetries(url string, timeout time.Duration) *Provider {
+	return &Provider{
+		url:     url,
+		retries: 1,
+		client: &http.Client{
+			Timeout: timeout,
+		},
+	}
+}
+
 func (provider *Provider) TestConnection() error {
 	retriesLeft := provider.retries
 	for retriesLeft > 0 {
@@ -87,9 +97,8 @@ func (provider *Provider) ReportTappedPods(pods []core.Pod) error {
 	tappedPodsUrl := fmt.Sprintf("%s/status/tappedPods", provider.url)
 
 	podInfos := kubernetes.GetPodInfosForPods(pods)
-	tapStatus := shared.TapStatus{Pods: podInfos}
 
-	if jsonValue, err := json.Marshal(tapStatus); err != nil {
+	if jsonValue, err := json.Marshal(podInfos); err != nil {
 		return fmt.Errorf("failed Marshal the tapped pods %w", err)
 	} else {
 		if response, err := provider.client.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {

cli/cmd/check.go

@@ -0,0 +1,20 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/telemetry"
+)
+
+var checkCmd = &cobra.Command{
+	Use:   "check",
+	Short: "Check the Mizu installation for potential problems",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		go telemetry.ReportRun("check", nil)
+		runMizuCheck()
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(checkCmd)
+}

cli/cmd/checkRunner.go

@@ -0,0 +1,186 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/shared/semver"
+	"net/http"
+)
+
+func runMizuCheck() {
+	logger.Log.Infof("Mizu install checks\n===================")
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel() // cancel will be called when this function exits
+
+	kubernetesProvider, kubernetesVersion, checkPassed := checkKubernetesApi()
+
+	if checkPassed {
+		checkPassed = checkKubernetesVersion(kubernetesVersion)
+	}
+
+	var isInstallCommand bool
+	if checkPassed {
+		checkPassed, isInstallCommand = checkMizuMode(ctx, kubernetesProvider)
+	}
+
+	if checkPassed {
+		checkPassed = checkAllResourcesExist(ctx, kubernetesProvider, isInstallCommand)
+	}
+
+	if checkPassed {
+		checkPassed = checkServerConnection(kubernetesProvider, cancel)
+	}
+
+	if checkPassed {
+		logger.Log.Infof("\nStatus check results are %v", fmt.Sprintf(uiUtils.Green, "√"))
+	} else {
+		logger.Log.Errorf("\nStatus check results are %v", fmt.Sprintf(uiUtils.Red, "✗"))
+	}
+}
+
+func checkKubernetesApi() (*kubernetes.Provider, *semver.SemVersion, bool) {
+	logger.Log.Infof("\nkubernetes-api\n--------------------")
+
+	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
+	if err != nil {
+		logger.Log.Errorf("%v can't initialize the client, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+		return nil, nil, false
+	}
+	logger.Log.Infof("%v can initialize the client", fmt.Sprintf(uiUtils.Green, "√"))
+
+	kubernetesVersion, err := kubernetesProvider.GetKubernetesVersion()
+	if err != nil {
+		logger.Log.Errorf("%v can't query the Kubernetes API, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+		return nil, nil, false
+	}
+	logger.Log.Infof("%v can query the Kubernetes API", fmt.Sprintf(uiUtils.Green, "√"))
+
+	return kubernetesProvider, kubernetesVersion, true
+}
+
+func checkMizuMode(ctx context.Context, kubernetesProvider *kubernetes.Provider) (bool, bool) {
+	logger.Log.Infof("\nmizu-mode\n--------------------")
+
+	if exist, err := kubernetesProvider.DoesDeploymentExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
+		logger.Log.Errorf("%v can't check mizu command, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+		return false, false
+	} else if exist {
+		logger.Log.Infof("%v mizu running with install command", fmt.Sprintf(uiUtils.Green, "√"))
+		return true, true
+	} else {
+		logger.Log.Infof("%v mizu running with tap command", fmt.Sprintf(uiUtils.Green, "√"))
+		return true, false
+	}
+}
+
+func checkKubernetesVersion(kubernetesVersion *semver.SemVersion) bool {
+	logger.Log.Infof("\nkubernetes-version\n--------------------")
+
+	if err := kubernetes.ValidateKubernetesVersion(kubernetesVersion); err != nil {
+		logger.Log.Errorf("%v not running the minimum Kubernetes API version, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+		return false
+	}
+
+	logger.Log.Infof("%v is running the minimum Kubernetes API version", fmt.Sprintf(uiUtils.Green, "√"))
+	return true
+}
+
+func checkServerConnection(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) bool {
+	logger.Log.Infof("\nmizu-connectivity\n--------------------")
+
+	serverUrl := GetApiServerUrl()
+
+	if response, err := http.Get(fmt.Sprintf("%s/", serverUrl)); err != nil || response.StatusCode != 200 {
+		startProxyReportErrorIfAny(kubernetesProvider, cancel)
+	}
+
+	apiServerProvider := apiserver.NewProvider(serverUrl, apiserver.DefaultRetries, apiserver.DefaultTimeout)
+	if err := apiServerProvider.TestConnection(); err != nil {
+		logger.Log.Errorf("%v couldn't connect to API server, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+		return false
+	}
+
+	logger.Log.Infof("%v connected successfully to API server", fmt.Sprintf(uiUtils.Green, "√"))
+	return true
+}
+
+func checkAllResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.Provider, isInstallCommand bool) bool {
+	logger.Log.Infof("\nmizu-existence\n--------------------")
+
+	exist, err := kubernetesProvider.DoesNamespaceExist(ctx, config.Config.MizuResourcesNamespace)
+	allResourcesExist := checkResourceExist(config.Config.MizuResourcesNamespace, "namespace", exist, err)
+
+	exist, err = kubernetesProvider.DoesConfigMapExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ConfigMapName)
+	allResourcesExist = checkResourceExist(kubernetes.ConfigMapName, "config map", exist, err)
+
+	exist, err = kubernetesProvider.DoesServiceAccountExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName)
+	allResourcesExist = checkResourceExist(kubernetes.ServiceAccountName, "service account", exist, err)
+
+	if config.Config.IsNsRestrictedMode() {
+		exist, err = kubernetesProvider.DoesRoleExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleName)
+		allResourcesExist = checkResourceExist(kubernetes.RoleName, "role", exist, err)
+
+		exist, err = kubernetesProvider.DoesRoleBindingExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleBindingName)
+		allResourcesExist = checkResourceExist(kubernetes.RoleBindingName, "role binding", exist, err)
+	} else {
+		exist, err = kubernetesProvider.DoesClusterRoleExist(ctx, kubernetes.ClusterRoleName)
+		allResourcesExist = checkResourceExist(kubernetes.ClusterRoleName, "cluster role", exist, err)
+
+		exist, err = kubernetesProvider.DoesClusterRoleBindingExist(ctx, kubernetes.ClusterRoleBindingName)
+		allResourcesExist = checkResourceExist(kubernetes.ClusterRoleBindingName, "cluster role binding", exist, err)
+	}
+
+	exist, err = kubernetesProvider.DoesServiceExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
+	allResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "service", exist, err)
+
+	if isInstallCommand {
+		allResourcesExist = checkInstallResourcesExist(ctx, kubernetesProvider)
+	} else {
+		allResourcesExist = checkTapResourcesExist(ctx, kubernetesProvider)
+	}
+
+	return allResourcesExist
+}
+
+func checkInstallResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
+	exist, err := kubernetesProvider.DoesRoleExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleName)
+	installResourcesExist := checkResourceExist(kubernetes.DaemonRoleName, "role", exist, err)
+
+	exist, err = kubernetesProvider.DoesRoleBindingExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleBindingName)
+	installResourcesExist = checkResourceExist(kubernetes.DaemonRoleBindingName, "role binding", exist, err)
+
+	exist, err = kubernetesProvider.DoesPersistentVolumeClaimExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName)
+	installResourcesExist = checkResourceExist(kubernetes.PersistentVolumeClaimName, "persistent volume claim", exist, err)
+
+	exist, err = kubernetesProvider.DoesDeploymentExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
+	installResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "deployment", exist, err)
+
+	return installResourcesExist
+}
+
+func checkTapResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
+	exist, err := kubernetesProvider.DoesPodExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
+	tapResourcesExist := checkResourceExist(kubernetes.ApiServerPodName, "pod", exist, err)
+
+	return tapResourcesExist
+}
+
+func checkResourceExist(resourceName string, resourceType string, exist bool, err error) bool {
+	if err != nil {
+		logger.Log.Errorf("%v error checking if '%v' %v exists, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), resourceName, resourceType, err)
+		return false
+	} else if !exist {
+		logger.Log.Errorf("%v '%v' %v doesn't exist", fmt.Sprintf(uiUtils.Red, "✗"), resourceName, resourceType)
+		return false
+	} else {
+		logger.Log.Infof("%v '%v' %v exists", fmt.Sprintf(uiUtils.Green, "√"), resourceName, resourceType)
+	}
+
+	return true
+}

cli/cmd/cleanRunner.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 )
 
@@ -11,5 +10,5 @@ func performCleanCommand() {
 		return
 	}
 
-	finishMizuExecution(kubernetesProvider, apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout), config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
+	finishMizuExecution(kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
 }

cli/cmd/common.go

@@ -6,18 +6,17 @@ import (
 	"errors"
 	"fmt"
 	"github.com/up9inc/mizu/cli/apiserver"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/resources"
-	"github.com/up9inc/mizu/cli/telemetry"
+	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 	"path"
 	"time"
 
 	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/errormessage"
-	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared/kubernetes"
 	"github.com/up9inc/mizu/shared/logger"
 )
@@ -27,14 +26,35 @@ func GetApiServerUrl() string {
 }
 
 func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
-	err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.ProxyHost, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
+	httpServer, err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.ProxyHost, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName, cancel)
 	if err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+
 			"Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName))
 		cancel()
+		return
 	}
 
-	logger.Log.Debugf("proxy ended")
+	apiProvider = apiserver.NewProviderWithoutRetries(GetApiServerUrl(), time.Second) // short check for proxy
+	if err := apiProvider.TestConnection(); err != nil {
+		logger.Log.Debugf("Couldn't connect using proxy, stopping proxy and trying to create port-forward")
+		if err := httpServer.Shutdown(context.Background()); err != nil {
+			logger.Log.Debugf("Error occurred while stopping proxy %v", errormessage.FormatError(err))
+		}
+
+		if err := kubernetes.NewPortForward(kubernetesProvider, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName, config.Config.Tap.GuiPort, cancel); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running port forward %v\n"+
+				"Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName))
+			cancel()
+			return
+		}
+
+		apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout) // long check for port-forward
+		if err := apiProvider.TestConnection(); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
+			cancel()
+			return
+		}
+	}
 }
 
 func getKubernetesProviderForCli() (*kubernetes.Provider, error) {
@@ -43,6 +63,23 @@ func getKubernetesProviderForCli() (*kubernetes.Provider, error) {
 		handleKubernetesProviderError(err)
 		return nil, err
 	}
+
+	if err := kubernetesProvider.ValidateNotProxy(); err != nil {
+		handleKubernetesProviderError(err)
+		return nil, err
+	}
+
+	kubernetesVersion, err := kubernetesProvider.GetKubernetesVersion()
+	if err != nil {
+		handleKubernetesProviderError(err)
+		return nil, err
+	}
+
+	if err := kubernetes.ValidateKubernetesVersion(kubernetesVersion); err != nil {
+		handleKubernetesProviderError(err)
+		return nil, err
+	}
+
 	return kubernetesProvider, nil
 }
 
@@ -55,8 +92,7 @@ func handleKubernetesProviderError(err error) {
 	}
 }
 
-func finishMizuExecution(kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string) {
-	telemetry.ReportAPICalls(apiProvider)
+func finishMizuExecution(kubernetesProvider *kubernetes.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string) {
 	removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
 	defer cancel()
 	dumpLogsIfNeeded(removalCtx, kubernetesProvider)

cli/cmd/installRunner.go

@@ -46,7 +46,8 @@ func runMizuInstall() {
 
 	if err = resources.CreateInstallMizuResources(ctx, kubernetesProvider, serializedValidationRules,
 		serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(),
-		config.Config.MizuResourcesNamespace, config.Config.AgentImage,
+		config.Config.MizuResourcesNamespace, config.Config.AgentImage, config.Config.BasenineImage,
+		config.Config.KratosImage, config.Config.KetoImage,
 		nil, defaultMaxEntriesDBSizeBytes, defaultResources, config.Config.ImagePullPolicy(),
 		config.Config.LogLevel(), false); err != nil {
 		var statusError *k8serrors.StatusError
@@ -89,6 +90,8 @@ func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Reso
 		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
 		AgentDatabasePath:      shared.DataDirPath,
 		StandaloneMode:         true,
+		ServiceMap:             config.Config.ServiceMap,
+		OAS:                    config.Config.OAS,
 	}
 
 	return &mizuAgentConfig
@@ -99,7 +102,8 @@ func watchApiServerPodReady(ctx context.Context, kubernetesProvider *kubernetes.
 	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
 	eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper)
 
-	timeAfter := time.After(30 * time.Second)
+	apiServerTimeoutSec := config.GetIntEnvConfig(config.ApiServerTimeoutSec, 120)
+	timeAfter := time.After(time.Duration(apiServerTimeoutSec) * time.Second)
 	for {
 		select {
 		case wEvent, ok := <-eventChan:

cli/cmd/tap.go

@@ -12,7 +12,6 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
-	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
@@ -26,7 +25,6 @@ var tapCmd = &cobra.Command{
 	Long: `Record the ingoing traffic of a kubernetes pod.
 Supported protocols are HTTP and gRPC.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("tap", config.Config.Tap)
 		RunMizuTap()
 		return nil
 	},

cli/cmd/tapRunner.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/up9inc/mizu/cli/resources"
+	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/utils"
 
 	"github.com/getkin/kin-openapi/openapi3"
@@ -23,7 +24,6 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
-	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/kubernetes"
@@ -124,7 +124,7 @@ func RunMizuTap() {
 	}
 
 	logger.Log.Infof("Waiting for Mizu Agent to start...")
-	if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel()); err != nil {
+	if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, config.Config.BasenineImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel()); err != nil {
 		var statusError *k8serrors.StatusError
 		if errors.As(err, &statusError) {
 			if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists {
@@ -138,7 +138,7 @@ func RunMizuTap() {
 		return
 	}
 
-	defer finishMizuExecution(kubernetesProvider, apiProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
+	defer finishTapExecution(kubernetesProvider)
 
 	go goUtils.HandleExcWrapper(watchApiServerEvents, ctx, kubernetesProvider, cancel)
 	go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel)
@@ -147,6 +147,12 @@ func RunMizuTap() {
 	utils.WaitForFinish(ctx, cancel)
 }
 
+func finishTapExecution(kubernetesProvider *kubernetes.Provider) {
+	telemetry.ReportTapTelemetry(apiProvider, config.Config.Tap, state.startTime)
+
+	finishMizuExecution(kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
+}
+
 func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 	mizuAgentConfig := shared.MizuAgentConfig{
 		MaxDBSizeBytes:         config.Config.Tap.MaxEntriesDBSizeBytes(),
@@ -156,6 +162,8 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 		TapperResources:        config.Config.Tap.TapperResources,
 		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
 		AgentDatabasePath:      shared.DataDirPath,
+		ServiceMap:             config.Config.ServiceMap,
+		OAS:                    config.Config.OAS,
 	}
 
 	return &mizuAgentConfig
@@ -304,7 +312,9 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
 	eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper)
 	isPodReady := false
-	timeAfter := time.After(25 * time.Second)
+
+	apiServerTimeoutSec := config.GetIntEnvConfig(config.ApiServerTimeoutSec, 120)
+	timeAfter := time.After(time.Duration(apiServerTimeoutSec) * time.Second)
 	for {
 		select {
 		case wEvent, ok := <-eventChan:
@@ -412,20 +422,15 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
 }
 
 func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, err error) {
-	go startProxyReportErrorIfAny(kubernetesProvider, cancel)
+	startProxyReportErrorIfAny(kubernetesProvider, cancel)
 
-	url := GetApiServerUrl()
-	if err := apiProvider.TestConnection(); err != nil {
-		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
-		cancel()
-		return
-	}
 	options, _ := getMizuApiFilteringOptions()
 	if err = startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", err))
 		cancel()
 	}
 
+	url := GetApiServerUrl()
 	logger.Log.Infof("Mizu is available at %s", url)
 	if !config.Config.HeadlessMode {
 		uiUtils.OpenBrowser(url)

cli/cmd/viewRunner.go

@@ -27,7 +27,7 @@ func runMizuView() {
 	url := config.Config.View.Url
 
 	if url == "" {
-		exists, err := kubernetesProvider.DoesServicesExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
+		exists, err := kubernetesProvider.DoesServiceExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
 		if err != nil {
 			logger.Log.Errorf("Failed to found mizu service %v", err)
 			cancel()
@@ -47,8 +47,7 @@ func runMizuView() {
 			return
 		}
 		logger.Log.Infof("Establishing connection to k8s cluster...")
-		go startProxyReportErrorIfAny(kubernetesProvider, cancel)
-
+		startProxyReportErrorIfAny(kubernetesProvider, cancel)
 	}
 
 	apiServerProvider := apiserver.NewProvider(url, apiserver.DefaultRetries, apiserver.DefaultTimeout)

cli/config/configStruct.go

@@ -2,14 +2,16 @@ package config
 
 import (
 	"fmt"
-	"github.com/op/go-logging"
-	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/mizu"
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/client-go/util/homedir"
 	"os"
 	"path"
 	"path/filepath"
+
+	"github.com/op/go-logging"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/shared"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/util/homedir"
 )
 
 const (
@@ -26,6 +28,9 @@ type ConfigStruct struct {
 	Auth                   configStructs.AuthConfig    `yaml:"auth"`
 	Config                 configStructs.ConfigConfig  `yaml:"config,omitempty"`
 	AgentImage             string                      `yaml:"agent-image,omitempty" readonly:""`
+	BasenineImage          string                      `yaml:"basenine-image,omitempty" readonly:""`
+	KratosImage            string                      `yaml:"kratos-image,omitempty" readonly:""`
+	KetoImage              string                      `yaml:"keto-image,omitempty" readonly:""`
 	ImagePullPolicyStr     string                      `yaml:"image-pull-policy" default:"Always"`
 	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
 	Telemetry              bool                        `yaml:"telemetry" default:"true"`
@@ -34,9 +39,11 @@ type ConfigStruct struct {
 	ConfigFilePath         string                      `yaml:"config-path,omitempty" readonly:""`
 	HeadlessMode           bool                        `yaml:"headless" default:"false"`
 	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
+	ServiceMap             bool                        `yaml:"service-map,omitempty" default:"false" readonly:""`
+	OAS                    bool                        `yaml:"oas,omitempty" default:"false" readonly:""`
 }
 
-func(config *ConfigStruct) validate() error {
+func (config *ConfigStruct) validate() error {
 	if _, err := logging.LogLevel(config.LogLevelStr); err != nil {
 		return fmt.Errorf("%s is not a valid log level, err: %v", config.LogLevelStr, err)
 	}
@@ -45,6 +52,9 @@ func(config *ConfigStruct) validate() error {
 }
 
 func (config *ConfigStruct) SetDefaults() {
+	config.BasenineImage = fmt.Sprintf("%s:%s", shared.BasenineImageRepo, shared.BasenineImageTag)
+	config.KratosImage = shared.KratosImageDefault
+	config.KetoImage = shared.KetoImageDefault
 	config.AgentImage = fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer)
 	config.ConfigFilePath = path.Join(mizu.GetMizuFolderPath(), "config.yaml")
 }

cli/config/envConfig.go

@@ -7,6 +7,7 @@ import (
 
 const (
 	ApiServerRetries = "API_SERVER_RETRIES"
+	ApiServerTimeoutSec = "API_SERVER_TIMEOUT_SEC"
 )
 
 func GetIntEnvConfig(key string, defaultValue int) int {

cli/go.sum

@@ -100,6 +100,7 @@ github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -336,6 +337,7 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

cli/resources/createResources.go

@@ -15,7 +15,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
-func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level) (bool, error) {
+func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, basenineImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level) (bool, error) {
 	if !isNsRestrictedMode {
 		if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil {
 			return false, err
@@ -42,6 +42,9 @@ func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.
 		Namespace:             mizuResourcesNamespace,
 		PodName:               kubernetes.ApiServerPodName,
 		PodImage:              agentImage,
+		BasenineImage:         basenineImage,
+		KratosImage:           "",
+		KetoImage:             "",
 		ServiceAccountName:    serviceAccountName,
 		IsNamespaceRestricted: isNsRestrictedMode,
 		SyncEntriesConfig:     syncEntriesConfig,
@@ -65,7 +68,7 @@ func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.
 	return mizuServiceAccountExists, nil
 }
 
-func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, noPersistentVolumeClaim bool) error {
+func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, basenineImage string, kratosImage string, ketoImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, noPersistentVolumeClaim bool) error {
 	if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil {
 		return err
 	}
@@ -95,6 +98,9 @@ func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kuberne
 		Namespace:             mizuResourcesNamespace,
 		PodName:               kubernetes.ApiServerPodName,
 		PodImage:              agentImage,
+		BasenineImage:         basenineImage,
+		KratosImage:           kratosImage,
+		KetoImage:             ketoImage,
 		ServiceAccountName:    serviceAccountName,
 		IsNamespaceRestricted: isNsRestrictedMode,
 		SyncEntriesConfig:     syncEntriesConfig,
@@ -113,7 +119,7 @@ func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kuberne
 	if err != nil {
 		return err
 	}
-	logger.Log.Infof("service/%v created",  kubernetes.ApiServerPodName)
+	logger.Log.Infof("service/%v created", kubernetes.ApiServerPodName)
 
 	return nil
 }

cli/telemetry/telemetry.go

@@ -4,21 +4,21 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"net/http"
-	"os"
-
 	"github.com/denisbrodbeck/machineid"
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/shared/logger"
+	"net/http"
+	"os"
+	"time"
 )
 
 const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
 
 func ReportRun(cmd string, args interface{}) {
 	if !shouldRunTelemetry() {
-		logger.Log.Debugf("not reporting telemetry")
+		logger.Log.Debug("not reporting telemetry")
 		return
 	}
 
@@ -28,7 +28,7 @@ func ReportRun(cmd string, args interface{}) {
 		"args": string(argsBytes),
 	}
 
-	if err := sendTelemetry("Execution", argsMap); err != nil {
+	if err := sendTelemetry(argsMap); err != nil {
 		logger.Log.Debug(err)
 		return
 	}
@@ -36,30 +36,31 @@ func ReportRun(cmd string, args interface{}) {
 	logger.Log.Debugf("successfully reported telemetry for cmd %v", cmd)
 }
 
-func ReportAPICalls(apiProvider *apiserver.Provider) {
+func ReportTapTelemetry(apiProvider *apiserver.Provider, args interface{}, startTime time.Time) {
 	if !shouldRunTelemetry() {
-		logger.Log.Debugf("not reporting telemetry")
+		logger.Log.Debug("not reporting telemetry")
 		return
 	}
 
 	generalStats, err := apiProvider.GetGeneralStats()
 	if err != nil {
-		logger.Log.Debugf("[ERROR] failed get general stats from api server %v", err)
+		logger.Log.Debugf("[ERROR] failed to get general stats from api server %v", err)
 		return
 	}
-
+	argsBytes, _ := json.Marshal(args)
 	argsMap := map[string]interface{}{
-		"apiCallsCount":         generalStats["EntriesCount"],
-		"firstAPICallTimestamp": generalStats["FirstEntryTimestamp"],
-		"lastAPICallTimestamp":  generalStats["LastEntryTimestamp"],
+		"cmd":                    "tap",
+		"args":                   string(argsBytes),
+		"executionTimeInSeconds": int(time.Since(startTime).Seconds()),
+		"apiCallsCount":          generalStats["EntriesCount"],
 	}
 
-	if err := sendTelemetry("APICalls", argsMap); err != nil {
+	if err := sendTelemetry(argsMap); err != nil {
 		logger.Log.Debug(err)
 		return
 	}
 
-	logger.Log.Debugf("successfully reported telemetry of api calls")
+	logger.Log.Debug("successfully reported telemetry of tap command")
 }
 
 func shouldRunTelemetry() bool {
@@ -77,13 +78,12 @@ func shouldRunTelemetry() bool {
 	return true
 }
 
-func sendTelemetry(telemetryType string, argsMap map[string]interface{}) error {
-	argsMap["telemetryType"] = telemetryType
+func sendTelemetry(argsMap map[string]interface{}) error {
 	argsMap["component"] = "mizu_cli"
 	argsMap["buildTimestamp"] = mizu.BuildTimestamp
 	argsMap["branch"] = mizu.Branch
 	argsMap["version"] = mizu.SemVer
-	argsMap["Platform"] = mizu.Platform
+	argsMap["platform"] = mizu.Platform
 
 	if machineId, err := machineid.ProtectedID("mizu"); err == nil {
 		argsMap["machineId"] = machineId

debug.Dockerfile

@@ -1,12 +1,14 @@
 # creates image in which mizu agent is remotely debuggable using delve
-FROM node:14-slim AS site-build
+FROM node:16-slim AS site-build
 
 WORKDIR /app/ui-build
 
-COPY ui .
+COPY ui/package.json .
+COPY ui/package-lock.json .
 RUN npm i
+COPY ui .
 RUN npm run build
-
+RUN npm run build-ent
 
 FROM golang:1.16-alpine AS builder
 # Set necessary environment variables needed for our image.
@@ -34,15 +36,16 @@ ARG SEM_VER=0.0.0
 COPY shared ../shared
 COPY tap ../tap
 COPY agent .
+# Include gcflags for debugging
 RUN go build -gcflags="all=-N -l" -o mizuagent .
 
 COPY devops/build_extensions_debug.sh ..
 RUN cd .. && /bin/bash build_extensions_debug.sh
 
-
 FROM golang:1.16-alpine
 
-RUN apk add bash libpcap-dev
+# Set necessary environment variables needed for our image.
+RUN apk add bash libpcap-dev gcc g++
 
 WORKDIR /app
 
@@ -50,9 +53,17 @@ WORKDIR /app
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
 COPY --from=builder ["/app/agent/build/extensions", "extensions"]
 COPY --from=site-build ["/app/ui-build/build", "site"]
+COPY --from=site-build ["/app/ui-build/build-ent", "site-standalone"]
+RUN mkdir /app/data/
 
-# install remote debugging tool
+# install delve
+ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 RUN go get github.com/go-delve/delve/cmd/dlv
 
+ENV GIN_MODE=debug
+
+# delve ports
+EXPOSE 2345 2346
+
+# this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
 ENTRYPOINT "/app/mizuagent"
-#CMD ["sh", "-c", "dlv --headless=true --listen=:2345 --log --api-version=2 --accept-multiclient exec ./mizuagent -- --api-server"]

deploy/kubernetes/helm-chart/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deploy/kubernetes/helm-chart/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: mizuhelm
+description: Mizu helm chart for Kubernetes
+type: application
+version: 0.1.1
+kubeVersion: ">= 1.16.0-0"
+appVersion: "0.21.29"

deploy/kubernetes/helm-chart/templates/PersistentVolumeClaim.yaml

@@ -0,0 +1,13 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ .Values.volumeClaim.name }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    limits:
+      storage: 700M
+    requests:
+      storage: 700M

deploy/kubernetes/helm-chart/templates/clusterRole.yaml

@@ -0,0 +1,30 @@
+{{- if .Values.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Values.rbac.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    mizu-cli-version: {{ .Chart.AppVersion }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+rules:
+  - apiGroups: [ "", "extensions", "apps" ]
+    resources: [ "endpoints", "pods", "services", "namespaces" ]
+    verbs: [ "get", "list", "watch" ]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Values.rbac.roleBindingName }}
+  labels:
+    mizu-cli-version: {{ .Chart.AppVersion }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Values.rbac.name }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.serviceAccountName }}
+    namespace: {{ .Release.Namespace }}
+  {{- end -}}

deploy/kubernetes/helm-chart/templates/configmap.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.configMap.name }}
+  namespace: {{ .Release.Namespace }}
+data:
+  mizu-config.json: >-
+    {"maxDBSizeBytes":200000000,"agentImage":"{{ .Values.container.tapper.image.repository }}:{{ .Values.container.tapper.image.tag }}","pullPolicy":"Always","logLevel":4,"tapperResources":{"CpuLimit":"750m","MemoryLimit":"1Gi","CpuRequests":"50m","MemoryRequests":"50Mi"},"mizuResourceNamespace":"{{ .Release.Namespace }}","agentDatabasePath":"/app/data/","standaloneMode":true}

deploy/kubernetes/helm-chart/templates/deployment.yaml

@@ -0,0 +1,128 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.pod.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ .Values.pod.name }}
+spec:
+  replicas: {{ .Values.deployment.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ .Values.pod.name }}
+  template:
+    metadata:
+      name: {{ .Values.pod.name }}
+      creationTimestamp: null
+      labels:
+        app: {{ .Values.pod.name }}
+    spec:
+      volumes:
+        - name: {{ .Values.configMap.name }}
+          configMap:
+            name: {{ .Values.configMap.name }}
+            defaultMode: 420
+        - name: {{ .Values.volumeClaim.name }}
+          persistentVolumeClaim:
+            claimName: {{ .Values.volumeClaim.name }}
+      containers:
+        - name: {{ .Values.pod.name }}
+          image: "{{ .Values.container.mizuAgent.image.repository }}:{{ .Values.container.mizuAgent.image.tag | default .Chart.AppVersion }}"
+          command:
+            - ./mizuagent
+            - '--api-server'
+          env:
+            - name: SYNC_ENTRIES_CONFIG
+            - name: LOG_LEVEL
+              value: INFO
+          resources:
+            limits:
+              cpu: 750m
+              memory: 1Gi
+            requests:
+              cpu: 50m
+              memory: 50Mi
+          volumeMounts:
+            - name: {{ .Values.configMap.name }}
+              mountPath: /app/config/
+            - name: {{ .Values.volumeClaim.name }}
+              mountPath: /app/data/
+          livenessProbe:
+            httpGet:
+              path: /echo
+              port: {{ .Values.pod.port }}
+              scheme: HTTP
+            initialDelaySeconds: 1
+            timeoutSeconds: 1
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          imagePullPolicy: Always
+        - name: {{ .Values.container.basenine.name }}
+          image: "{{ .Values.container.basenine.image.repository }}:{{ .Values.container.basenine.image.tag | default .Chart.AppVersion }}"
+          command:
+            - /basenine
+          args:
+            - '-addr'
+            - 0.0.0.0
+            - '-port'
+            - '9099'
+            - '-persistent'
+          workingDir: /app/data/
+          resources:
+            limits:
+              cpu: 750m
+              memory: 1Gi
+            requests:
+              cpu: 50m
+              memory: 50Mi
+          volumeMounts:
+            - name: {{ .Values.configMap.name }}
+              mountPath: /app/config/
+            - name: {{ .Values.volumeClaim.name }}
+              mountPath: /app/data/
+          readinessProbe:
+            tcpSocket:
+              port: 9099
+            timeoutSeconds: 1
+            periodSeconds: 1
+            successThreshold: 1
+            failureThreshold: 3
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          imagePullPolicy: Always
+        - name: kratos
+          image: "{{ .Values.container.kratos.image.repository }}:{{ .Values.container.kratos.image.tag | default .Chart.AppVersion }}"
+          resources:
+            limits:
+              cpu: 750m
+              memory: 1Gi
+            requests:
+              cpu: 50m
+              memory: 50Mi
+          volumeMounts:
+            - name: {{ .Values.configMap.name }}
+              mountPath: /app/config/
+            - name: {{ .Values.volumeClaim.name }}
+              mountPath: /app/data/
+          readinessProbe:
+            httpGet:
+              path: /health/ready
+              port: 4433
+              scheme: HTTP
+            timeoutSeconds: 1
+            periodSeconds: 1
+            successThreshold: 1
+            failureThreshold: 3
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          imagePullPolicy: Always
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 0
+      dnsPolicy: ClusterFirstWithHostNet
+      serviceAccountName: {{ .Values.serviceAccountName }}
+      serviceAccount: {{ .Values.serviceAccountName }}
+      securityContext: { }
+      schedulerName: default-scheduler

deploy/kubernetes/helm-chart/templates/role.yaml

@@ -0,0 +1,29 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Values.roleName }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    mizu-cli-version: {{ .Chart.AppVersion }}
+rules:
+  - apiGroups: [ "apps" ]
+    resources: [ "daemonsets" ]
+    verbs: [ "patch", "get", "list", "create", "delete" ]
+  - apiGroups: [ "events.k8s.i" ]
+    resources: [ "events" ]
+    verbs: [ "list", "watch" ]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Values.roleBindingName }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Values.roleName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.serviceAccountName }}
+    namespace: {{ .Release.Namespace }}
+---

deploy/kubernetes/helm-chart/templates/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.service.name }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - name: api
+      port: {{ .Values.service.port }}
+      targetPort: {{ .Values.pod.port }}
+      protocol: TCP
+  selector:
+    app: {{ .Values.pod.name }}

deploy/kubernetes/helm-chart/templates/serviceAccount.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.serviceAccountName }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    mizu-cli-version: {{ .Chart.AppVersion }}

deploy/kubernetes/helm-chart/values.yaml

@@ -0,0 +1,51 @@
+# Default values for mizu.
+rbac:
+  create: true
+  name: "mizu-cluster-role"
+  roleBindingName: "mizu-role-binding"
+
+serviceAccountName: "mizu-service-account"
+
+roleName: "mizu-role-daemon"
+roleBindingName: "mizu-role-binding-daemon"
+
+service:
+  name: "mizu-api-server"
+  type: ClusterIP
+  port: 80
+
+pod:
+  name: "mizu-api-server"
+  port: 8899
+
+container:
+  mizuAgent:
+    image:
+      repository: "gcr.io/up9-docker-hub/mizu/main"
+      tag: "0.22.0"
+  tapper:
+    image:
+      repository: "gcr.io/up9-docker-hub/mizu/main"
+      tag: "0.22.0"
+  basenine:
+    name: "basenine"
+    port: 9099
+    image:
+      repository: "ghcr.io/up9inc/basenine"
+      tag: "v0.3.0"
+  kratos:
+    name: "kratos"
+    port: 4433
+    image:
+      repository: "gcr.io/up9-docker-hub/mizu-kratos/stable"
+      tag: "0.0.0"
+
+deployment:
+  replicaCount: 1
+
+configMap:
+  name: "mizu-config"
+
+volumeClaim:
+  create: true
+  name: "mizu-volume-claim"