Install page on enter event handler (#640)

Support source IP resolving for HTTP traffic in Istio service mesh.

If X-Forwarded-For HTTP request header is present, replace the source address with the left-most address in X-Forwarded-For (earliest in Envoy implementation of this header).

This allows Mizu to resolve source IPs in Istio service meshes, if the use_remote_address option is on. Added instructions on how to turn it on.

Dockerfile

@@ -41,12 +41,6 @@ RUN go build -ldflags="-s -w \
      -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
      -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
-# Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.19/basenine_linux_amd64 ./basenine_linux_amd64
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.19/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
-RUN shasum -a 256 -c basenine_linux_amd64.sha256
-RUN chmod +x ./basenine_linux_amd64
-
 COPY devops/build_extensions.sh ..
 RUN cd .. && /bin/bash build_extensions.sh
 
@@ -58,7 +52,6 @@ WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
-COPY --from=builder ["/app/agent-build/basenine_linux_amd64", "/usr/local/bin/basenine"]
 COPY --from=builder ["/app/agent/build/extensions", "extensions"]
 COPY --from=site-build ["/app/ui-build/build", "site"]
 RUN mkdir /app/data/

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui agent cli tap docker
+.PHONY: help ui extensions extensions-debug agent agent-debug cli tap docker
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -28,6 +28,9 @@ ui: ## Build UI.
 cli: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build
 
+cli-debug: ## Build CLI.
+	@echo "building cli"; cd cli && $(MAKE) build-debug
+
 build-cli-ci: ## Build CLI for CI.
 	@echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci
 
@@ -37,6 +40,12 @@ agent: ## Build agent.
 	${MAKE} extensions
 	@ls -l agent/build
 
+agent-debug: ## Build agent for debug.
+	@(echo "building mizu agent for debug.." )
+	@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
+	${MAKE} extensions-debug
+	@ls -l agent/build
+
 docker: ## Build and publish agent docker image.
 	$(MAKE) push-docker
 
@@ -62,7 +71,7 @@ push-cli: ## Build and publish CLI.
 	gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
-clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
+clean: clean-ui clean-agent clean-cli clean-docker clean-extensions ## Clean all build artifacts.
 
 clean-ui: ## Clean UI.
 	@(rm -rf ui/build ; echo "UI cleanup done" )
@@ -73,9 +82,15 @@ clean-agent: ## Clean agent.
 clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
+clean-extensions:  ## Clean extensions
+	@(rm -rf tap/extensions/*.so ; echo "Extensions cleanup done" )
+
 clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
+extensions-debug:
+	devops/build_extensions_debug.sh
+
 extensions:
 	devops/build_extensions.sh
 

acceptanceTests/cypress.json

@@ -4,5 +4,11 @@
   "viewportHeight": 1080,
   "video": false,
   "screenshotOnRunFailure": false,
-  "testFiles": ["tests/GuiPort.js"]
+  "testFiles":
+  ["tests/GuiPort.js",
+  "tests/MultipleNamespaces.js",
+  "tests/Regex.js"],
+  "env": {
+    "testUrl": "http://localhost:8899/"
+  }
 }

acceptanceTests/cypress/integration/page_objects/StatusBar.js

@@ -0,0 +1,46 @@
+const columns = {podName : 1, namespace : 2, tapping : 3};
+const greenStatusImageSrc = '/static/media/success.662997eb.svg';
+
+function getDomPathInStatusBar(line, column) {
+    return `.expandedStatusBar > :nth-child(2) > > :nth-child(2) > :nth-child(${line}) > :nth-child(${column})`;
+}
+
+export function checkLine(line, expectedValues) {
+    cy.get(getDomPathInStatusBar(line, columns.podName)).invoke('text').then(podValue => {
+        const podName = getOnlyPodName(podValue);
+        expect(podName).to.equal(expectedValues.podName);
+
+        cy.get(getDomPathInStatusBar(line, columns.namespace)).invoke('text').then(namespaceValue => {
+            expect(namespaceValue).to.equal(expectedValues.namespace);
+            cy.get(getDomPathInStatusBar(line, columns.tapping)).children().should('have.attr', 'src', greenStatusImageSrc);
+        });
+    });
+}
+
+export function findLineAndCheck(expectedValues) {
+    cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(1)').then(pods => {
+        cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(2)').then(namespaces => {
+            // organizing namespaces array
+            const podObjectsArray = Object.values(pods ?? {});
+            const namespacesObjectsArray = Object.values(namespaces ?? {});
+            let lineNumber = -1;
+            namespacesObjectsArray.forEach((namespaceObj, index) => {
+                const currentLine = index + 1;
+                lineNumber = (namespaceObj.getAttribute && namespaceObj.innerHTML === expectedValues.namespace && (getOnlyPodName(podObjectsArray[index].innerHTML)) === expectedValues.podName) ? currentLine : lineNumber;
+            });
+            lineNumber === -1 ? throwError(expectedValues) : checkLine(lineNumber, expectedValues);
+        });
+    });
+}
+
+function throwError(expectedValues) {
+    throw new Error(`The pod named ${expectedValues.podName} doesn't match any namespace named ${expectedValues.namespace}`);
+}
+
+export function getExpectedDetailsDict(podName, namespace) {
+    return {podName : podName, namespace : namespace};
+}
+
+function getOnlyPodName(podElementFullStr) {
+    return podElementFullStr.substring(0, podElementFullStr.indexOf('-'));
+}

acceptanceTests/cypress/integration/tests/GuiPort.js

@@ -1,8 +1,8 @@
 it('check', function () {
-    cy.visit(`http://localhost:${Cypress.env('port')}/`)
+    cy.visit(`http://localhost:${Cypress.env('port')}/`);
 
-    cy.get('.header').should('be.visible')
-    cy.get('.TrafficPageHeader').should('be.visible')
-    cy.get('.TrafficPage-ListContainer').should('be.visible')
-    cy.get('.TrafficPage-Container').should('be.visible')
-})
+    cy.get('.header').should('be.visible');
+    cy.get('.TrafficPageHeader').should('be.visible');
+    cy.get('.TrafficPage-ListContainer').should('be.visible');
+    cy.get('.TrafficPage-Container').should('be.visible');
+});

acceptanceTests/cypress/integration/tests/MultipleNamespaces.js

@@ -0,0 +1,18 @@
+import {findLineAndCheck, getExpectedDetailsDict} from '../page_objects/StatusBar';
+
+it('opening', function () {
+    cy.visit(Cypress.env('testUrl'));
+    cy.get('.podsCount').trigger('mouseover');
+});
+
+[1, 2, 3].map(doItFunc);
+
+function doItFunc(number) {
+    const podName = Cypress.env(`name${number}`);
+    const namespace = Cypress.env(`namespace${number}`);
+
+    it(`verifying the pod (${podName}, ${namespace})`, function () {
+        findLineAndCheck(getExpectedDetailsDict(podName, namespace));
+    });
+}
+

acceptanceTests/cypress/integration/tests/Regex.js

@@ -0,0 +1,11 @@
+import {getExpectedDetailsDict, checkLine} from '../page_objects/StatusBar';
+
+
+it('opening', function () {
+    cy.visit(Cypress.env('testUrl'));
+    cy.get('.podsCount').trigger('mouseover');
+
+    cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) >').should('have.length', 1); // one line
+
+    checkLine(1, getExpectedDetailsDict(Cypress.env('name'), Cypress.env('namespace')));
+});

acceptanceTests/logs_test.go

@@ -81,11 +81,16 @@ func TestLogs(t *testing.T) {
 		logsFileNames = append(logsFileNames, file.Name)
 	}
 
-	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
+	if !Contains(logsFileNames, "mizu.mizu-api-server.mizu-api-server.log") {
 		t.Errorf("api server logs not found")
 		return
 	}
 
+	if !Contains(logsFileNames, "mizu.mizu-api-server.basenine.log") {
+		t.Errorf("basenine logs not found")
+		return
+	}
+
 	if !Contains(logsFileNames, "mizu_cli.log") {
 		t.Errorf("cli logs not found")
 		return
@@ -174,11 +179,16 @@ func TestLogsPath(t *testing.T) {
 		logsFileNames = append(logsFileNames, file.Name)
 	}
 
-	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
+	if !Contains(logsFileNames, "mizu.mizu-api-server.mizu-api-server.log") {
 		t.Errorf("api server logs not found")
 		return
 	}
 
+	if !Contains(logsFileNames, "mizu.mizu-api-server.basenine.log") {
+		t.Errorf("basenine logs not found")
+		return
+	}
+
 	if !Contains(logsFileNames, "mizu_cli.log") {
 		t.Errorf("cli logs not found")
 		return

acceptanceTests/tap_test.go

@@ -151,6 +151,7 @@ func TestTapAllNamespaces(t *testing.T) {
 
 	expectedPods := []PodDescriptor{
 		{Name: "httpbin", Namespace: "mizu-tests"},
+		{Name: "httpbin2", Namespace: "mizu-tests"},
 		{Name: "httpbin", Namespace: "mizu-tests2"},
 	}
 
@@ -184,25 +185,8 @@ func TestTapAllNamespaces(t *testing.T) {
 		return
 	}
 
-	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
-	requestResult, requestErr := executeHttpGetRequest(podsUrl)
-	if requestErr != nil {
-		t.Errorf("failed to get tap status, err: %v", requestErr)
-		return
-	}
-
-	pods, err := getPods(requestResult)
-	if err != nil {
-		t.Errorf("failed to get pods, err: %v", err)
-		return
-	}
-
-	for _, expectedPod := range expectedPods {
-		if !isPodDescriptorInPodArray(pods, expectedPod) {
-			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
-			return
-		}
-	}
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
+		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
 }
 
 func TestTapMultipleNamespaces(t *testing.T) {
@@ -250,30 +234,8 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		return
 	}
 
-	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
-	requestResult, requestErr := executeHttpGetRequest(podsUrl)
-	if requestErr != nil {
-		t.Errorf("failed to get tap status, err: %v", requestErr)
-		return
-	}
-
-	pods, err := getPods(requestResult)
-	if err != nil {
-		t.Errorf("failed to get pods, err: %v", err)
-		return
-	}
-
-	if len(expectedPods) != len(pods) {
-		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
-		return
-	}
-
-	for _, expectedPod := range expectedPods {
-		if !isPodDescriptorInPodArray(pods, expectedPod) {
-			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
-			return
-		}
-	}
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
+		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
 }
 
 func TestTapRegex(t *testing.T) {
@@ -318,30 +280,8 @@ func TestTapRegex(t *testing.T) {
 		return
 	}
 
-	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
-	requestResult, requestErr := executeHttpGetRequest(podsUrl)
-	if requestErr != nil {
-		t.Errorf("failed to get tap status, err: %v", requestErr)
-		return
-	}
-
-	pods, err := getPods(requestResult)
-	if err != nil {
-		t.Errorf("failed to get pods, err: %v", err)
-		return
-	}
-
-	if len(expectedPods) != len(pods) {
-		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
-		return
-	}
-
-	for _, expectedPod := range expectedPods {
-		if !isPodDescriptorInPodArray(pods, expectedPod) {
-			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
-			return
-		}
-	}
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Regex.js\" --env name=%v,namespace=%v",
+		expectedPods[0].Name, expectedPods[0].Namespace))
 }
 
 func TestTapDryRun(t *testing.T) {
@@ -862,11 +802,16 @@ func TestTapDumpLogs(t *testing.T) {
 		logsFileNames = append(logsFileNames, file.Name)
 	}
 
-	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
+	if !Contains(logsFileNames, "mizu.mizu-api-server.mizu-api-server.log") {
 		t.Errorf("api server logs not found")
 		return
 	}
 
+	if !Contains(logsFileNames, "mizu.mizu-api-server.basenine.log") {
+		t.Errorf("basenine logs not found")
+		return
+	}
+
 	if !Contains(logsFileNames, "mizu_cli.log") {
 		t.Errorf("cli logs not found")
 		return

acceptanceTests/testsUtils.go

@@ -92,7 +92,7 @@ func getDefaultCommandArgs() []string {
 	setFlag := "--set"
 	telemetry := "telemetry=false"
 	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
-	imagePullPolicy := "image-pull-policy=Never"
+	imagePullPolicy := "image-pull-policy=IfNotPresent"
 	headless := "headless=true"
 
 	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
@@ -183,16 +183,16 @@ func tryExecuteFunc(executeFunc func() error) (err interface{}) {
 }
 
 func waitTapPodsReady(apiServerUrl string) error {
-	resolvingUrl := fmt.Sprintf("%v/status/tappersCount", apiServerUrl)
+	resolvingUrl := fmt.Sprintf("%v/status/connectedTappersCount", apiServerUrl)
 	tapPodsReadyFunc := func() error {
 		requestResult, requestErr := executeHttpGetRequest(resolvingUrl)
 		if requestErr != nil {
 			return requestErr
 		}
 
-		tappersCount := requestResult.(float64)
-		if tappersCount == 0 {
-			return fmt.Errorf("no tappers running")
+		connectedTappersCount := requestResult.(float64)
+		if connectedTappersCount == 0 {
+			return fmt.Errorf("no connected tappers running")
 		}
 		time.Sleep(waitAfterTapPodsReady)
 		return nil

agent/go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
+	github.com/chanced/openapi v0.0.6
 	github.com/djherbis/atime v1.0.0
 	github.com/getkin/kin-openapi v0.76.0
 	github.com/gin-contrib/static v0.0.1
@@ -12,12 +13,13 @@ require (
 	github.com/go-playground/universal-translator v0.17.0
 	github.com/go-playground/validator/v10 v10.5.0
 	github.com/google/martian v2.1.0+incompatible
+	github.com/google/uuid v1.1.2
 	github.com/gorilla/websocket v1.4.2
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/ory/kratos-client-go v0.8.2-alpha.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/up9inc/basenine/client/go v0.0.0-20211215185650-10083bb9a1b3
+	github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0

agent/go.sum

@@ -82,6 +82,12 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
+github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a h1:zG6t+4krPXcCKtLbjFvAh+fKN1d0qfD+RaCj+680OU8=
+github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a/go.mod h1:yhcmlFk1hxuZ+5XZbupzT/cEm/eE4ZvWbmsW1+Q/aZE=
+github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44 h1:4NOJMtvZaOA6cI2gkIuXk/2b5KTOvm/R4zyPy/yLCM4=
+github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44/go.mod h1:XVNfXN5kgZST4PQ0W/oBAHJku2OteCeHxjAbvfd0ARM=
+github.com/chanced/openapi v0.0.6 h1:2giGS47+T8/7MN2hfRHSIUPx86Qksk+J/ciIWcAM7hY=
+github.com/chanced/openapi v0.0.6/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -119,6 +125,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
+github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -253,8 +261,10 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -316,6 +326,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -432,6 +443,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE=
+github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -465,18 +478,29 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.12.0 h1:61wEp/qfvFnqKH/WCI3M8HuRut+mHT6Mr82QrFmM2SY=
+github.com/tidwall/gjson v1.12.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.3 h1:5+deguEhHSEjmuICXZ21uSSsXotWMA0orU783+Z7Cp8=
+github.com/tidwall/sjson v1.2.3/go.mod h1:5WdjKx3AQMvCJ4RG6/2UYT7dLrGvJUV1x4jdTAyGvZs=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/up9inc/basenine/client/go v0.0.0-20211215185650-10083bb9a1b3 h1:FeDCVOBFVpZA5/O5hfPdGTn0rdR2jTEYo3iB2htELI4=
-github.com/up9inc/basenine/client/go v0.0.0-20211215185650-10083bb9a1b3/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d h1:WTz53dcfqCIWZpZLQoHbIcNc21s0ZHEZH7EqMPp99qQ=
+github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=
+github.com/wI2L/jsondiff v0.1.1/go.mod h1:bAbJSAJXZtfOCZ5y3v7Mfb6UQa3DGdGFjQj1cNv8EcM=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
@@ -863,5 +887,6 @@ sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

agent/main.go

@@ -11,12 +11,12 @@ import (
 	"mizuserver/pkg/controllers"
 	"mizuserver/pkg/middlewares"
 	"mizuserver/pkg/models"
+	"mizuserver/pkg/oas"
 	"mizuserver/pkg/routes"
 	"mizuserver/pkg/up9"
 	"mizuserver/pkg/utils"
 	"net/http"
 	"os"
-	"os/exec"
 	"os/signal"
 	"path"
 	"path/filepath"
@@ -115,13 +115,13 @@ func main() {
 
 		go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
 	} else if *apiServerMode {
-		startBasenineServer(shared.BasenineHost, shared.BaseninePort)
+		configureBasenineServer(shared.BasenineHost, shared.BaseninePort)
 		startTime = time.Now().UnixNano() / int64(time.Millisecond)
 		api.StartResolving(*namespace)
 
 		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-
+		enableExpFeatureIfNeeded()
 		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
 		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
 
@@ -149,16 +149,13 @@ func main() {
 	logger.Log.Info("Exiting")
 }
 
-func startBasenineServer(host string, port string) {
-	cmd := exec.Command("basenine", "-addr", host, "-port", port, "-persistent")
-	cmd.Dir = config.Config.AgentDatabasePath
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	err := cmd.Start()
-	if err != nil {
-		logger.Log.Panicf("Failed starting Basenine: %v", err)
+func enableExpFeatureIfNeeded() {
+	if config.Config.OAS {
+		oas.GetOasGeneratorInstance().Start()
 	}
+}
 
+func configureBasenineServer(host string, port string) {
 	if !wait.New(
 		wait.WithProto("tcp"),
 		wait.WithWait(200*time.Millisecond),
@@ -166,25 +163,16 @@ func startBasenineServer(host string, port string) {
 		wait.WithDeadline(5*time.Second),
 		wait.WithDebug(true),
 	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
-		logger.Log.Panicf("Basenine is not available: %v", err)
+		logger.Log.Panicf("Basenine is not available!")
 	}
 
-	// Make a channel to gracefully exit Basenine.
-	channel := make(chan os.Signal)
-	signal.Notify(channel, os.Interrupt, syscall.SIGTERM)
-
-	// Handle the channel.
-	go func() {
-		<-channel
-		cmd.Process.Signal(syscall.SIGTERM)
-	}()
-
 	// Limit the database size to default 200MB
-	err = basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
+	err := basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
 	if err != nil {
 		logger.Log.Panicf("Error while limiting database size: %v", err)
 	}
 
+	// Define the macros
 	for _, extension := range extensions {
 		macros := extension.Dissector.Macros()
 		for macro, expanded := range macros {
@@ -252,7 +240,7 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 
 	app.Use(DisableRootStaticCache())
 
-	if err := setUIMode(); err != nil {
+	if err := setUIFlags(); err != nil {
 		logger.Log.Errorf("Error setting ui mode, err: %v", err)
 	}
 	app.Use(static.ServeRoot("/", "./site"))
@@ -267,12 +255,15 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 		routes.InstallRoutes(app)
 	}
 
+	if config.Config.OAS {
+		routes.OASRoutes(app)
+	}
+
 	routes.QueryRoutes(app)
 	routes.EntriesRoutes(app)
 	routes.MetadataRoutes(app)
 	routes.StatusRoutes(app)
 	routes.NotFoundRoute(app)
-
 	utils.StartServer(app)
 }
 
@@ -287,13 +278,14 @@ func DisableRootStaticCache() gin.HandlerFunc {
 	}
 }
 
-func setUIMode() error {
+func setUIFlags() error {
 	read, err := ioutil.ReadFile(uiIndexPath)
 	if err != nil {
 		return err
 	}
 
 	replacedContent := strings.Replace(string(read), "__IS_STANDALONE__", strconv.FormatBool(config.Config.StandaloneMode), 1)
+	replacedContent = strings.Replace(replacedContent, "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS), 1)
 
 	err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
 	if err != nil {

agent/pkg/api/main.go

@@ -19,6 +19,7 @@ import (
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	"mizuserver/pkg/models"
+	"mizuserver/pkg/oas"
 	"mizuserver/pkg/resolver"
 	"mizuserver/pkg/utils"
 
@@ -119,15 +120,12 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
 		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
-		baseEntry := extension.Dissector.Summarize(mizuEntry)
-		mizuEntry.Base = baseEntry
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
 				var httpPair tapApi.HTTPRequestResponsePair
 				json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair)
 
 				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
-				baseEntry.ContractStatus = contract.Status
 				mizuEntry.ContractStatus = contract.Status
 				mizuEntry.ContractRequestReason = contract.RequestReason
 				mizuEntry.ContractResponseReason = contract.ResponseReason
@@ -137,8 +135,10 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 			harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
 			if err == nil {
 				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
-				baseEntry.Rules = rules
+				mizuEntry.Rules = rules
 			}
+
+			oas.GetOasGeneratorInstance().PushEntry(harEntry)
 		}
 
 		data, err := json.Marshal(mizuEntry)

agent/pkg/api/socket_routes.go

@@ -16,6 +16,7 @@ import (
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 type EventHandlers interface {
@@ -131,18 +132,10 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 						return
 					}
 
-					var dataMap map[string]interface{}
-					err = json.Unmarshal(bytes, &dataMap)
+					var entry *tapApi.Entry
+					err = json.Unmarshal(bytes, &entry)
 
-					var base map[string]interface{}
-					switch dataMap["base"].(type) {
-					case map[string]interface{}:
-						base = dataMap["base"].(map[string]interface{})
-						base["id"] = uint(dataMap["id"].(float64))
-					default:
-						logger.Log.Debugf("Base field has an unrecognized type: %+v", dataMap)
-						continue
-					}
+					base := tapApi.Summarize(entry)
 
 					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
 					SendToSocket(socketId, baseEntryBytes)

agent/pkg/api/socket_server_handlers.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tappers"
 	"mizuserver/pkg/up9"
 	"sync"
 
@@ -29,7 +30,7 @@ func init() {
 func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
-		providers.TapperAdded()
+		tappers.Connected()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
 		socketListLock.Lock()
@@ -41,7 +42,7 @@ func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
-		providers.TapperRemoved()
+		tappers.Disconnected()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
 		socketListLock.Lock()

agent/pkg/controllers/config_controller.go

@@ -11,18 +11,20 @@ import (
 	"mizuserver/pkg/config"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tapConfig"
+	"mizuserver/pkg/providers/tappedPods"
+	"mizuserver/pkg/providers/tappers"
 	"net/http"
 	"regexp"
 	"time"
 )
 
-var globalTapConfig = &models.TapConfig{}
 var cancelTapperSyncer context.CancelFunc
 
 func PostTapConfig(c *gin.Context) {
-	tapConfig := &models.TapConfig{}
+	requestTapConfig := &models.TapConfig{}
 
-	if err := c.Bind(tapConfig); err != nil {
+	if err := c.Bind(requestTapConfig); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
@@ -30,14 +32,14 @@ func PostTapConfig(c *gin.Context) {
 	if cancelTapperSyncer != nil {
 		cancelTapperSyncer()
 
-		providers.TapStatus = shared.TapStatus{}
-		providers.TappersStatus = make(map[string]shared.TapperStatus)
+		tappedPods.Set([]*shared.PodInfo{})
+		tappers.ResetStatus()
 
 		broadcastTappedPodsStatus()
 	}
 
 	var tappedNamespaces []string
-	for namespace, tapped := range tapConfig.TappedNamespaces {
+	for namespace, tapped := range requestTapConfig.TappedNamespaces {
 		if tapped {
 			tappedNamespaces = append(tappedNamespaces, namespace)
 		}
@@ -45,7 +47,7 @@ func PostTapConfig(c *gin.Context) {
 
 	podRegex, _ := regexp.Compile(".*")
 
-	kubernetesProvider, err := kubernetes.NewProviderInCluster()
+	kubernetesProvider, err := providers.GetKubernetesProvider()
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, err)
 		return
@@ -60,13 +62,13 @@ func PostTapConfig(c *gin.Context) {
 	}
 
 	cancelTapperSyncer = cancel
-	globalTapConfig = tapConfig
+	tapConfig.Save(requestTapConfig)
 
 	c.JSON(http.StatusOK, "OK")
 }
 
 func GetTapConfig(c *gin.Context) {
-	kubernetesProvider, err := kubernetes.NewProviderInCluster()
+	kubernetesProvider, err := providers.GetKubernetesProvider()
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, err)
 		return
@@ -81,20 +83,22 @@ func GetTapConfig(c *gin.Context) {
 		return
 	}
 
+	savedTapConfig := tapConfig.Get()
+
 	tappedNamespaces := make(map[string]bool)
 	for _, namespace := range namespaces {
 		if namespace.Name == config.Config.MizuResourcesNamespace {
 			continue
 		}
 
-		tappedNamespaces[namespace.Name] = globalTapConfig.TappedNamespaces[namespace.Name]
+		tappedNamespaces[namespace.Name] = savedTapConfig.TappedNamespaces[namespace.Name]
 	}
 
-	tapConfig := models.TapConfig{TappedNamespaces: tappedNamespaces}
-	c.JSON(http.StatusOK, tapConfig)
+	tapConfigToReturn := models.TapConfig{TappedNamespaces: tappedNamespaces}
+	c.JSON(http.StatusOK, tapConfigToReturn)
 }
 
-func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, istio bool) (*kubernetes.MizuTapperSyncer, error) {
+func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, serviceMesh bool) (*kubernetes.MizuTapperSyncer, error) {
 	tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
 		TargetNamespaces:         targetNamespaces,
 		PodFilterRegex:           podFilterRegex,
@@ -106,7 +110,7 @@ func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, t
 		IgnoredUserAgents:        ignoredUserAgents,
 		MizuApiFilteringOptions:  mizuApiFilteringOptions,
 		MizuServiceAccountExists: true, //assume service account exists since install mode will not function without it anyway
-		Istio:                    istio,
+		ServiceMesh:              serviceMesh,
 	}, time.Now())
 
 	if err != nil {
@@ -129,7 +133,7 @@ func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, t
 					return
 				}
 
-				providers.TapStatus = shared.TapStatus{Pods: kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods)}
+				tappedPods.Set(kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods))
 				broadcastTappedPodsStatus()
 			case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut:
 				if !ok {
@@ -137,7 +141,7 @@ func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, t
 					return
 				}
 
-				addTapperStatus(tapperStatus)
+				tappers.SetStatus(&tapperStatus)
 				broadcastTappedPodsStatus()
 			case <-ctx.Done():
 				logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")

agent/pkg/controllers/entries_controller.go

@@ -64,8 +64,8 @@ func GetEntries(c *gin.Context) {
 	var dataSlice []interface{}
 
 	for _, row := range data {
-		var dataMap map[string]interface{}
-		err = json.Unmarshal(row, &dataMap)
+		var entry *tapApi.Entry
+		err = json.Unmarshal(row, &entry)
 		if err != nil {
 			c.JSON(http.StatusBadRequest, gin.H{
 				"error":     true,
@@ -76,8 +76,7 @@ func GetEntries(c *gin.Context) {
 			return // exit
 		}
 
-		base := dataMap["base"].(map[string]interface{})
-		base["id"] = uint(dataMap["id"].(float64))
+		base := tapApi.Summarize(entry)
 
 		dataSlice = append(dataSlice, base)
 	}
@@ -95,9 +94,19 @@ func GetEntries(c *gin.Context) {
 }
 
 func GetEntry(c *gin.Context) {
+	singleEntryRequest := &models.SingleEntryRequest{}
+
+	if err := c.BindQuery(singleEntryRequest); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+	validationError := validation.Validate(singleEntryRequest)
+	if validationError != nil {
+		c.JSON(http.StatusBadRequest, validationError)
+	}
+
 	id, _ := strconv.Atoi(c.Param("id"))
-	var entry tapApi.MizuEntry
-	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id)
+	var entry *tapApi.Entry
+	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id, singleEntryRequest.Query)
 	if Error(c, err) {
 		return // exit
 	}
@@ -125,7 +134,7 @@ func GetEntry(c *gin.Context) {
 		json.Unmarshal(inrec, &rules)
 	}
 
-	c.JSON(http.StatusOK, tapApi.MizuEntryWrapper{
+	c.JSON(http.StatusOK, tapApi.EntryWrapper{
 		Protocol:       entry.Protocol,
 		Representation: string(representation),
 		BodySize:       bodySize,

agent/pkg/controllers/oas_controller.go

@@ -0,0 +1,62 @@
+package controllers
+
+import (
+	"github.com/chanced/openapi"
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/oas"
+	"net/http"
+)
+
+func GetOASServers(c *gin.Context) {
+	m := make([]string, 0)
+	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
+		m = append(m, key.(string))
+		return true
+	})
+
+	c.JSON(http.StatusOK, m)
+}
+
+func GetOASSpec(c *gin.Context) {
+	res, ok := oas.GetOasGeneratorInstance().ServiceSpecs.Load(c.Param("id"))
+	if !ok {
+		c.JSON(http.StatusNotFound, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       "Service not found among specs",
+		})
+		return // exit
+	}
+
+	gen := res.(*oas.SpecGen)
+	spec, err := gen.GetSpec()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       err,
+		})
+		return // exit
+	}
+
+	c.JSON(http.StatusOK, spec)
+}
+
+func GetOASAllSpecs(c *gin.Context) {
+	res := map[string]*openapi.OpenAPI{}
+	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
+		svc := key.(string)
+		gen := value.(*oas.SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			logger.Log.Warningf("Failed to obtain spec for service %s: %s", svc, err)
+			return true
+		}
+		res[svc] = spec
+		return true
+	})
+	c.JSON(http.StatusOK, res)
+}

agent/pkg/controllers/oas_controller_test.go

@@ -0,0 +1,43 @@
+package controllers
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/oas"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestGetOASServers(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+
+	GetOASServers(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+	return
+}
+
+func TestGetOASAllSpecs(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+
+	GetOASAllSpecs(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+	return
+}
+
+func TestGetOASSpec(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+
+	c.Params = []gin.Param{{Key: "id", Value: "some"}}
+
+	GetOASSpec(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+	return
+}

agent/pkg/controllers/status_controller.go

@@ -8,43 +8,41 @@ import (
 	"mizuserver/pkg/api"
 	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
+	"mizuserver/pkg/providers/tappedPods"
+	"mizuserver/pkg/providers/tappers"
 	"mizuserver/pkg/up9"
-	"mizuserver/pkg/utils"
 	"mizuserver/pkg/validation"
 	"net/http"
 )
 
 func HealthCheck(c *gin.Context) {
-	tappers := make([]shared.TapperStatus, 0)
-	for _, value := range providers.TappersStatus {
-		tappers = append(tappers, value)
+	tappersStatus := make([]*shared.TapperStatus, 0)
+	for _, value := range tappers.GetStatus() {
+		tappersStatus = append(tappersStatus, value)
 	}
 
 	response := shared.HealthResponse{
-		TapStatus:     providers.TapStatus,
-		TappersCount:  providers.TappersCount,
-		TappersStatus: tappers,
+		TappedPods:            tappedPods.Get(),
+		ConnectedTappersCount: tappers.GetConnectedCount(),
+		TappersStatus:         tappersStatus,
 	}
 	c.JSON(http.StatusOK, response)
 }
 
 func PostTappedPods(c *gin.Context) {
-	tapStatus := &shared.TapStatus{}
-	if err := c.Bind(tapStatus); err != nil {
+	var requestTappedPods []*shared.PodInfo
+	if err := c.Bind(&requestTappedPods); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
-	if err := validation.Validate(tapStatus); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-		return
-	}
-	logger.Log.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods))
-	providers.TapStatus.Pods = tapStatus.Pods
+
+	logger.Log.Infof("[Status] POST request: %d tapped pods", len(requestTappedPods))
+	tappedPods.Set(requestTappedPods)
 	broadcastTappedPodsStatus()
 }
 
 func broadcastTappedPodsStatus() {
-	tappedPodsStatus := utils.GetTappedPodsStatus()
+	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
 
 	message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
 	if jsonBytes, err := json.Marshal(message); err != nil {
@@ -54,14 +52,6 @@ func broadcastTappedPodsStatus() {
 	}
 }
 
-func addTapperStatus(tapperStatus shared.TapperStatus) {
-	if providers.TappersStatus == nil {
-		providers.TappersStatus = make(map[string]shared.TapperStatus)
-	}
-
-	providers.TappersStatus[tapperStatus.NodeName] = tapperStatus
-}
-
 func PostTapperStatus(c *gin.Context) {
 	tapperStatus := &shared.TapperStatus{}
 	if err := c.Bind(tapperStatus); err != nil {
@@ -75,12 +65,12 @@ func PostTapperStatus(c *gin.Context) {
 	}
 
 	logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus)
-	addTapperStatus(*tapperStatus)
+	tappers.SetStatus(tapperStatus)
 	broadcastTappedPodsStatus()
 }
 
-func GetTappersCount(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.TappersCount)
+func GetConnectedTappersCount(c *gin.Context) {
+	c.JSON(http.StatusOK, tappers.GetConnectedCount())
 }
 
 func GetAuthStatus(c *gin.Context) {
@@ -94,7 +84,7 @@ func GetAuthStatus(c *gin.Context) {
 }
 
 func GetTappingStatus(c *gin.Context) {
-	tappedPodsStatus := utils.GetTappedPodsStatus()
+	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
 	c.JSON(http.StatusOK, tappedPodsStatus)
 }
 

agent/pkg/controllers/user_controller.go

@@ -18,7 +18,7 @@ func Login(c *gin.Context) {
 func Logout(c *gin.Context) {
 	token := c.GetHeader("x-session-token")
 	if err := providers.Logout(token, c.Request.Context()); err != nil {
-		c.AbortWithStatusJSON(401, gin.H{"error": "error occured while logging out, the session might still be valid"})
+		c.AbortWithStatusJSON(500, gin.H{"error": "error occured while logging out, the session might still be valid"})
 	} else {
 		c.JSON(200, "")
 	}

agent/pkg/models/models.go

@@ -12,7 +12,7 @@ import (
 	"github.com/up9inc/mizu/tap"
 )
 
-func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error {
+func GetEntry(r *tapApi.Entry, v tapApi.DataUnmarshaler) error {
 	return v.UnmarshalData(r)
 }
 
@@ -28,6 +28,10 @@ type EntriesRequest struct {
 	TimeoutMs int    `form:"timeoutMs" validate:"min=1"`
 }
 
+type SingleEntryRequest struct {
+	Query string `form:"query"`
+}
+
 type EntriesResponse struct {
 	Data []interface{}      `json:"data"`
 	Meta *basenine.Metadata `json:"meta"`
@@ -35,7 +39,7 @@ type EntriesResponse struct {
 
 type WebSocketEntryMessage struct {
 	*shared.WebSocketMessageMetadata
-	Data map[string]interface{} `json:"data,omitempty"`
+	Data *tapApi.BaseEntry `json:"data,omitempty"`
 }
 
 type WebSocketTappedEntryMessage struct {
@@ -74,7 +78,7 @@ type WebSocketStartTimeMessage struct {
 	Data int64 `json:"data"`
 }
 
-func CreateBaseEntryWebSocketMessage(base map[string]interface{}) ([]byte, error) {
+func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) {
 	message := &WebSocketEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
 			MessageType: shared.WebSocketMessageTypeEntry,

agent/pkg/oas/feeder_test.go

@@ -0,0 +1,158 @@
+package oas
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func getFiles(baseDir string) (result []string, err error) {
+	result = make([]string, 0, 0)
+	logger.Log.Infof("Reading files from tree: %s", baseDir)
+
+	// https://yourbasic.org/golang/list-files-in-directory/
+	err = filepath.Walk(baseDir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			ext := strings.ToLower(filepath.Ext(path))
+			if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
+				result = append(result, path)
+			}
+
+			return nil
+		})
+
+	sort.SliceStable(result, func(i, j int) bool {
+		return fileSize(result[i]) < fileSize(result[j])
+	})
+
+	logger.Log.Infof("Got files: %d", len(result))
+	return result, err
+}
+
+func fileSize(fname string) int64 {
+	fi, err := os.Stat(fname)
+	if err != nil {
+		panic(err)
+	}
+
+	return fi.Size()
+}
+
+func feedEntries(fromFiles []string) (err error) {
+	for _, file := range fromFiles {
+		logger.Log.Info("Processing file: " + file)
+		ext := strings.ToLower(filepath.Ext(file))
+		switch ext {
+		case ".har":
+			err = feedFromHAR(file)
+			if err != nil {
+				logger.Log.Warning("Failed processing file: " + err.Error())
+				continue
+			}
+		case ".ldjson":
+			err = feedFromLDJSON(file)
+			if err != nil {
+				logger.Log.Warning("Failed processing file: " + err.Error())
+				continue
+			}
+		default:
+			return errors.New("Unsupported file extension: " + ext)
+		}
+	}
+
+	return nil
+}
+
+func feedFromHAR(file string) error {
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		return err
+	}
+
+	var harDoc har.HAR
+	err = json.Unmarshal(data, &harDoc)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range harDoc.Log.Entries {
+		GetOasGeneratorInstance().PushEntry(entry)
+	}
+
+	return nil
+}
+
+func feedFromLDJSON(file string) error {
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	reader := bufio.NewReader(fd)
+
+	var meta map[string]interface{}
+
+	buf := strings.Builder{}
+	for {
+		substr, isPrefix, err := reader.ReadLine()
+		if err == io.EOF {
+			break
+		}
+
+		buf.WriteString(string(substr))
+		if isPrefix {
+			continue
+		}
+
+		line := buf.String()
+		buf.Reset()
+
+		if meta == nil {
+			err := json.Unmarshal([]byte(line), &meta)
+			if err != nil {
+				return err
+			}
+		} else {
+			var entry har.Entry
+			err := json.Unmarshal([]byte(line), &entry)
+			if err != nil {
+				logger.Log.Warningf("Failed decoding entry: %s", line)
+			}
+			GetOasGeneratorInstance().PushEntry(&entry)
+		}
+	}
+
+	return nil
+}
+
+func TestFilesList(t *testing.T) {
+	res, err := getFiles("./test_artifacts/")
+	t.Log(len(res))
+	t.Log(res)
+	if err != nil || len(res) != 2 {
+		t.Logf("Should return 2 files but returned %d", len(res))
+		t.FailNow()
+	}
+}

agent/pkg/oas/gibberish.go

@@ -0,0 +1,108 @@
+package oas
+
+import (
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+var (
+	patBase64   = regexp.MustCompile(`^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$`)
+	patUuid4    = regexp.MustCompile(`(?i)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
+	patEmail    = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
+	patHexLower = regexp.MustCompile(`(0x)?[0-9a-f]{6,}`)
+	patHexUpper = regexp.MustCompile(`(0x)?[0-9A-F]{6,}`)
+)
+
+func IsGibberish(str string) bool {
+	if patBase64.MatchString(str) && len(str) > 32 {
+		return true
+	}
+
+	if patUuid4.MatchString(str) {
+		return true
+	}
+
+	if patEmail.MatchString(str) {
+		return true
+	}
+
+	if patHexLower.MatchString(str) || patHexUpper.MatchString(str) {
+		return true
+	}
+
+	noise := noiseLevel(str)
+	if noise >= 0.2 {
+		return true
+	}
+
+	return false
+}
+
+func noiseLevel(str string) (score float64) {
+	// opinionated algo of certain char pairs marking the non-human strings
+	prev := *new(rune)
+	cnt := 0.0
+	for _, char := range str {
+		cnt += 1
+		if prev > 0 {
+			switch {
+			// continued class of upper/lower/digit adds no noise
+			case unicode.IsUpper(prev) && unicode.IsUpper(char):
+			case unicode.IsLower(prev) && unicode.IsLower(char):
+			case unicode.IsDigit(prev) && unicode.IsDigit(char):
+
+			// upper =>
+			case unicode.IsUpper(prev) && unicode.IsLower(char):
+				score += 0.25
+			case unicode.IsUpper(prev) && unicode.IsDigit(char):
+				score += 0.25
+
+			// lower =>
+			case unicode.IsLower(prev) && unicode.IsUpper(char):
+				score += 0.75
+			case unicode.IsLower(prev) && unicode.IsDigit(char):
+				score += 0.25
+
+			// digit =>
+			case unicode.IsDigit(prev) && unicode.IsUpper(char):
+				score += 0.75
+			case unicode.IsDigit(prev) && unicode.IsLower(char):
+				score += 0.75
+
+			// the rest is 100% noise
+			default:
+				score += 1
+			}
+		}
+		prev = char
+	}
+
+	score /= cnt // weigh it
+
+	return score
+}
+
+func IsVersionString(component string) bool {
+	if component == "" {
+		return false
+	}
+
+	hasV := false
+	if strings.HasPrefix(component, "v") {
+		component = component[1:]
+		hasV = true
+	}
+
+	for _, c := range component {
+		if string(c) != "." && !unicode.IsDigit(c) {
+			return false
+		}
+	}
+
+	if !hasV && strings.Contains(component, ".") {
+		return false
+	}
+
+	return true
+}

agent/pkg/oas/gibberish_test.go

@@ -0,0 +1,71 @@
+package oas
+
+import "testing"
+
+func TestNegative(t *testing.T) {
+	cases := []string{
+		"",
+		"b", // can be valid hexadecimal
+		"GetUniversalVariableUser",
+		"callback",
+		"runs",
+		"tcfv2",
+		"StartUpCheckout",
+		"GetCart",
+	}
+
+	for _, str := range cases {
+		if IsGibberish(str) {
+			t.Errorf("Mistakenly true: %s", str)
+		}
+	}
+}
+
+func TestPositive(t *testing.T) {
+	cases := []string{
+		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
+		"952bea17-3776-11ea-9341-42010a84012a",
+		"456795af-b48f-4a8d-9b37-3e932622c2f0",
+		"0a0d0174-b338-4520-a1c3-24f7e3d5ec50.html",
+		"6120c057c7a97b03f6986f1b",
+		"610bc3fd5a77a7fa25033fb0",
+		"610bd0315a77a7fa25034368",
+		"610bd0315a77a7fa25034368zh",
+		"710a462e",
+		"1554507871",
+		"qwerqwerasdfqwer@protonmai.com",
+		"john.dow.1981@protonmail.com",
+		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
+		"11ca096cbc224a67360493d44a9903",
+		"c738338322370b47a79251f7510dd", // prefixed hex
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
+		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
+		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
+		"730970532670-compute@developer.gserviceaccount.com",
+		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
+		"AAAA028295945",
+		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
+		"n63nd45qsj",
+		"n9z9QGNiz",
+		"proxy.3d2100fd7107262ecb55ce6847f01fa5.html",
+		"r-ext-5579e00a95c90",
+		"r-ext-5579e8b12f11e",
+		"r-v4-5c92513c9e04a",
+		"r-v4-5c92513c9e04a-0-c.monitoring.json",
+		"segments-1563566437171.639994",
+		"t_52d94268-8810-4a7e-ba87-ffd657a6752f",
+		"timeouts-1563566437171.639994",
+
+		// TODO
+		// "fb6cjraf9cejut2a",
+		// "Fxvd1timk", // questionable
+		// "JEHJW4BKVFDRTMTUQLHKK5WVAU",
+	}
+
+	for _, str := range cases {
+		if !IsGibberish(str) {
+			t.Errorf("Mistakenly false: %s", str)
+		}
+	}
+}

agent/pkg/oas/ignores.go

@@ -0,0 +1,75 @@
+package oas
+
+import "strings"
+
+var ignoredExtensions = []string{"gif", "svg", "css", "png", "ico", "js", "woff2", "woff", "jpg", "jpeg", "swf", "ttf", "map", "webp", "otf", "mp3"}
+
+var ignoredCtypePrefixes = []string{"image/", "font/", "video/", "audio/", "text/javascript"}
+var ignoredCtypes = []string{"application/javascript", "application/x-javascript", "text/css", "application/font-woff2", "application/font-woff", "application/x-font-woff"}
+
+var ignoredHeaders = []string{
+	"a-im", "accept",
+	"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-type", "cookie",
+	"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
+	"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
+	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests",
+	"server", "user-agent", "via", "warning", "strict-transport-security",
+	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data",
+	"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
+	"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
+	"x-wap-profile", "x-scheme",
+	"newrelic", "x-cloud-trace-context", "sentry-trace",
+	"expires", "set-cookie", "p3p", "location", "content-security-policy", "content-security-policy-report-only",
+	"last-modified", "content-language",
+	"keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by",
+	"age", "allow", "www-authenticate",
+}
+
+var ignoredHeaderPrefixes = []string{
+	":", "accept-", "access-control-", "if-", "sec-", "grpc-",
+	"x-forwarded-", "x-original-",
+	"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
+	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-",
+}
+
+func isCtypeIgnored(ctype string) bool {
+	for _, prefix := range ignoredCtypePrefixes {
+		if strings.HasPrefix(ctype, prefix) {
+			return true
+		}
+	}
+
+	for _, toIgnore := range ignoredCtypes {
+		if ctype == toIgnore {
+			return true
+		}
+	}
+	return false
+}
+
+func isExtIgnored(path string) bool {
+	for _, extIgn := range ignoredExtensions {
+		if strings.HasSuffix(path, "."+extIgn) {
+			return true
+		}
+	}
+	return false
+}
+
+func isHeaderIgnored(name string) bool {
+	name = strings.ToLower(name)
+
+	for _, ignore := range ignoredHeaders {
+		if name == ignore {
+			return true
+		}
+	}
+
+	for _, prefix := range ignoredHeaderPrefixes {
+		if strings.HasPrefix(name, prefix) {
+			return true
+		}
+	}
+
+	return false
+}

agent/pkg/oas/oas_generator.go

@@ -0,0 +1,107 @@
+package oas
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"net/url"
+	"sync"
+)
+
+var (
+	syncOnce sync.Once
+	instance *oasGenerator
+)
+
+func GetOasGeneratorInstance() *oasGenerator {
+	syncOnce.Do(func() {
+		instance = newOasGenerator()
+		logger.Log.Debug("Oas Generator Initialized")
+	})
+	return instance
+}
+
+func (g *oasGenerator) Start() {
+	if g.started {
+		return
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	g.cancel = cancel
+	g.ctx = ctx
+	g.entriesChan = make(chan har.Entry, 100) // buffer up to 100 entries for OAS processing
+	g.ServiceSpecs = &sync.Map{}
+	g.started = true
+	go instance.runGeneretor()
+}
+
+func (g *oasGenerator) runGeneretor() {
+	for {
+		select {
+		case <-g.ctx.Done():
+			logger.Log.Infof("OAS Generator was canceled")
+			return
+
+		case entry, ok := <-g.entriesChan:
+			if !ok {
+				logger.Log.Infof("OAS Generator - entries channel closed")
+				break
+			}
+			u, err := url.Parse(entry.Request.URL)
+			if err != nil {
+				logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
+			}
+
+			val, found := g.ServiceSpecs.Load(u.Host)
+			var gen *SpecGen
+			if !found {
+				gen = NewGen(u.Scheme + "://" + u.Host)
+				g.ServiceSpecs.Store(u.Host, gen)
+			} else {
+				gen = val.(*SpecGen)
+			}
+
+			opId, err := gen.feedEntry(entry)
+			if err != nil {
+				txt, suberr := json.Marshal(entry)
+				if suberr == nil {
+					logger.Log.Debugf("Problematic entry: %s", txt)
+				}
+
+				logger.Log.Warningf("Failed processing entry: %s", err)
+				continue
+			}
+
+			logger.Log.Debugf("Handled entry %s as opId: %s", entry.Request.URL, opId) // TODO: set opId back to entry?
+		}
+	}
+}
+
+func (g *oasGenerator) PushEntry(entry *har.Entry) {
+	if !g.started {
+		return
+	}
+	select {
+	case g.entriesChan <- *entry:
+	default:
+		logger.Log.Warningf("OAS Generator - entry wasn't sent to channel because the channel has no buffer or there is no receiver")
+	}
+}
+
+func newOasGenerator() *oasGenerator {
+	return &oasGenerator{
+		started:      false,
+		ctx:          nil,
+		cancel:       nil,
+		ServiceSpecs: nil,
+		entriesChan:  nil,
+	}
+}
+
+type oasGenerator struct {
+	started      bool
+	ctx          context.Context
+	cancel       context.CancelFunc
+	ServiceSpecs *sync.Map
+	entriesChan  chan har.Entry
+}

agent/pkg/oas/specgen.go

@@ -0,0 +1,497 @@
+package oas
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/google/uuid"
+	"github.com/up9inc/mizu/shared/logger"
+	"mime"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+type reqResp struct { // hello, generics in Go
+	Req  *har.Request
+	Resp *har.Response
+}
+
+type SpecGen struct {
+	oas  *openapi.OpenAPI
+	tree *Node
+	lock sync.Mutex
+}
+
+func NewGen(server string) *SpecGen {
+	spec := new(openapi.OpenAPI)
+	spec.Version = "3.1.0"
+
+	info := openapi.Info{Title: server}
+	info.Version = "0.0"
+	spec.Info = &info
+	spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
+
+	spec.Servers = make([]*openapi.Server, 0)
+	spec.Servers = append(spec.Servers, &openapi.Server{URL: server})
+
+	gen := SpecGen{oas: spec, tree: new(Node)}
+	return &gen
+}
+
+func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
+	g.oas = oas
+	for pathStr, pathObj := range oas.Paths.Items {
+		pathSplit := strings.Split(string(pathStr), "/")
+		g.tree.getOrSet(pathSplit, pathObj)
+	}
+}
+
+func (g *SpecGen) feedEntry(entry har.Entry) (string, error) {
+	g.lock.Lock()
+	defer g.lock.Unlock()
+
+	opId, err := g.handlePathObj(&entry)
+	if err != nil {
+		return "", err
+	}
+
+	// NOTE: opId can be empty for some failed entries
+	return opId, err
+}
+
+func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
+	g.lock.Lock()
+	defer g.lock.Unlock()
+
+	g.tree.compact()
+
+	for _, pathop := range g.tree.listOps() {
+		if pathop.op.Summary == "" {
+			pathop.op.Summary = pathop.path
+		}
+	}
+
+	// put paths back from tree into OAS
+	g.oas.Paths = g.tree.listPaths()
+
+	suggestTags(g.oas)
+
+	// to make a deep copy, no better idea than marshal+unmarshal
+	specText, err := json.MarshalIndent(g.oas, "", "\t")
+	if err != nil {
+		return nil, err
+	}
+
+	spec := new(openapi.OpenAPI)
+	err = json.Unmarshal(specText, spec)
+	if err != nil {
+		return nil, err
+	}
+
+	return spec, err
+}
+
+func suggestTags(oas *openapi.OpenAPI) {
+	paths := getPathsKeys(oas.Paths.Items)
+	for len(paths) > 0 {
+		group := make([]string, 0)
+		group = append(group, paths[0])
+		paths = paths[1:]
+
+		pathsClone := append(paths[:0:0], paths...)
+		for _, path := range pathsClone {
+			if getSimilarPrefix([]string{group[0], path}) != "" {
+				group = append(group, path)
+				paths = deleteFromSlice(paths, path)
+			}
+		}
+
+		common := getSimilarPrefix(group)
+
+		if len(group) > 1 {
+			for _, path := range group {
+				pathObj := oas.Paths.Items[openapi.PathValue(path)]
+				for _, op := range getOps(pathObj) {
+					if op.Tags == nil {
+						op.Tags = make([]string, 0)
+					}
+					// only add tags if not present
+					if len(op.Tags) == 0 {
+						op.Tags = append(op.Tags, common)
+					}
+				}
+			}
+		}
+
+		//groups[common] = group
+	}
+}
+
+func getSimilarPrefix(strs []string) string {
+	chunked := make([][]string, 0)
+	for _, item := range strs {
+		chunked = append(chunked, strings.Split(item, "/"))
+	}
+
+	cmn := longestCommonXfix(chunked, true)
+	res := make([]string, 0)
+	for _, chunk := range cmn {
+		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
+			res = append(res, chunk)
+		}
+	}
+	return strings.Join(res[1:], ".")
+}
+
+func deleteFromSlice(s []string, val string) []string {
+	temp := s[:0]
+	for _, x := range s {
+		if x != val {
+			temp = append(temp, x)
+		}
+	}
+	return temp
+}
+
+func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
+	keys := make([]string, len(mymap))
+
+	i := 0
+	for k := range mymap {
+		keys[i] = string(k)
+		i++
+	}
+	return keys
+}
+
+func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
+	urlParsed, err := url.Parse(entry.Request.URL)
+	if err != nil {
+		return "", err
+	}
+
+	if isExtIgnored(urlParsed.Path) {
+		logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
+	}
+
+	ctype := getRespCtype(entry.Response)
+	if isCtypeIgnored(ctype) {
+		logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
+	}
+
+	if entry.Response.Status < 100 {
+		logger.Log.Debugf("Dropped traffic entry due to status<100: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	if entry.Response.Status == 301 || entry.Response.Status == 308 {
+		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	split := strings.Split(urlParsed.Path, "/")
+	node := g.tree.getOrSet(split, new(openapi.PathObj))
+	opObj, err := handleOpObj(entry, node.ops)
+
+	if opObj != nil {
+		return opObj.OperationID, err
+	}
+
+	return "", err
+}
+
+func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation, error) {
+	isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
+	opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
+	if err != nil {
+		return nil, err
+	}
+
+	if !isSuccess && wasMissing {
+		logger.Log.Debugf("Dropped traffic entry due to failed status and no known endpoint at: %s", entry.StartedDateTime)
+		return nil, nil
+	}
+
+	err = handleRequest(entry.Request, opObj, isSuccess)
+	if err != nil {
+		return nil, err
+	}
+
+	err = handleResponse(entry.Response, opObj, isSuccess)
+	if err != nil {
+		return nil, err
+	}
+
+	return opObj, nil
+}
+
+func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error {
+	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
+
+	qstrGW := nvParams{
+		In: openapi.InQuery,
+		Pairs: func() []NVPair {
+			return qstrToNVP(req.QueryString)
+		},
+		IsIgnored:      func(name string) bool { return false },
+		GeneralizeName: func(name string) string { return name },
+	}
+	handleNameVals(qstrGW, &opObj.Parameters)
+
+	hdrGW := nvParams{
+		In: openapi.InHeader,
+		Pairs: func() []NVPair {
+			return hdrToNVP(req.Headers)
+		},
+		IsIgnored:      isHeaderIgnored,
+		GeneralizeName: strings.ToLower,
+	}
+	handleNameVals(hdrGW, &opObj.Parameters)
+
+	if req.PostData != nil && req.PostData.Text != "" && isSuccess {
+		reqBody, err := getRequestBody(req, opObj, isSuccess)
+		if err != nil {
+			return err
+		}
+
+		if reqBody != nil {
+			reqCtype := getReqCtype(req)
+			reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, err)
+			if err != nil {
+				return err
+			}
+
+			_ = reqMedia
+		}
+	}
+	return nil
+}
+
+func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool) error {
+	// TODO: we don't support "default" response
+	respObj, err := getResponseObj(resp, opObj, isSuccess)
+	if err != nil {
+		return err
+	}
+
+	handleRespHeaders(resp.Headers, respObj)
+
+	respCtype := getRespCtype(resp)
+	respContent := respObj.Content
+	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, err)
+	if err != nil {
+		return err
+	}
+	_ = respMedia
+	return nil
+}
+
+func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj) {
+	visited := map[string]*openapi.HeaderObj{}
+	for _, pair := range reqHeaders {
+		if isHeaderIgnored(pair.Name) {
+			continue
+		}
+
+		nameGeneral := strings.ToLower(pair.Name)
+
+		initHeaders(respObj)
+		objHeaders := respObj.Headers
+		param := findHeaderByName(&respObj.Headers, pair.Name)
+		if param == nil {
+			param = createHeader(openapi.TypeString)
+			objHeaders[nameGeneral] = param
+		}
+		exmp := &param.Examples
+		err := fillParamExample(&exmp, pair.Value)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+		visited[nameGeneral] = param
+	}
+
+	// maintain "required" flag
+	if respObj.Headers != nil {
+		for name, param := range respObj.Headers {
+			paramObj, err := param.ResolveHeader(headerResolver)
+			if err != nil {
+				logger.Log.Warningf("Failed to resolve param: %s", err)
+				continue
+			}
+
+			_, ok := visited[strings.ToLower(name)]
+			if !ok {
+				flag := false
+				paramObj.Required = &flag
+			}
+		}
+	}
+
+	return
+}
+
+func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err error) (*openapi.MediaType, error) {
+	content, found := respContent[ctype]
+	if !found {
+		respContent[ctype] = &openapi.MediaType{}
+		content = respContent[ctype]
+	}
+
+	var text string
+	if reqResp.Req != nil {
+		text = reqResp.Req.PostData.Text
+	} else {
+		text = decRespText(reqResp.Resp.Content)
+	}
+
+	var exampleMsg []byte
+	// try treating it as json
+	any, isJSON := anyJSON(text)
+	if isJSON {
+		// re-marshal with forced indent
+		exampleMsg, err = json.MarshalIndent(any, "", "\t")
+		if err != nil {
+			panic("Failed to re-marshal value, super-strange")
+		}
+	} else {
+		exampleMsg, err = json.Marshal(text)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	content.Example = exampleMsg
+	return respContent[ctype], nil
+}
+
+func decRespText(content *har.Content) (res string) {
+	res = string(content.Text)
+	if content.Encoding == "base64" {
+		data, err := base64.StdEncoding.DecodeString(res)
+		if err != nil {
+			logger.Log.Warningf("error decoding response text as base64: %s", err)
+		} else {
+			res = string(data)
+		}
+	}
+	return
+}
+
+func getRespCtype(resp *har.Response) string {
+	var ctype string
+	ctype = resp.Content.MimeType
+	for _, hdr := range resp.Headers {
+		if strings.ToLower(hdr.Name) == "content-type" {
+			ctype = hdr.Value
+		}
+	}
+
+	mediaType, _, err := mime.ParseMediaType(ctype)
+	if err != nil {
+		return ""
+	}
+	return mediaType
+}
+
+func getReqCtype(req *har.Request) string {
+	var ctype string
+	ctype = req.PostData.MimeType
+	for _, hdr := range req.Headers {
+		if strings.ToLower(hdr.Name) == "content-type" {
+			ctype = hdr.Value
+		}
+	}
+
+	mediaType, _, err := mime.ParseMediaType(ctype)
+	if err != nil {
+		return ""
+	}
+	return mediaType
+}
+
+func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool) (*openapi.ResponseObj, error) {
+	statusStr := strconv.Itoa(resp.Status)
+
+	var response openapi.Response
+	response, found := opObj.Responses[statusStr]
+	if !found {
+		if opObj.Responses == nil {
+			opObj.Responses = map[string]openapi.Response{}
+		}
+
+		opObj.Responses[statusStr] = &openapi.ResponseObj{Content: map[string]*openapi.MediaType{}}
+		response = opObj.Responses[statusStr]
+	}
+
+	resResponse, err := response.ResolveResponse(responseResolver)
+	if err != nil {
+		return nil, err
+	}
+
+	if isSuccess {
+		resResponse.Description = "Successful call with status " + statusStr
+	} else {
+		resResponse.Description = "Failed call with status " + statusStr
+	}
+	return resResponse, nil
+}
+
+func getRequestBody(req *har.Request, opObj *openapi.Operation, isSuccess bool) (*openapi.RequestBodyObj, error) {
+	if opObj.RequestBody == nil {
+		opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
+	}
+
+	reqBody, err := opObj.RequestBody.ResolveRequestBody(reqBodyResolver)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: maintain required flag for it, but only consider successful responses
+	//reqBody.Content[]
+
+	return reqBody, nil
+}
+
+func getOpObj(pathObj *openapi.PathObj, method string, createIfNone bool) (*openapi.Operation, bool, error) {
+	method = strings.ToLower(method)
+	var op **openapi.Operation
+
+	switch method {
+	case "get":
+		op = &pathObj.Get
+	case "put":
+		op = &pathObj.Put
+	case "post":
+		op = &pathObj.Post
+	case "delete":
+		op = &pathObj.Delete
+	case "options":
+		op = &pathObj.Options
+	case "head":
+		op = &pathObj.Head
+	case "patch":
+		op = &pathObj.Patch
+	case "trace":
+		op = &pathObj.Trace
+	default:
+		return nil, false, errors.New("unsupported HTTP method: " + method)
+	}
+
+	isMissing := false
+	if *op == nil {
+		isMissing = true
+		if createIfNone {
+			*op = &openapi.Operation{Responses: map[string]openapi.Response{}}
+			newUUID := uuid.New().String()
+			(**op).OperationID = newUUID
+		} else {
+			return nil, isMissing, nil
+		}
+	}
+
+	return *op, isMissing, nil
+}

agent/pkg/oas/specgen_test.go

@@ -0,0 +1,197 @@
+package oas
+
+import (
+	"encoding/json"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+)
+
+// if started via env, write file into subdir
+func writeFiles(label string, spec *openapi.OpenAPI) {
+	if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
+		path := "./oas-samples"
+		err := os.MkdirAll(path, 0o755)
+		if err != nil {
+			panic(err)
+		}
+		content, err := json.MarshalIndent(spec, "", "\t")
+		if err != nil {
+			panic(err)
+		}
+		err = ioutil.WriteFile(path+"/"+label+".json", content, 0644)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+func TestEntries(t *testing.T) {
+	files, err := getFiles("./test_artifacts/")
+	// files, err = getFiles("/media/bigdisk/UP9")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	GetOasGeneratorInstance().Start()
+
+	if err := feedEntries(files); err != nil {
+		t.Log(err)
+		t.Fail()
+	}
+
+	loadStartingOAS()
+
+	svcs := strings.Builder{}
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+		gen := val.(*SpecGen)
+		svc := key.(string)
+		svcs.WriteString(svc + ",")
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+			return false
+		}
+
+		err = spec.Validate()
+		if err != nil {
+			specText, _ := json.MarshalIndent(spec, "", "\t")
+			t.Log(string(specText))
+			t.Log(err)
+			t.FailNow()
+		}
+
+		return true
+	})
+
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+		svc := key.(string)
+		gen := val.(*SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		specText, _ := json.MarshalIndent(spec, "", "\t")
+		t.Logf("%s", string(specText))
+
+		err = spec.Validate()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+		writeFiles(svc, spec)
+
+		return true
+	})
+
+}
+
+func TestFileLDJSON(t *testing.T) {
+	GetOasGeneratorInstance().Start()
+	file := "test_artifacts/output_rdwtyeoyrj.har.ldjson"
+	err := feedFromLDJSON(file)
+	if err != nil {
+		logger.Log.Warning("Failed processing file: " + err.Error())
+		t.Fail()
+	}
+
+	loadStartingOAS()
+
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(_, val interface{}) bool {
+		gen := val.(*SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		specText, _ := json.MarshalIndent(spec, "", "\t")
+		t.Logf("%s", string(specText))
+
+		err = spec.Validate()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		return true
+	})
+}
+
+func loadStartingOAS() {
+	file := "test_artifacts/catalogue.json"
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		panic(err)
+	}
+
+	var doc *openapi.OpenAPI
+	err = json.Unmarshal(data, &doc)
+	if err != nil {
+		panic(err)
+	}
+
+	gen := NewGen("catalogue")
+	gen.StartFromSpec(doc)
+
+	GetOasGeneratorInstance().ServiceSpecs.Store("catalogue", gen)
+
+	return
+}
+
+func TestEntriesNegative(t *testing.T) {
+	files := []string{"invalid"}
+	err := feedEntries(files)
+	if err == nil {
+		t.Logf("Should have failed")
+		t.Fail()
+	}
+}
+
+func TestLoadValidHAR(t *testing.T) {
+	inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}`
+	var entry *har.Entry
+	var err = json.Unmarshal([]byte(inp), &entry)
+	if err != nil {
+		t.Logf("Failed to decode entry: %s", err)
+		// t.FailNow() demonstrates the problem of library
+	}
+}
+
+func TestLoadValid3_1(t *testing.T) {
+	fd, err := os.Open("test_artifacts/catalogue.json")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	var oas openapi.OpenAPI
+	err = json.Unmarshal(data, &oas)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	return
+}

agent/pkg/oas/test_artifacts/catalogue.json

@@ -0,0 +1,51 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "Preloaded",
+    "version": "0.1",
+    "description": "Test file for loading pre-existing OAS"
+  },
+  "paths": {
+    "/catalogue/{id}": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ],
+      "get": {
+        "parameters": [        {
+          "name": "non-required-header",
+          "in": "header",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+        ]
+      }
+    },
+    "/catalogue/{id}/details": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "style": "simple",
+          "required": true,
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ]
+    }
+  }
+}

agent/pkg/oas/test_artifacts/output_rdwtyeoyrj.har.ldjson

@@ -0,0 +1,13 @@
+{"messageType": "http", "_source": null, "firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78415179Z", "time": 13, "request": {"method": "GET", "url": "http://catalogue/catalogue/size?tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "x-some", "value": "demo val"},{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [{"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "22"}], "content": {"size": 22, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInNpemUiOjl9", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 22}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 13}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.784918698Z", "time": 19, "request": {"method": "GET", "url": "http://catalogue/catalogue?page=1&size=6&tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "page", "value": "1"}, {"name": "size", "value": "6"}, {"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "1927"}], "content": {"size": 1927, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19LHsiaWQiOiI4MzdhYjE0MS0zOTllLTRjMWYtOWFiYy1iYWNlNDAyOTZiYWMiLCJuYW1lIjoiQ2F0IHNvY2tzIiwiZGVzY3JpcHRpb24iOiJjb25zZXF1YXQgYW1ldCBjdXBpZGF0YXQgbWluaW0gbGFib3J1bSB0ZW1wb3IgZWxpdCBleCBjb25zZXF1YXQgaW4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2NrczIuanBnIl0sInByaWNlIjoxNSwiY291bnQiOjE3NSwidGFnIjpbImJyb3duIiwiZm9ybWFsIiwiZ3JlZW4iXX1dCg==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1927}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 19}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78418182Z", "time": 7, "request": {"method": "GET", "url": "http://catalogue/tags", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "107"}], "content": {"size": 107, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInRhZ3MiOlsiYnJvd24iLCJnZWVrIiwiZm9ybWFsIiwiYmx1ZSIsInNraW4iLCJyZWQiLCJhY3Rpb24iLCJzcG9ydCIsImJsYWNrIiwibWFnaWMiLCJncmVlbiJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 107}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 7}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:18.131501482Z", "time": 5, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}, {"name": "x-some", "value": "demoval"}], ",queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 5}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:18.379836908Z", "time": 14, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 14}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.920540124Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/808a2de1-1aaa-4c25-a9b9-6612e8f29a38", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "275"}], "content": {"size": 275, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NzM4LCJkZXNjcmlwdGlvbiI6IkEgbWF0dXJlIHNvY2ssIGNyb3NzZWQsIHdpdGggYW4gYWlyIG9mIG5vbmNoYWxhbmNlLiIsImlkIjoiODA4YTJkZTEtMWFhYS00YzI1LWE5YjktNjYxMmU4ZjI5YTM4IiwiaW1hZ2VVcmwiOlsiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMS5qcGVnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMi5qcGVnIl0sIm5hbWUiOiJDcm9zc2VkIiwicHJpY2UiOjE3LjMyLCJ0YWciOlsiYmx1ZSIsInJlZCIsImFjdGlvbiIsImZvcm1hbCJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 275}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.921609501Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?sort=id&size=3&tags=blue", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "3"}, {"name": "tags", "value": "blue"}, {"name": "sort", "value": "id"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "789"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}], "content": {"size": 789, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJuYW1lIjoiQ29sb3VyZnVsIiwiZGVzY3JpcHRpb24iOiJwcm9pZGVudCBvY2NhZWNhdCBpcnVyZSBldCBleGNlcHRldXIgbGFib3JlIG1pbmltIG5pc2kgYW1ldCBpcnVyZSIsImltYWdlVXJsIjpbIi9jYXRhbG9ndWUvaW1hZ2VzL2NvbG91cmZ1bF9zb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIl0sInByaWNlIjoxOCwiY291bnQiOjQzOCwidGFnIjpbImJsdWUiXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiYmx1ZSJdfV0K", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 789}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.923197848Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:23.175549218Z", "time": 26, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 26}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.239777333Z", "time": 10, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Transfer-Encoding", "value": "chunked"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "X-Application-Context", "value": "carts:80"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 10}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.725866772Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?size=5", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "5"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "1643"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}], "content": {"size": 1643, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19XQo=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1643}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.729303217Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}

agent/pkg/oas/tree.go

@@ -0,0 +1,205 @@
+package oas
+
+import (
+	"github.com/chanced/openapi"
+	"github.com/up9inc/mizu/shared/logger"
+	"strconv"
+	"strings"
+)
+
+type NodePath = []string
+
+type Node struct {
+	constant *string
+	param    *openapi.ParameterObj
+	ops      *openapi.PathObj
+	parent   *Node
+	children []*Node
+}
+
+func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Node) {
+	if pathObjToSet == nil {
+		panic("Invalid function call")
+	}
+
+	pathChunk := path[0]
+	chunkIsParam := strings.HasPrefix(pathChunk, "{") && strings.HasSuffix(pathChunk, "}")
+	chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
+
+	var paramObj *openapi.ParameterObj
+	if chunkIsParam && pathObjToSet != nil {
+		paramObj = findParamByName(pathObjToSet.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
+	}
+
+	if paramObj == nil {
+		node = n.searchInConstants(pathChunk)
+	}
+
+	if node == nil {
+		node = n.searchInParams(paramObj, chunkIsGibberish)
+	}
+
+	// still no node found, should create it
+	if node == nil {
+		node = new(Node)
+		node.parent = n
+		n.children = append(n.children, node)
+
+		if paramObj != nil {
+			node.param = paramObj
+		} else if chunkIsGibberish {
+			initParams(&pathObjToSet.Parameters)
+
+			newParam := n.createParam()
+			node.param = newParam
+
+			appended := append(*pathObjToSet.Parameters, newParam)
+			pathObjToSet.Parameters = &appended
+		} else {
+			node.constant = &pathChunk
+		}
+	}
+
+	// add example if it's a param
+	if node.param != nil && !chunkIsParam {
+		exmp := &node.param.Examples
+		err := fillParamExample(&exmp, pathChunk)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+	}
+
+	// TODO: eat up trailing slash, in a smart way: node.ops!=nil && path[1]==""
+	if len(path) > 1 {
+		return node.getOrSet(path[1:], pathObjToSet)
+	} else if node.ops == nil {
+		node.ops = pathObjToSet
+	}
+
+	return node
+}
+
+func (n *Node) createParam() *openapi.ParameterObj {
+	name := "param"
+
+	// REST assumption, not always correct
+	if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
+		name = *n.constant
+		name = name[:len(name)-2] + "Id"
+	} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
+		name = *n.constant
+		name = name[:len(name)-1] + "Id"
+	}
+
+	newParam := createSimpleParam(name, "path", "string")
+	x := n.countParentParams()
+	if x > 1 {
+		newParam.Name = newParam.Name + strconv.Itoa(x)
+	}
+
+	return newParam
+}
+
+func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunkIsGibberish bool) *Node {
+	// look among params
+	if paramObj != nil || chunkIsGibberish {
+		for _, subnode := range n.children {
+			if subnode.constant != nil {
+				continue
+			}
+
+			// TODO: check the regex pattern of param? for exceptions etc
+
+			if paramObj != nil {
+				// TODO: mergeParam(subnode.param, paramObj)
+				return subnode
+			} else {
+				return subnode
+			}
+		}
+	}
+	return nil
+}
+
+func (n *Node) searchInConstants(pathChunk string) *Node {
+	// look among constants
+	for _, subnode := range n.children {
+		if subnode.constant == nil {
+			continue
+		}
+
+		if *subnode.constant == pathChunk {
+			return subnode
+		}
+	}
+	return nil
+}
+
+func (n *Node) compact() {
+	// TODO
+}
+
+func (n *Node) listPaths() *openapi.Paths {
+	paths := &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
+
+	var strChunk string
+	if n.constant != nil {
+		strChunk = *n.constant
+	} else if n.param != nil {
+		strChunk = "{" + n.param.Name + "}"
+	} else {
+		// this is the root node
+	}
+
+	// add self
+	if n.ops != nil {
+		paths.Items[openapi.PathValue(strChunk)] = n.ops
+	}
+
+	// recurse into children
+	for _, child := range n.children {
+		subPaths := child.listPaths()
+		for path, pathObj := range subPaths.Items {
+			var concat string
+			if n.parent == nil {
+				concat = string(path)
+			} else {
+				concat = strChunk + "/" + string(path)
+			}
+			paths.Items[openapi.PathValue(concat)] = pathObj
+		}
+	}
+
+	return paths
+}
+
+type PathAndOp struct {
+	path string
+	op   *openapi.Operation
+}
+
+func (n *Node) listOps() []PathAndOp {
+	res := make([]PathAndOp, 0)
+	for path, pathObj := range n.listPaths().Items {
+		for _, op := range getOps(pathObj) {
+			res = append(res, PathAndOp{path: string(path), op: op})
+		}
+	}
+	return res
+}
+
+func (n *Node) countParentParams() int {
+	res := 0
+	node := n
+	for {
+		if node.param != nil {
+			res++
+		}
+
+		if node.parent == nil {
+			break
+		}
+		node = node.parent
+	}
+	return res
+}

agent/pkg/oas/utils.go

@@ -0,0 +1,344 @@
+package oas
+
+import (
+	"encoding/json"
+	"errors"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"strconv"
+	"strings"
+)
+
+func exampleResolver(ref string) (*openapi.ExampleObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func responseResolver(ref string) (*openapi.ResponseObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func reqBodyResolver(ref string) (*openapi.RequestBodyObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func paramResolver(ref string) (*openapi.ParameterObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func headerResolver(ref string) (*openapi.HeaderObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func initParams(obj **openapi.ParameterList) {
+	if *obj == nil {
+		var params openapi.ParameterList
+		params = make([]openapi.Parameter, 0)
+		*obj = &params
+	}
+}
+
+func initHeaders(respObj *openapi.ResponseObj) {
+	if respObj.Headers == nil {
+		var created openapi.Headers
+		created = map[string]openapi.Header{}
+		respObj.Headers = created
+	}
+}
+
+func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *openapi.ParameterObj {
+	if name == "" {
+		panic("Cannot create parameter with empty name")
+	}
+	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
+	schema := new(openapi.SchemaObj)
+	schema.Type = make(openapi.Types, 0)
+	schema.Type = append(schema.Type, ptype)
+
+	style := openapi.StyleSimple
+	if in == openapi.InQuery {
+		style = openapi.StyleForm
+	}
+
+	newParam := openapi.ParameterObj{
+		Name:     name,
+		In:       in,
+		Style:    string(style),
+		Examples: map[string]openapi.Example{},
+		Schema:   schema,
+		Required: &required,
+	}
+	return &newParam
+}
+
+func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (pathParam *openapi.ParameterObj) {
+	caseInsensitive := in == openapi.InHeader
+	for _, param := range *params {
+		paramObj, err := param.ResolveParameter(paramResolver)
+		if err != nil {
+			logger.Log.Warningf("Failed to resolve reference: %s", err)
+			continue
+		}
+
+		if paramObj.In != in {
+			continue
+		}
+
+		if paramObj.Name == name || (caseInsensitive && strings.ToLower(paramObj.Name) == strings.ToLower(name)) {
+			pathParam = paramObj
+			break
+		}
+	}
+	return pathParam
+}
+
+func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj {
+	for hname, param := range *headers {
+		hdrObj, err := param.ResolveHeader(headerResolver)
+		if err != nil {
+			logger.Log.Warningf("Failed to resolve reference: %s", err)
+			continue
+		}
+
+		if strings.ToLower(hname) == strings.ToLower(name) {
+			return hdrObj
+		}
+	}
+	return nil
+}
+
+type NVPair struct {
+	Name  string
+	Value string
+}
+
+type nvParams struct {
+	In             openapi.In
+	Pairs          func() []NVPair
+	IsIgnored      func(name string) bool
+	GeneralizeName func(name string) string
+}
+
+func qstrToNVP(list []har.QueryString) []NVPair {
+	res := make([]NVPair, len(list))
+	for idx, val := range list {
+		res[idx] = NVPair{Name: val.Name, Value: val.Value}
+	}
+	return res
+}
+
+func hdrToNVP(list []har.Header) []NVPair {
+	res := make([]NVPair, len(list))
+	for idx, val := range list {
+		res[idx] = NVPair{Name: val.Name, Value: val.Value}
+	}
+	return res
+}
+
+func handleNameVals(gw nvParams, params **openapi.ParameterList) {
+	visited := map[string]*openapi.ParameterObj{}
+	for _, pair := range gw.Pairs() {
+		if gw.IsIgnored(pair.Name) {
+			continue
+		}
+
+		nameGeneral := gw.GeneralizeName(pair.Name)
+
+		initParams(params)
+		param := findParamByName(*params, gw.In, pair.Name)
+		if param == nil {
+			param = createSimpleParam(nameGeneral, gw.In, openapi.TypeString)
+			appended := append(**params, param)
+			*params = &appended
+		}
+		exmp := &param.Examples
+		err := fillParamExample(&exmp, pair.Value)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+		visited[nameGeneral] = param
+	}
+
+	// maintain "required" flag
+	if *params != nil {
+		for _, param := range **params {
+			paramObj, err := param.ResolveParameter(paramResolver)
+			if err != nil {
+				logger.Log.Warningf("Failed to resolve param: %s", err)
+				continue
+			}
+			if paramObj.In != gw.In {
+				continue
+			}
+
+			_, ok := visited[strings.ToLower(paramObj.Name)]
+			if !ok {
+				flag := false
+				paramObj.Required = &flag
+			}
+		}
+	}
+}
+
+func createHeader(ptype openapi.SchemaType) *openapi.HeaderObj {
+	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
+	schema := new(openapi.SchemaObj)
+	schema.Type = make(openapi.Types, 0)
+	schema.Type = append(schema.Type, ptype)
+
+	style := openapi.StyleSimple
+	newParam := openapi.HeaderObj{
+		Style:    string(style),
+		Examples: map[string]openapi.Example{},
+		Schema:   schema,
+		Required: &required,
+	}
+	return &newParam
+}
+
+func fillParamExample(param **openapi.Examples, exampleValue string) error {
+	if **param == nil {
+		**param = map[string]openapi.Example{}
+	}
+
+	cnt := 0
+	for _, example := range **param {
+		cnt++
+		exampleObj, err := example.ResolveExample(exampleResolver)
+		if err != nil {
+			continue
+		}
+
+		var value string
+		err = json.Unmarshal(exampleObj.Value, &value)
+		if err != nil {
+			logger.Log.Warningf("Failed decoding parameter example into string: %s", err)
+			continue
+		}
+
+		if value == exampleValue || cnt > 5 { // 5 examples is enough
+			return nil
+		}
+	}
+
+	valMsg, err := json.Marshal(exampleValue)
+	if err != nil {
+		return err
+	}
+
+	themap := **param
+	themap["example #"+strconv.Itoa(cnt)] = &openapi.ExampleObj{Value: valMsg}
+
+	return nil
+}
+
+func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.com/jpillora/longestcommon
+	empty := make([]string, 0)
+	//short-circuit empty list
+	if len(strs) == 0 {
+		return empty
+	}
+	xfix := strs[0]
+	//short-circuit single-element list
+	if len(strs) == 1 {
+		return xfix
+	}
+	//compare first to rest
+	for _, str := range strs[1:] {
+		xfixl := len(xfix)
+		strl := len(str)
+		//short-circuit empty strings
+		if xfixl == 0 || strl == 0 {
+			return empty
+		}
+		//maximum possible length
+		maxl := xfixl
+		if strl < maxl {
+			maxl = strl
+		}
+		//compare letters
+		if pre {
+			//prefix, iterate left to right
+			for i := 0; i < maxl; i++ {
+				if xfix[i] != str[i] {
+					xfix = xfix[:i]
+					break
+				}
+			}
+		} else {
+			//suffix, iternate right to left
+			for i := 0; i < maxl; i++ {
+				xi := xfixl - i - 1
+				si := strl - i - 1
+				if xfix[xi] != str[si] {
+					xfix = xfix[xi+1:]
+					break
+				}
+			}
+		}
+	}
+	return xfix
+}
+
+// returns all non-nil ops in PathObj
+func getOps(pathObj *openapi.PathObj) []*openapi.Operation {
+	ops := []**openapi.Operation{&pathObj.Get, &pathObj.Patch, &pathObj.Put, &pathObj.Options, &pathObj.Post, &pathObj.Trace, &pathObj.Head, &pathObj.Delete}
+	res := make([]*openapi.Operation, 0)
+	for _, opp := range ops {
+		if *opp == nil {
+			continue
+		}
+		res = append(res, *opp)
+	}
+	return res
+}
+
+// parses JSON into any possible value
+func anyJSON(text string) (anyVal interface{}, isJSON bool) {
+	isJSON = true
+	asMap := map[string]interface{}{}
+	err := json.Unmarshal([]byte(text), &asMap)
+	if err == nil && asMap != nil {
+		return asMap, isJSON
+	}
+
+	asArray := make([]interface{}, 0)
+	err = json.Unmarshal([]byte(text), &asArray)
+	if err == nil && asArray != nil {
+		return asArray, isJSON
+	}
+
+	asString := ""
+	sPtr := &asString
+	err = json.Unmarshal([]byte(text), &sPtr)
+	if err == nil && sPtr != nil {
+		return asString, isJSON
+	}
+
+	asInt := 0
+	intPtr := &asInt
+	err = json.Unmarshal([]byte(text), &intPtr)
+	if err == nil && intPtr != nil {
+		return asInt, isJSON
+	}
+
+	asFloat := 0.0
+	floatPtr := &asFloat
+	err = json.Unmarshal([]byte(text), &floatPtr)
+	if err == nil && floatPtr != nil {
+		return asFloat, isJSON
+	}
+
+	asBool := false
+	boolPtr := &asBool
+	err = json.Unmarshal([]byte(text), &boolPtr)
+	if err == nil && boolPtr != nil {
+		return asBool, isJSON
+	}
+
+	if text == "null" {
+		return nil, isJSON
+	}
+
+	return nil, false
+}

agent/pkg/oas/utils_test.go

@@ -0,0 +1,35 @@
+package oas
+
+import (
+	"testing"
+)
+
+func TestAnyJSON(t *testing.T) {
+	testCases := []struct {
+		inp    string
+		isJSON bool
+		out    interface{}
+	}{
+		{`{"key": 1, "keyNull": null}`, true, nil},
+		{`[{"key": "val"}, ["subarray"], "string", 1, 2.2, true, null]`, true, nil},
+		{`"somestring"`, true, "somestring"},
+		{"0", true, 0},
+		{"0.5", true, 0.5},
+		{"true", true, true},
+		{"null", true, nil},
+		{"sabbra cadabra", false, nil},
+		{"0.1.2.3", false, nil},
+	}
+	for _, tc := range testCases {
+		any, isJSON := anyJSON(tc.inp)
+		if isJSON != tc.isJSON {
+			t.Errorf("Parse flag mismatch: %t != %t", tc.isJSON, isJSON)
+		} else if isJSON && tc.out != nil && tc.out != any {
+			t.Errorf("%s != %s", any, tc.out)
+		} else if tc.inp == "null" && any != nil {
+			t.Errorf("null has to parse as nil (but got %s)", any)
+		} else {
+			t.Logf("%s => %s", tc.inp, any)
+		}
+	}
+}

agent/pkg/providers/kubernetes_provider.go

@@ -0,0 +1,27 @@
+package providers
+
+import (
+	"github.com/up9inc/mizu/shared/kubernetes"
+	"sync"
+)
+
+var lock = &sync.Mutex{}
+
+var kubernetesProvider *kubernetes.Provider
+
+func GetKubernetesProvider() (*kubernetes.Provider, error) {
+	if kubernetesProvider == nil {
+		lock.Lock()
+		defer lock.Unlock()
+
+		if kubernetesProvider == nil {
+			var err error
+			kubernetesProvider, err = kubernetes.NewProviderInCluster()
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return kubernetesProvider, nil
+}

agent/pkg/providers/status_provider.go

@@ -8,19 +8,14 @@ import (
 	"github.com/up9inc/mizu/tap"
 	"mizuserver/pkg/models"
 	"os"
-	"sync"
 	"time"
 )
 
 const tlsLinkRetainmentTime = time.Minute * 15
 
 var (
-	TappersCount         int
-	TapStatus            shared.TapStatus
-	TappersStatus        map[string]shared.TapperStatus
-	authStatus           *models.AuthStatus
-	RecentTLSLinks       = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
-	tappersCountLock     = sync.Mutex{}
+	authStatus     *models.AuthStatus
+	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
 )
 
 func GetAuthStatus() (*models.AuthStatus, error) {
@@ -68,15 +63,3 @@ func GetAllRecentTLSAddresses() []string {
 
 	return recentTLSLinks
 }
-
-func TapperAdded() {
-	tappersCountLock.Lock()
-	TappersCount++
-	tappersCountLock.Unlock()
-}
-
-func TapperRemoved() {
-	tappersCountLock.Lock()
-	TappersCount--
-	tappersCountLock.Unlock()
-}

agent/pkg/providers/tapConfig/tap_config_provider.go

@@ -0,0 +1,42 @@
+package tapConfig
+
+import (
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/utils"
+	"os"
+	"sync"
+)
+
+const FilePath = shared.DataDirPath + "tap-config.json"
+
+var (
+	lock     = &sync.Mutex{}
+	syncOnce sync.Once
+	config   *models.TapConfig
+)
+
+func Get() *models.TapConfig {
+	syncOnce.Do(func() {
+		if err := utils.ReadJsonFile(FilePath, &config); err != nil {
+			config = &models.TapConfig{TappedNamespaces: make(map[string]bool)}
+
+			if !os.IsNotExist(err) {
+				logger.Log.Errorf("Error reading tap config from file, err: %v", err)
+			}
+		}
+	})
+
+	return config
+}
+
+func Save(tapConfigToSave *models.TapConfig) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	config = tapConfigToSave
+	if err := utils.SaveJsonFile(FilePath, config); err != nil {
+		logger.Log.Errorf("Error saving tap config, err: %v", err)
+	}
+}

agent/pkg/providers/tappedPods/tapped_pods_provider.go

@@ -0,0 +1,56 @@
+package tappedPods
+
+import (
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/providers/tappers"
+	"mizuserver/pkg/utils"
+	"os"
+	"strings"
+	"sync"
+)
+
+const FilePath = shared.DataDirPath + "tapped-pods.json"
+
+var (
+	lock       = &sync.Mutex{}
+	syncOnce   sync.Once
+	tappedPods []*shared.PodInfo
+)
+
+func Get() []*shared.PodInfo {
+	syncOnce.Do(func() {
+		if err := utils.ReadJsonFile(FilePath, &tappedPods); err != nil {
+			if !os.IsNotExist(err) {
+				logger.Log.Errorf("Error reading tapped pods from file, err: %v", err)
+			}
+		}
+	})
+
+	return tappedPods
+}
+
+func Set(tappedPodsToSet []*shared.PodInfo) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	tappedPods = tappedPodsToSet
+	if err := utils.SaveJsonFile(FilePath, tappedPods); err != nil {
+		logger.Log.Errorf("Error saving tapped pods, err: %v", err)
+	}
+}
+
+func GetTappedPodsStatus() []shared.TappedPodStatus {
+	tappedPodsStatus := make([]shared.TappedPodStatus, 0)
+	for _, pod := range Get() {
+		var status string
+		if tapperStatus, ok := tappers.GetStatus()[pod.NodeName]; ok {
+			status = strings.ToLower(tapperStatus.Status)
+		}
+
+		isTapped := status == "running"
+		tappedPodsStatus = append(tappedPodsStatus, shared.TappedPodStatus{Name: pod.Name, Namespace: pod.Namespace, IsTapped: isTapped})
+	}
+
+	return tappedPodsStatus
+}

agent/pkg/providers/tappers/tappers_provider.go

@@ -0,0 +1,82 @@
+package tappers
+
+import (
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/utils"
+	"os"
+	"sync"
+)
+
+const FilePath = shared.DataDirPath + "tappers-status.json"
+
+var (
+	lockStatus = &sync.Mutex{}
+	syncOnce   sync.Once
+	status     map[string]*shared.TapperStatus
+
+	lockConnectedCount = &sync.Mutex{}
+	connectedCount     int
+)
+
+func GetStatus() map[string]*shared.TapperStatus {
+	initStatus()
+
+	return status
+}
+
+func SetStatus(tapperStatus *shared.TapperStatus) {
+	initStatus()
+
+	lockStatus.Lock()
+	defer lockStatus.Unlock()
+
+	status[tapperStatus.NodeName] = tapperStatus
+
+	saveStatus()
+}
+
+func ResetStatus() {
+	lockStatus.Lock()
+	defer lockStatus.Unlock()
+
+	status = make(map[string]*shared.TapperStatus)
+
+	saveStatus()
+}
+
+func GetConnectedCount() int {
+	return connectedCount
+}
+
+func Connected() {
+	lockConnectedCount.Lock()
+	defer lockConnectedCount.Unlock()
+
+	connectedCount++
+}
+
+func Disconnected() {
+	lockConnectedCount.Lock()
+	defer lockConnectedCount.Unlock()
+
+	connectedCount--
+}
+
+func initStatus() {
+	syncOnce.Do(func() {
+		if err := utils.ReadJsonFile(FilePath, &status); err != nil {
+			status = make(map[string]*shared.TapperStatus)
+
+			if !os.IsNotExist(err) {
+				logger.Log.Errorf("Error reading tappers status from file, err: %v", err)
+			}
+		}
+	})
+}
+
+func saveStatus() {
+	if err := utils.SaveJsonFile(FilePath, status); err != nil {
+		logger.Log.Errorf("Error saving tappers status, err: %v", err)
+	}
+}

agent/pkg/providers/user_provider.go

@@ -108,7 +108,7 @@ func AnyUserExists(ctx context.Context) (bool, error) {
 
 func Logout(token string, ctx context.Context) error {
 	logoutRequest := client.V0alpha2Api.SubmitSelfServiceLogoutFlowWithoutBrowser(ctx)
-	logoutRequest.SubmitSelfServiceLogoutFlowWithoutBrowserBody(ory.SubmitSelfServiceLogoutFlowWithoutBrowserBody{
+	logoutRequest = logoutRequest.SubmitSelfServiceLogoutFlowWithoutBrowserBody(ory.SubmitSelfServiceLogoutFlowWithoutBrowserBody{
 		SessionToken: token,
 	})
 	if response, err := logoutRequest.Execute(); err != nil {

agent/pkg/routes/oas_routes.go

@@ -0,0 +1,18 @@
+package routes
+
+import (
+	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/middlewares"
+
+	"github.com/gin-gonic/gin"
+)
+
+// OASRoutes methods to access OAS spec
+func OASRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/oas")
+	routeGroup.Use(middlewares.RequiresAuth())
+
+	routeGroup.GET("/", controllers.GetOASServers)     // list of servers in OAS map
+	routeGroup.GET("/all", controllers.GetOASAllSpecs) // list of servers in OAS map
+	routeGroup.GET("/:id", controllers.GetOASSpec)     // get OAS spec for given server
+}

agent/pkg/routes/status_routes.go

@@ -15,7 +15,7 @@ func StatusRoutes(ginApp *gin.Engine) {
 
 	routeGroup.POST("/tappedPods", controllers.PostTappedPods)
 	routeGroup.POST("/tapperStatus", controllers.PostTapperStatus)
-	routeGroup.GET("/tappersCount", controllers.GetTappersCount)
+	routeGroup.GET("/connectedTappersCount", controllers.GetConnectedTappersCount)
 	routeGroup.GET("/tap", controllers.GetTappingStatus)
 
 	routeGroup.GET("/auth", controllers.GetAuthStatus)

agent/pkg/up9/main.go

@@ -243,7 +243,7 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 			var dataMap map[string]interface{}
 			err = json.Unmarshal(dataBytes, &dataMap)
 
-			var entry tapApi.MizuEntry
+			var entry tapApi.Entry
 			if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil {
 				continue
 			}

agent/pkg/utils/utils.go

@@ -2,13 +2,13 @@ package utils
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/providers"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
 	"os/signal"
-	"strings"
 	"syscall"
 	"time"
 
@@ -45,16 +45,6 @@ func StartServer(app *gin.Engine) {
 	}
 }
 
-func GetTappedPodsStatus() []shared.TappedPodStatus {
-	tappedPodsStatus := make([]shared.TappedPodStatus, 0)
-	for _, pod := range providers.TapStatus.Pods {
-		status := strings.ToLower(providers.TappersStatus[pod.NodeName].Status)
-		isTapped := status == "running"
-		tappedPodsStatus = append(tappedPodsStatus, shared.TappedPodStatus{Name: pod.Name, Namespace: pod.Namespace, IsTapped: isTapped})
-	}
-	return tappedPodsStatus
-}
-
 func CheckErr(e error) {
 	if e != nil {
 		logger.Log.Errorf("%v", e)
@@ -71,3 +61,27 @@ func SetHostname(address, newHostname string) string {
 	return replacedUrl.String()
 
 }
+
+func ReadJsonFile(filePath string, value interface{}) error {
+	if content, err := ioutil.ReadFile(filePath); err != nil {
+		return err
+	} else {
+		if err = json.Unmarshal(content, value); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func SaveJsonFile(filePath string, value interface{}) error {
+	if data, err := json.Marshal(value); err != nil {
+		return err
+	} else {
+		if err = ioutil.WriteFile(filePath, data, 0644); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cli/Makefile

@@ -14,8 +14,12 @@ help: ## This help.
 install:
 	go install mizu.go
 
+build-debug:
+	export GCLFAGS='-gcflags="all=-N -l"'
+	${MAKE} build
+
 build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
-	go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
+	go build ${GCLFAGS} -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.Platform=$(SUFFIX)' \

cli/apiserver/provider.go

@@ -87,9 +87,8 @@ func (provider *Provider) ReportTappedPods(pods []core.Pod) error {
 	tappedPodsUrl := fmt.Sprintf("%s/status/tappedPods", provider.url)
 
 	podInfos := kubernetes.GetPodInfosForPods(pods)
-	tapStatus := shared.TapStatus{Pods: podInfos}
 
-	if jsonValue, err := json.Marshal(tapStatus); err != nil {
+	if jsonValue, err := json.Marshal(podInfos); err != nil {
 		return fmt.Errorf("failed Marshal the tapped pods %w", err)
 	} else {
 		if response, err := provider.client.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {

cli/cmd/installRunner.go

@@ -4,6 +4,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/up9inc/mizu/shared/kubernetes"
+	core "k8s.io/api/core/v1"
+	"regexp"
+	"time"
 
 	"github.com/creasty/defaults"
 	"github.com/up9inc/mizu/cli/config"
@@ -58,7 +62,21 @@ func runMizuInstall() {
 		return
 	}
 
-	logger.Log.Infof(uiUtils.Magenta, "Created Mizu Agent components, run `mizu view` to connect to the mizu daemon instance")
+	logger.Log.Infof("Waiting for Mizu server to start...")
+	readyChan := make(chan string)
+	readyErrorChan := make(chan error)
+	go watchApiServerPodReady(ctx, kubernetesProvider, readyChan, readyErrorChan)
+
+	select {
+	case readyMessage := <-readyChan:
+		logger.Log.Infof(readyMessage)
+	case err := <-readyErrorChan:
+		defer resources.CleanUpMizuResources(ctx, cancel, kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("%v", errormessage.FormatError(err)))
+		return
+	}
+
+	logger.Log.Infof(uiUtils.Magenta, "Installation completed, run `mizu view` to connect to the mizu daemon instance")
 }
 
 func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Resources) *shared.MizuAgentConfig {
@@ -71,7 +89,64 @@ func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Reso
 		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
 		AgentDatabasePath:      shared.DataDirPath,
 		StandaloneMode:         true,
+		ServiceMap:             config.Config.ServiceMap,
+		OAS:                    config.Config.OAS,
 	}
 
 	return &mizuAgentConfig
 }
+
+func watchApiServerPodReady(ctx context.Context, kubernetesProvider *kubernetes.Provider, readyChan chan string, readyErrorChan chan error) {
+	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", kubernetes.ApiServerPodName))
+	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
+	eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper)
+
+	timeAfter := time.After(1 * time.Minute)
+	for {
+		select {
+		case wEvent, ok := <-eventChan:
+			if !ok {
+				eventChan = nil
+				continue
+			}
+
+			switch wEvent.Type {
+			case kubernetes.EventAdded:
+				logger.Log.Debugf("Watching API Server pod ready loop, added")
+			case kubernetes.EventDeleted:
+				logger.Log.Debugf("Watching API Server pod ready loop, %s removed", kubernetes.ApiServerPodName)
+			case kubernetes.EventModified:
+				modifiedPod, err := wEvent.ToPod()
+				if err != nil {
+					readyErrorChan <- err
+					return
+				}
+
+				logger.Log.Debugf("Watching API Server pod ready loop, modified: %v", modifiedPod.Status.Phase)
+
+				if modifiedPod.Status.Phase == core.PodRunning {
+					readyChan <- fmt.Sprintf("%v pod is running", modifiedPod.Name)
+					return
+				}
+			case kubernetes.EventBookmark:
+				break
+			case kubernetes.EventError:
+				break
+			}
+		case err, ok := <-errorChan:
+			if !ok {
+				errorChan = nil
+				continue
+			}
+
+			readyErrorChan <- fmt.Errorf("[ERROR] Agent creation, watching %v namespace, error: %v", config.Config.MizuResourcesNamespace, err)
+			return
+		case <-timeAfter:
+			readyErrorChan <- fmt.Errorf("mizu API server was not ready in time")
+			return
+		case <-ctx.Done():
+			logger.Log.Debugf("Watching API Server pod ready loop, ctx done")
+			return
+		}
+	}
+}

cli/cmd/tap.go

@@ -119,5 +119,5 @@ func init() {
 	tapCmd.Flags().StringP(configStructs.WorkspaceTapName, "w", defaultTapConfig.Workspace, "Uploads traffic to your UP9 workspace for further analysis (requires auth)")
 	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules")
 	tapCmd.Flags().String(configStructs.ContractFile, defaultTapConfig.ContractFile, "OAS/Swagger file to validate to monitor the contracts")
-	tapCmd.Flags().Bool(configStructs.IstioName, defaultTapConfig.Istio, "Record decrypted traffic if the cluster configured with istio and mtls")
+	tapCmd.Flags().Bool(configStructs.ServiceMeshName, defaultTapConfig.ServiceMesh, "Record decrypted traffic if the cluster is configured with a service mesh and with mtls")
 }

cli/cmd/tapRunner.go

@@ -156,6 +156,8 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 		TapperResources:        config.Config.Tap.TapperResources,
 		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
 		AgentDatabasePath:      shared.DataDirPath,
+		ServiceMap:             config.Config.ServiceMap,
+		OAS:                    config.Config.OAS,
 	}
 
 	return &mizuAgentConfig
@@ -192,7 +194,7 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider
 		IgnoredUserAgents:        config.Config.Tap.IgnoredUserAgents,
 		MizuApiFilteringOptions:  mizuApiFilteringOptions,
 		MizuServiceAccountExists: state.mizuServiceAccountExists,
-		Istio:                    config.Config.Tap.Istio,
+		ServiceMesh:              config.Config.Tap.ServiceMesh,
 	}, startTime)
 
 	if err != nil {

cli/config/configStruct.go

@@ -34,9 +34,11 @@ type ConfigStruct struct {
 	ConfigFilePath         string                      `yaml:"config-path,omitempty" readonly:""`
 	HeadlessMode           bool                        `yaml:"headless" default:"false"`
 	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
+	ServiceMap             bool                        `yaml:"service-map,omitempty" default:"false" readonly:""`
+	OAS                    bool                        `yaml:"oas,omitempty" default:"false" readonly:""`
 }
 
-func(config *ConfigStruct) validate() error {
+func (config *ConfigStruct) validate() error {
 	if _, err := logging.LogLevel(config.LogLevelStr); err != nil {
 		return fmt.Errorf("%s is not a valid log level, err: %v", config.LogLevelStr, err)
 	}

cli/config/configStructs/tapConfig.go

@@ -22,7 +22,7 @@ const (
 	WorkspaceTapName              = "workspace"
 	EnforcePolicyFile             = "traffic-validation-file"
 	ContractFile                  = "contract"
-	IstioName                     = "istio"
+	ServiceMeshName               = "service-mesh"
 )
 
 type TapConfig struct {
@@ -44,7 +44,7 @@ type TapConfig struct {
 	AskUploadConfirmation   bool             `yaml:"ask-upload-confirmation" default:"true"`
 	ApiServerResources      shared.Resources `yaml:"api-server-resources"`
 	TapperResources         shared.Resources `yaml:"tapper-resources"`
-	Istio                   bool             `yaml:"istio" default:"false"`
+	ServiceMesh             bool             `yaml:"service-mesh" default:"false"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {

cli/mizu/fsUtils/mizuLogsUtils.go

@@ -38,18 +38,20 @@ func DumpLogs(ctx context.Context, provider *kubernetes.Provider, filePath strin
 	defer zipWriter.Close()
 
 	for _, pod := range pods {
-		logs, err := provider.GetPodLogs(ctx, pod.Namespace, pod.Name)
-		if err != nil {
-			logger.Log.Errorf("Failed to get logs, %v", err)
-			continue
-		} else {
-			logger.Log.Debugf("Successfully read log length %d for pod: %s.%s", len(logs), pod.Namespace, pod.Name)
-		}
+		for _, container := range pod.Spec.Containers {
+			logs, err := provider.GetPodLogs(ctx, pod.Namespace, pod.Name, container.Name)
+			if err != nil {
+				logger.Log.Errorf("Failed to get logs, %v", err)
+				continue
+			} else {
+				logger.Log.Debugf("Successfully read log length %d for pod: %s.%s.%s", len(logs), pod.Namespace, pod.Name, container.Name)
+			}
 
-		if err := AddStrToZip(zipWriter, logs, fmt.Sprintf("%s.%s.log", pod.Namespace, pod.Name)); err != nil {
-			logger.Log.Errorf("Failed write logs, %v", err)
-		} else {
-			logger.Log.Debugf("Successfully added log length %d from pod: %s.%s", len(logs), pod.Namespace, pod.Name)
+			if err := AddStrToZip(zipWriter, logs, fmt.Sprintf("%s.%s.%s.log", pod.Namespace, pod.Name, container.Name)); err != nil {
+				logger.Log.Errorf("Failed write logs, %v", err)
+			} else {
+				logger.Log.Debugf("Successfully added log length %d from pod: %s.%s.%s", len(logs), pod.Namespace, pod.Name, container.Name)
+			}
 		}
 	}
 

debug.Dockerfile

@@ -3,11 +3,12 @@ FROM node:14-slim AS site-build
 
 WORKDIR /app/ui-build
 
-COPY ui .
+COPY ui/package.json .
+COPY ui/package-lock.json .
 RUN npm i
+COPY ui .
 RUN npm run build
 
-
 FROM golang:1.16-alpine AS builder
 # Set necessary environment variables needed for our image.
 ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
@@ -34,32 +35,33 @@ ARG SEM_VER=0.0.0
 COPY shared ../shared
 COPY tap ../tap
 COPY agent .
+# Include gcflags for debugging
 RUN go build -gcflags="all=-N -l" -o mizuagent .
 
-# Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.19/basenine_linux_amd64 ./basenine_linux_amd64
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.19/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
-RUN shasum -a 256 -c basenine_linux_amd64.sha256
-RUN chmod +x ./basenine_linux_amd64
-
 COPY devops/build_extensions_debug.sh ..
 RUN cd .. && /bin/bash build_extensions_debug.sh
 
-
 FROM golang:1.16-alpine
 
-RUN apk add bash libpcap-dev
+# Set necessary environment variables needed for our image.
+RUN apk add bash libpcap-dev gcc g++
 
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
-COPY --from=builder ["/app/agent-build/basenine_linux_amd64", "/usr/local/bin/basenine"]
 COPY --from=builder ["/app/agent/build/extensions", "extensions"]
 COPY --from=site-build ["/app/ui-build/build", "site"]
+RUN mkdir /app/data/
 
-# install remote debugging tool
+# install delve
+ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 RUN go get github.com/go-delve/delve/cmd/dlv
 
+ENV GIN_MODE=debug
+
+# delve ports
+EXPOSE 2345 2346
+
+# this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
 ENTRYPOINT "/app/mizuagent"
-#CMD ["sh", "-c", "dlv --headless=true --listen=:2345 --log --api-version=2 --accept-multiclient exec ./mizuagent -- --api-server"]

deploy/kubernetes/helm-chart/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deploy/kubernetes/helm-chart/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: mizuhelm
+description: Mizu helm chart for Kubernetes
+type: application
+version: 0.1.1
+kubeVersion: ">= 1.16.0-0"
+appVersion: "0.21.29"

deploy/kubernetes/helm-chart/templates/PersistentVolumeClaim.yaml

@@ -0,0 +1,13 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ .Values.volumeClaim.name }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    limits:
+      storage: 700M
+    requests:
+      storage: 700M

deploy/kubernetes/helm-chart/templates/clusterRole.yaml

@@ -0,0 +1,30 @@
+{{- if .Values.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Values.rbac.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    mizu-cli-version: {{ .Chart.AppVersion }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+rules:
+  - apiGroups: [ "", "extensions", "apps" ]
+    resources: [ "endpoints", "pods", "services", "namespaces" ]
+    verbs: [ "get", "list", "watch" ]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Values.rbac.roleBindingName }}
+  labels:
+    mizu-cli-version: {{ .Chart.AppVersion }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Values.rbac.name }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.serviceAccountName }}
+    namespace: {{ .Release.Namespace }}
+  {{- end -}}

deploy/kubernetes/helm-chart/templates/configmap.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.configMap.name }}
+  namespace: {{ .Release.Namespace }}
+data:
+  mizu-config.json: >-
+    {"maxDBSizeBytes":200000000,"agentImage":"{{ .Values.container.tapper.image.repository }}:{{ .Values.container.tapper.image.tag }}","pullPolicy":"Always","logLevel":4,"tapperResources":{"CpuLimit":"750m","MemoryLimit":"1Gi","CpuRequests":"50m","MemoryRequests":"50Mi"},"mizuResourceNamespace":"{{ .Release.Namespace }}","agentDatabasePath":"/app/data/","standaloneMode":true}

deploy/kubernetes/helm-chart/templates/deployment.yaml

@@ -0,0 +1,128 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.pod.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ .Values.pod.name }}
+spec:
+  replicas: {{ .Values.deployment.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ .Values.pod.name }}
+  template:
+    metadata:
+      name: {{ .Values.pod.name }}
+      creationTimestamp: null
+      labels:
+        app: {{ .Values.pod.name }}
+    spec:
+      volumes:
+        - name: {{ .Values.configMap.name }}
+          configMap:
+            name: {{ .Values.configMap.name }}
+            defaultMode: 420
+        - name: {{ .Values.volumeClaim.name }}
+          persistentVolumeClaim:
+            claimName: {{ .Values.volumeClaim.name }}
+      containers:
+        - name: {{ .Values.pod.name }}
+          image: "{{ .Values.container.mizuAgent.image.repository }}:{{ .Values.container.mizuAgent.image.tag | default .Chart.AppVersion }}"
+          command:
+            - ./mizuagent
+            - '--api-server'
+          env:
+            - name: SYNC_ENTRIES_CONFIG
+            - name: LOG_LEVEL
+              value: INFO
+          resources:
+            limits:
+              cpu: 750m
+              memory: 1Gi
+            requests:
+              cpu: 50m
+              memory: 50Mi
+          volumeMounts:
+            - name: {{ .Values.configMap.name }}
+              mountPath: /app/config/
+            - name: {{ .Values.volumeClaim.name }}
+              mountPath: /app/data/
+          livenessProbe:
+            httpGet:
+              path: /echo
+              port: {{ .Values.pod.port }}
+              scheme: HTTP
+            initialDelaySeconds: 1
+            timeoutSeconds: 1
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          imagePullPolicy: Always
+        - name: {{ .Values.container.basenine.name }}
+          image: "{{ .Values.container.basenine.image.repository }}:{{ .Values.container.basenine.image.tag | default .Chart.AppVersion }}"
+          command:
+            - /basenine
+          args:
+            - '-addr'
+            - 0.0.0.0
+            - '-port'
+            - '9099'
+            - '-persistent'
+          workingDir: /app/data/
+          resources:
+            limits:
+              cpu: 750m
+              memory: 1Gi
+            requests:
+              cpu: 50m
+              memory: 50Mi
+          volumeMounts:
+            - name: {{ .Values.configMap.name }}
+              mountPath: /app/config/
+            - name: {{ .Values.volumeClaim.name }}
+              mountPath: /app/data/
+          readinessProbe:
+            tcpSocket:
+              port: 9099
+            timeoutSeconds: 1
+            periodSeconds: 1
+            successThreshold: 1
+            failureThreshold: 3
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          imagePullPolicy: Always
+        - name: kratos
+          image: "{{ .Values.container.kratos.image.repository }}:{{ .Values.container.kratos.image.tag | default .Chart.AppVersion }}"
+          resources:
+            limits:
+              cpu: 750m
+              memory: 1Gi
+            requests:
+              cpu: 50m
+              memory: 50Mi
+          volumeMounts:
+            - name: {{ .Values.configMap.name }}
+              mountPath: /app/config/
+            - name: {{ .Values.volumeClaim.name }}
+              mountPath: /app/data/
+          readinessProbe:
+            httpGet:
+              path: /health/ready
+              port: 4433
+              scheme: HTTP
+            timeoutSeconds: 1
+            periodSeconds: 1
+            successThreshold: 1
+            failureThreshold: 3
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          imagePullPolicy: Always
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 0
+      dnsPolicy: ClusterFirstWithHostNet
+      serviceAccountName: {{ .Values.serviceAccountName }}
+      serviceAccount: {{ .Values.serviceAccountName }}
+      securityContext: { }
+      schedulerName: default-scheduler

deploy/kubernetes/helm-chart/templates/role.yaml

@@ -0,0 +1,29 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Values.roleName }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    mizu-cli-version: {{ .Chart.AppVersion }}
+rules:
+  - apiGroups: [ "apps" ]
+    resources: [ "daemonsets" ]
+    verbs: [ "patch", "get", "list", "create", "delete" ]
+  - apiGroups: [ "events.k8s.i" ]
+    resources: [ "events" ]
+    verbs: [ "list", "watch" ]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Values.roleBindingName }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Values.roleName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.serviceAccountName }}
+    namespace: {{ .Release.Namespace }}
+---

deploy/kubernetes/helm-chart/templates/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.service.name }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - name: api
+      port: {{ .Values.service.port }}
+      targetPort: {{ .Values.pod.port }}
+      protocol: TCP
+  selector:
+    app: {{ .Values.pod.name }}

deploy/kubernetes/helm-chart/templates/serviceAccount.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.serviceAccountName }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    mizu-cli-version: {{ .Chart.AppVersion }}

deploy/kubernetes/helm-chart/values.yaml

@@ -0,0 +1,51 @@
+# Default values for mizu.
+rbac:
+  create: true
+  name: "mizu-cluster-role"
+  roleBindingName: "mizu-role-binding"
+
+serviceAccountName: "mizu-service-account"
+
+roleName: "mizu-role-daemon"
+roleBindingName: "mizu-role-binding-daemon"
+
+service:
+  name: "mizu-api-server"
+  type: ClusterIP
+  port: 80
+
+pod:
+  name: "mizu-api-server"
+  port: 8899
+
+container:
+  mizuAgent:
+    image:
+      repository: "709825985650.dkr.ecr.us-east-1.amazonaws.com/up9/mizufree"
+      tag: "0.21.29"
+  tapper:
+    image:
+      repository: "709825985650.dkr.ecr.us-east-1.amazonaws.com/up9/mizufree"
+      tag: "0.21.29"
+  basenine:
+    name: "basenine"
+    port: 9099
+    image:
+      repository: "709825985650.dkr.ecr.us-east-1.amazonaws.com/up9/basenine"
+      tag: "v0.3.0"
+  kratos:
+    name: "kratos"
+    port: 4433
+    image:
+      repository: "709825985650.dkr.ecr.us-east-1.amazonaws.com/up9/kratos"
+      tag: "0.0.0"
+
+deployment:
+  replicaCount: 1
+
+configMap:
+  name: "mizu-config"
+
+volumeClaim:
+  create: true
+  name: "mizu-volume-claim"

devops/build_extensions.sh

@@ -2,6 +2,7 @@
 
 for f in tap/extensions/*; do
     if [ -d "$f" ]; then
+        echo Building extension: $f
         extension=$(basename $f) && \
         cd tap/extensions/${extension} && \
         go build -buildmode=plugin -o ../${extension}.so .  && \

docs/CONFIGURATION.md

@@ -25,7 +25,7 @@ Please make sure to use full option name (`tap.dry-run` as opposed to `dry-run`
 
 * `dump-logs` - if set to `true`, saves log files for all Mizu components (tapper, api-server, CLI) in a zip file under `$HOME/.mizu`. Default value is `false`
 
-* `image-pull-policy` - container image pull policy for Kubernetes, default value `Always`. Other accepted values are `Never` or `IfNotExist`. Please mind the implications when changing this.
+* `image-pull-policy` - container image pull policy for Kubernetes, default value `Always`. Other accepted values are `Never` or `IfNotPresent`. Please mind the implications when changing this.
 
 * `kube-config-path` - path to alternative kubeconfig file to use for all interactions with Kubernetes cluster. By default - `$HOME/.kubeconfig`
 

docs/PERMISSIONS.md

@@ -1,16 +1,92 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg)
+
 # Kubernetes permissions for MIZU  
 
-This document describes in details all permissions required for full and correct operation of Mizu
+This document describes in details all permissions required for full and correct operation of Mizu.
+
+## Editting permissions
+
+During installation, Mizu creates a `ServiceAccount` and the roles it requires. No further action is required.
+However, if there is a need, it is possible to make changes to Mizu permissions.
+
+### Adding permissions on top of Mizu's defaults
+
+Mizu pods use the `ServiceAccount` `mizu-service-account`. Permissions can be added to Mizu by creating `ClusterRoleBindings` and `RoleBindings` that target that `ServiceAccount`.
+
+For example, in order to add a `PodSecurityPolicy` which allows Mizu to run `hostNetwork` and `privileged` pods, create the following resources:
+
+```yaml
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: my-mizu-psp
+spec:
+  hostNetwork: true
+  privileged: true
+  allowedCapabilities:
+    - "*"
+  fsGroup:
+    rule: RunAsAny
+  runAsUser:
+    rule: RunAsAny
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  volumes:
+  - "*"
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: my-mizu-clusterrole
+rules:
+- apiGroups:
+  - policy
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+  resourceNames:
+  - my-mizu-psp
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: my-mizu-clusterrolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: my-mizu-clusterrole
+subjects:
+- kind: ServiceAccount
+  name: mizu-service-account # The service account used by Mizu
+  namespace: mizu
+```
+
+With this setup, when Mizu starts and creates `mizu-service-account`, this account will be subject to `my-mizu-psp` via `my-mizu-clusterrolebinding`.
+When Mizu cleans up resources, the above resources will remain available for future executions.
+
+### Replacing Mizu's default permissions with custom permissions
+
+Mizu does not create its `ServiceAccounts`, `ClusterRoles`, `ClusterRoleBindings`, `Roles` or `RoleBindings` if resources by the same name already exist. In order to replace Mizu's defaults, simply create your resources before running Mizu.
+
+For example, creating a `ClusterRole` by the name of `mizu-cluster-role` before running Mizu will cause Mizu to use that `ClusterRole` instead of the default one created by Mizu.
+
+Notes:
+
+1. The resource names must match Mizu's default names.
+2. User-managed resources must not have the label `app.kubernetes.io/managed-by=mizu`. Remove the label or set it to another value.
+
+## List of permissions
 
 We broke down this list into few categories:
+
 - Required - what is needed for `mizu` to run properly on your k8s cluster
-- Optional - permissions needed for proper name resolving for service & pod IPs 
-   - addition required for policy validation 
- 
+- Optional - permissions needed for proper name resolving for service & pod IPs
+  - addition required for policy validation
 
-
-# Required permissions
+### Required permissions
 
 Mizu needs following permissions on your Kubernetes cluster to run properly
 
@@ -57,7 +133,7 @@ Mizu needs following permissions on your Kubernetes cluster to run properly
   - get
 ```
 
-## Permissions required running with install command or (optional) for service / pod name resolving
+#### Permissions required running with install command or (optional) for service / pod name resolving
 
 Mandatory permissions for running with install command.
 
@@ -178,7 +254,7 @@ Optional for service/pod name resolving in non install standalone
   - watch
 ```
 
-## Permissions for Policy rules validation feature (opt)
+#### Permissions for Policy rules validation feature (opt)
 
 Optionally, in order to use the policy rules validation feature, Mizu requires the following additional permissions:
 
@@ -195,7 +271,7 @@ Optionally, in order to use the policy rules validation feature, Mizu requires t
 
 - - -
 
-## Namespace-Restricted mode
+#### Namespace-Restricted mode
 
 Alternatively, in order to restrict Mizu to one namespace only (by setting `agent.namespace` in the config file), Mizu needs the following permissions in that namespace:
 
@@ -235,7 +311,7 @@ Alternatively, in order to restrict Mizu to one namespace only (by setting `agen
   - get
 ```
 
-### Name resolving in Namespace-Restricted mode (opt)
+##### Name resolving in Namespace-Restricted mode (opt)
 
 To restrict Mizu to one namespace while also resolving IPs, Mizu needs the following permissions in that namespace:
 

docs/SERVICE_MESH.md

@@ -1,37 +1,75 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg)
-# Istio mutual tls (mtls) with Mizu
+
+# Service mesh mutual tls (mtls) with Mizu
+
 This document describe how Mizu tapper handles workloads configured with mtls, making the internal traffic between services in a cluster to be encrypted.
 
-Besides Istio there are other service meshes that implement mtls. However, as of now Istio is the most used one, and this is why we are focusing on it.
+The list of service meshes supported by Mizu include:
 
-In order to create an Istio setup for development, follow those steps:
-1. Deploy a sample application to a Kubernetes cluster, the sample application needs to make internal service to service calls
-2. SSH to one of the nodes, and run `tcpdump`
-3. Make sure you see the internal service to service calls in a plain text
-4. Deploy Istio to the cluster - make sure it is attached to all pods of the sample application, and that it is configured with mtls (default)
-5. Run `tcpdump` again, make sure you don't see the internal service to service calls in a plain text 
+- Istio
+- Linkerd (beta)
 
-## The connection between Istio and Envoy
-In order to implement its service mesh capabilities, [Istio](https://istio.io) use an [Envoy](https://www.envoyproxy.io) sidecar in front of every pod in the cluster. The Envoy is responsible for the mtls communication, and that's why we are focusing on Envoy proxy.
+## Installation
+
+### Optional: Allow source IP resolving in Istio
+
+When using Istio, in order to enable Mizu to reslove source IPs to names, turn on the [use_remote_address](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for) option in Istio sidecar Envoys.
+This setting causes the Envoys to append to `X-Forwarded-For` request header. Mizu in turn uses the `X-Forwarded-For` header to determine the true source IPs.
+One way to turn on the `use_remote_address` HTTP connection manager option is by applying an `EnvoyFilter`:
+
+```yaml
+apiVersion: networking.istio.io/v1alpha3
+kind: EnvoyFilter
+metadata:
+  name: mizu-xff
+  namespace: istio-system # as defined in meshConfig resource.
+spec:
+  configPatches:
+  - applyTo: NETWORK_FILTER
+    match:
+      context: SIDECAR_OUTBOUND # will match outbound listeners in all sidecars
+    patch:
+      operation: MERGE
+      value:
+        typed_config:
+          "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager"
+          use_remote_address: true
+```
+
+Save the above text to `mizu-xff-envoyfilter.yaml` and run `kubectl apply -f mizu-xff-envoyfilter.yaml`.
+
+With Istio, mizu does not resolve source IPs for non-HTTP traffic.
+
+## Implementation
+
+### Istio support
+
+#### The connection between Istio and Envoy
+
+In order to implement its service mesh capabilities, [Istio](https://istio.io) uses an [Envoy](https://www.envoyproxy.io) sidecar in front of every pod in the cluster. The Envoy is responsible for the mtls communication, and that's why we are focusing on Envoy proxy.
 
 In the future we might see more players in that field, then we'll have to either add support for each of them or go with a unified eBPF solution.
 
-## Network namespaces
+#### Network namespaces
+
 A [linux network namespace](https://man7.org/linux/man-pages/man7/network_namespaces.7.html) is an isolation that limit the process view of the network. In the container world it used to isolate one container from another. In the Kubernetes world it used to isolate a pod from another. That means that two containers running on the same pod share the same network namespace. A container can reach a container in the same pod by accessing `localhost`.
 
 An Envoy proxy configured with mtls receives the inbound traffic directed to the pod, decrypts it and sends it via `localhost` to the target container.
 
-## Tapping mtls traffic
-In order for Mizu to be able to see the decrypted traffic it needs to listen on the same network namespace of the target pod. Multiple threads of the same process can have different network namespaces. 
+#### Tapping mtls traffic
+
+In order for Mizu to be able to see the decrypted traffic it needs to listen on the same network namespace of the target pod. Multiple threads of the same process can have different network namespaces.
 
 [gopacket](https://github.com/google/gopacket) uses [libpacp](https://github.com/the-tcpdump-group/libpcap) by default for capturing the traffic. Libpacap doesn't support network namespaces and we can't ask it to listen to traffic on a different namespace. However, we can change the network namespace of the calling thread and then start libpcap to see the traffic on a different namespace.
 
-## Finding the network namespace of a running process
+#### Finding the network namespace of a running process
+
 The network namespace of a running process can be found in `/proc/PID/ns/net` link. Once we have this link, we can ask Linux to change the network namespace of a thread to this one.
 
 This mean that Mizu needs to have access to the `/proc` (procfs) of the running node.
 
-## Finding the network namespace of a running pod
+#### Finding the network namespace of a running pod
+
 In order for Mizu to be able to listen to mtls traffic, it needs to get the PIDs of the the running pods, filter them according to the user filters and then start listen to their internal network namespace traffic.
 
 There is no official way in Kubernetes to get from pod to PID. The CRI implementation purposefully doesn't force a pod to be a processes on the host. It can be a Virtual Machine as well like [Kata containers](https://katacontainers.io)
@@ -42,5 +80,16 @@ Once Mizu detects an Envoy process, it need to check whether this specific Envoy
 
 Istio sends an `INSTANCE_IP` environment variable to every Envoy proxy process. By examining the Envoy process's environment variables we can see whether it's relevant or not. Examining a process environment variables is done by reading the `/proc/PID/envion` file.
 
-## Edge cases
+#### Edge cases
+
 The method we use to find Envoy processes and correlate them to the cluster IPs may be inaccurate in certain situations. If, for example, a user runs an Envoy process manually, and set its `INSTANCE_IP` environment variable to one of the `CLUSTER_IPS` the tapper gets, then Mizu will capture traffic for it.
+
+## Development
+
+In order to create a service mesh setup for development, follow those steps:
+
+1. Deploy a sample application to a Kubernetes cluster, the sample application needs to make internal service to service calls
+2. SSH to one of the nodes, and run `tcpdump`
+3. Make sure you see the internal service to service calls in a plain text
+4. Deploy a service mesh (Istio, Linkerd) to the cluster - make sure it is attached to all pods of the sample application, and that it is configured with mtls (default)
+5. Run `tcpdump` again, make sure you don't see the internal service to service calls in a plain text

shared/consts.go

@@ -14,6 +14,8 @@ const (
 	GoGCEnvVar                       = "GOGC"
 	DefaultApiServerPort             = 8899
 	LogLevelEnvVar                   = "LOG_LEVEL"
-	BasenineHost                     = "localhost"
+	BasenineHost                     = "127.0.0.1"
 	BaseninePort                     = "9099"
+	BasenineImageRepo                = "ghcr.io/up9inc/basenine"
+	BasenineImageTag                 = "v0.3.0"
 )

shared/kubernetes/mizuTapperSyncer.go

@@ -44,7 +44,7 @@ type TapperSyncerConfig struct {
 	IgnoredUserAgents        []string
 	MizuApiFilteringOptions  api.TrafficFilteringOptions
 	MizuServiceAccountExists bool
-	Istio                    bool
+	ServiceMesh              bool
 }
 
 func CreateAndStartMizuTapperSyncer(ctx context.Context, kubernetesProvider *Provider, config TapperSyncerConfig, startTime time.Time) (*MizuTapperSyncer, error) {
@@ -316,7 +316,7 @@ func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error {
 			tapperSyncer.config.ImagePullPolicy,
 			tapperSyncer.config.MizuApiFilteringOptions,
 			tapperSyncer.config.LogLevel,
-			tapperSyncer.config.Istio,
+			tapperSyncer.config.ServiceMesh,
 		); err != nil {
 			return err
 		}

shared/kubernetes/provider.go

@@ -278,6 +278,36 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 				},
 			},
 		},
+		{
+			Name:            "basenine",
+			Image:           fmt.Sprintf("%s:%s", shared.BasenineImageRepo, shared.BasenineImageTag),
+			ImagePullPolicy: opts.ImagePullPolicy,
+			VolumeMounts:    volumeMounts,
+			ReadinessProbe: &core.Probe{
+				FailureThreshold: 3,
+				Handler: core.Handler{
+					TCPSocket: &core.TCPSocketAction{
+						Port: intstr.Parse(shared.BaseninePort),
+					},
+				},
+				PeriodSeconds:    1,
+				SuccessThreshold: 1,
+				TimeoutSeconds:   1,
+			},
+			Resources: core.ResourceRequirements{
+				Limits: core.ResourceList{
+					"cpu":    cpuLimit,
+					"memory": memLimit,
+				},
+				Requests: core.ResourceList{
+					"cpu":    cpuRequests,
+					"memory": memRequests,
+				},
+			},
+			Command:    []string{"/basenine"},
+			Args:       []string{"-addr", "0.0.0.0", "-port", shared.BaseninePort, "-persistent"},
+			WorkingDir: shared.DataDirPath,
+		},
 	}
 
 	if createAuthContainer {
@@ -690,7 +720,7 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
 	return nil
 }
 
-func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodMap map[string][]core.Pod, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, logLevel logging.Level, istio bool) error {
+func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodMap map[string][]core.Pod, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, logLevel logging.Level, serviceMesh bool) error {
 	logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodMap), namespace, daemonSetName, podImage, tapperPodName)
 
 	if len(nodeToTappedPodMap) == 0 {
@@ -715,8 +745,8 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 		"--nodefrag",
 	}
 
-	if istio {
-		mizuCmd = append(mizuCmd, "--procfs", procfsMountPath, "--istio")
+	if serviceMesh {
+		mizuCmd = append(mizuCmd, "--procfs", procfsMountPath, "--servicemesh")
 	}
 
 	agentContainer := applyconfcore.Container()
@@ -726,7 +756,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 
 	caps := applyconfcore.Capabilities().WithDrop("ALL").WithAdd("NET_RAW").WithAdd("NET_ADMIN")
 
-	if istio {
+	if serviceMesh {
 		caps = caps.WithAdd("SYS_ADMIN")    // for reading /proc/PID/net/ns
 		caps = caps.WithAdd("SYS_PTRACE")   // for setting netns to other process
 		caps = caps.WithAdd("DAC_OVERRIDE") // for reading /proc/PID/environ
@@ -913,8 +943,8 @@ func (provider *Provider) ListAllNamespaces(ctx context.Context) ([]core.Namespa
 	return namespaces.Items, err
 }
 
-func (provider *Provider) GetPodLogs(ctx context.Context, namespace string, podName string) (string, error) {
-	podLogOpts := core.PodLogOptions{}
+func (provider *Provider) GetPodLogs(ctx context.Context, namespace string, podName string, containerName string) (string, error) {
+	podLogOpts := core.PodLogOptions{Container: containerName}
 	req := provider.clientSet.CoreV1().Pods(namespace).GetLogs(podName, &podLogOpts)
 	podLogs, err := req.Stream(ctx)
 	if err != nil {

shared/kubernetes/utils.go

@@ -73,10 +73,10 @@ func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod {
 	return missingPods
 }
 
-func GetPodInfosForPods(pods []core.Pod) []shared.PodInfo {
-	podInfos := make([]shared.PodInfo, 0)
+func GetPodInfosForPods(pods []core.Pod) []*shared.PodInfo {
+	podInfos := make([]*shared.PodInfo, 0)
 	for _, pod := range pods {
-		podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace, NodeName: pod.Spec.NodeName})
+		podInfos = append(podInfos, &shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace, NodeName: pod.Spec.NodeName})
 	}
 	return podInfos
 }

shared/models.go

@@ -41,6 +41,8 @@ type MizuAgentConfig struct {
 	MizuResourcesNamespace string        `json:"mizuResourceNamespace"`
 	AgentDatabasePath      string        `json:"agentDatabasePath"`
 	StandaloneMode         bool          `json:"standaloneMode"`
+	ServiceMap             bool          `json:"serviceMap"`
+	OAS                    bool          `json:"oas"`
 }
 
 type WebSocketMessageMetadata struct {
@@ -81,10 +83,6 @@ type TappedPodStatus struct {
 	IsTapped  bool   `json:"isTapped"`
 }
 
-type TapStatus struct {
-	Pods []PodInfo `json:"pods"`
-}
-
 type PodInfo struct {
 	Namespace string `json:"namespace"`
 	Name      string `json:"name"`
@@ -124,9 +122,9 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc
 }
 
 type HealthResponse struct {
-	TapStatus     TapStatus      `json:"tapStatus"`
-	TappersCount  int            `json:"tappersCount"`
-	TappersStatus []TapperStatus `json:"tappersStatus"`
+	TappedPods            []*PodInfo      `json:"tappedPods"`
+	ConnectedTappersCount int             `json:"connectedTappersCount"`
+	TappersStatus         []*TapperStatus `json:"tappersStatus"`
 }
 
 type VersionResponse struct {

tap/api/api.go

@@ -81,7 +81,7 @@ type OutputChannelItem struct {
 	Timestamp      int64
 	ConnectionInfo *ConnectionInfo
 	Pair           *RequestResponsePair
-	Summary        *BaseEntryDetails
+	Summary        *BaseEntry
 }
 
 type SuperTimer struct {
@@ -97,8 +97,7 @@ type Dissector interface {
 	Register(*Extension)
 	Ping()
 	Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID, counterPair *CounterPair, superTimer *SuperTimer, superIdentifier *SuperIdentifier, emitter Emitter, options *TrafficFilteringOptions) error
-	Analyze(item *OutputChannelItem, resolvedSource string, resolvedDestination string) *MizuEntry
-	Summarize(entry *MizuEntry) *BaseEntryDetails
+	Analyze(item *OutputChannelItem, resolvedSource string, resolvedDestination string) *Entry
 	Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error)
 	Macros() map[string]string
 }
@@ -117,7 +116,7 @@ func (e *Emitting) Emit(item *OutputChannelItem) {
 	e.AppStats.IncMatchedPairs()
 }
 
-type MizuEntry struct {
+type Entry struct {
 	Id                     uint                   `json:"id"`
 	Protocol               Protocol               `json:"proto"`
 	Source                 *TCP                   `json:"src"`
@@ -127,13 +126,13 @@ type MizuEntry struct {
 	StartTime              time.Time              `json:"startTime"`
 	Request                map[string]interface{} `json:"request"`
 	Response               map[string]interface{} `json:"response"`
-	Base                   *BaseEntryDetails      `json:"base"`
 	Summary                string                 `json:"summary"`
 	Method                 string                 `json:"method"`
 	Status                 int                    `json:"status"`
 	ElapsedTime            int64                  `json:"elapsedTime"`
 	Path                   string                 `json:"path"`
 	IsOutgoing             bool                   `json:"isOutgoing,omitempty"`
+	Rules                  ApplicableRules        `json:"rules,omitempty"`
 	ContractStatus         ContractStatus         `json:"contractStatus,omitempty"`
 	ContractRequestReason  string                 `json:"contractRequestReason,omitempty"`
 	ContractResponseReason string                 `json:"contractResponseReason,omitempty"`
@@ -141,22 +140,22 @@ type MizuEntry struct {
 	HTTPPair               string                 `json:"httpPair,omitempty"`
 }
 
-type MizuEntryWrapper struct {
+type EntryWrapper struct {
 	Protocol       Protocol                 `json:"protocol"`
 	Representation string                   `json:"representation"`
 	BodySize       int64                    `json:"bodySize"`
-	Data           MizuEntry                `json:"data"`
+	Data           *Entry                   `json:"data"`
 	Rules          []map[string]interface{} `json:"rulesMatched,omitempty"`
 	IsRulesEnabled bool                     `json:"isRulesEnabled"`
 }
 
-type BaseEntryDetails struct {
+type BaseEntry struct {
 	Id             uint            `json:"id"`
-	Protocol       Protocol        `json:"protocol,omitempty"`
+	Protocol       Protocol        `json:"proto,omitempty"`
 	Url            string          `json:"url,omitempty"`
 	Path           string          `json:"path,omitempty"`
 	Summary        string          `json:"summary,omitempty"`
-	StatusCode     int             `json:"statusCode"`
+	StatusCode     int             `json:"status"`
 	Method         string          `json:"method,omitempty"`
 	Timestamp      int64           `json:"timestamp,omitempty"`
 	Source         *TCP            `json:"src"`
@@ -182,11 +181,29 @@ type Contract struct {
 	Content        string         `json:"content"`
 }
 
-type DataUnmarshaler interface {
-	UnmarshalData(*MizuEntry) error
+func Summarize(entry *Entry) *BaseEntry {
+	return &BaseEntry{
+		Id:             entry.Id,
+		Protocol:       entry.Protocol,
+		Path:           entry.Path,
+		Summary:        entry.Summary,
+		StatusCode:     entry.Status,
+		Method:         entry.Method,
+		Timestamp:      entry.Timestamp,
+		Source:         entry.Source,
+		Destination:    entry.Destination,
+		IsOutgoing:     entry.IsOutgoing,
+		Latency:        entry.ElapsedTime,
+		Rules:          entry.Rules,
+		ContractStatus: entry.ContractStatus,
+	}
 }
 
-func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
+type DataUnmarshaler interface {
+	UnmarshalData(*Entry) error
+}
+
+func (bed *BaseEntry) UnmarshalData(entry *Entry) error {
 	bed.Protocol = entry.Protocol
 	bed.Id = entry.Id
 	bed.Path = entry.Path

tap/extensions/amqp/main.go

@@ -223,7 +223,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.Entry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
 
@@ -261,7 +261,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 
 	request["url"] = summary
 	reqDetails["method"] = request["method"]
-	return &api.MizuEntry{
+	return &api.Entry{
 		Protocol: protocol,
 		Source: &api.TCP{
 			Name: resolvedSource,
@@ -286,25 +286,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 
 }
 
-func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
-	return &api.BaseEntryDetails{
-		Id:          entry.Id,
-		Protocol:    protocol,
-		Summary:     entry.Summary,
-		StatusCode:  entry.Status,
-		Method:      entry.Method,
-		Timestamp:   entry.Timestamp,
-		Source:      entry.Source,
-		Destination: entry.Destination,
-		IsOutgoing:  entry.IsOutgoing,
-		Latency:     entry.ElapsedTime,
-		Rules: api.ApplicableRules{
-			Latency: 0,
-			Status:  false,
-		},
-	}
-}
-
 func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
 	bodySize = 0
 	representation := make(map[string]interface{}, 0)

tap/extensions/http/handlers.go

@@ -21,9 +21,32 @@ func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *ap
 		FilterSensitiveData(item, options)
 	}
 
+	replaceForwardedFor(item)
+
 	emitter.Emit(item)
 }
 
+func replaceForwardedFor(item *api.OutputChannelItem) {
+	if item.Protocol.Name != "http" {
+		return
+	}
+
+	request := item.Pair.Request.Payload.(api.HTTPPayload).Data.(*http.Request)
+
+	forwardedFor := request.Header.Get("X-Forwarded-For")
+	if forwardedFor == "" {
+		return
+	}
+
+	ips := strings.Split(forwardedFor, ",")
+	lastIP := strings.TrimSpace(ips[0])
+
+	item.ConnectionInfo.ClientIP = lastIP
+	// Erase the port field. Because the proxy terminates the connection from the client, the port that we see here
+	// is not the source port on the client side.
+	item.ConnectionInfo.ClientPort = ""
+}
+
 func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
 	streamID, messageHTTP1, isGrpc, err := http2Assembler.readMessage()
 	if err != nil {

tap/extensions/http/main.go

@@ -157,7 +157,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	return nil
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.Entry {
 	var host, authority, path string
 
 	request := item.Pair.Request.Payload.(map[string]interface{})
@@ -241,7 +241,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		elapsedTime = 0
 	}
 	httpPair, _ := json.Marshal(item.Pair)
-	return &api.MizuEntry{
+	return &api.Entry{
 		Protocol: item.Protocol,
 		Source: &api.TCP{
 			Name: resolvedSource,
@@ -267,26 +267,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	}
 }
 
-func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
-	return &api.BaseEntryDetails{
-		Id:          entry.Id,
-		Protocol:    entry.Protocol,
-		Path:        entry.Path,
-		Summary:     entry.Summary,
-		StatusCode:  entry.Status,
-		Method:      entry.Method,
-		Timestamp:   entry.Timestamp,
-		Source:      entry.Source,
-		Destination: entry.Destination,
-		IsOutgoing:  entry.IsOutgoing,
-		Latency:     entry.ElapsedTime,
-		Rules: api.ApplicableRules{
-			Latency: 0,
-			Status:  false,
-		},
-	}
-}
-
 func representRequest(request map[string]interface{}) (repRequest []interface{}) {
 	details, _ := json.Marshal([]api.TableData{
 		{

tap/extensions/kafka/main.go

@@ -62,7 +62,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.Entry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
 	apiKey := ApiKey(reqDetails["apiKey"].(float64))
@@ -146,7 +146,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	if elapsedTime < 0 {
 		elapsedTime = 0
 	}
-	return &api.MizuEntry{
+	return &api.Entry{
 		Protocol: _protocol,
 		Source: &api.TCP{
 			Name: resolvedSource,
@@ -171,25 +171,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	}
 }
 
-func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
-	return &api.BaseEntryDetails{
-		Id:          entry.Id,
-		Protocol:    _protocol,
-		Summary:     entry.Summary,
-		StatusCode:  entry.Status,
-		Method:      entry.Method,
-		Timestamp:   entry.Timestamp,
-		Source:      entry.Source,
-		Destination: entry.Destination,
-		IsOutgoing:  entry.IsOutgoing,
-		Latency:     entry.ElapsedTime,
-		Rules: api.ApplicableRules{
-			Latency: 0,
-			Status:  false,
-		},
-	}
-}
-
 func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
 	bodySize = 0
 	representation := make(map[string]interface{}, 0)

tap/extensions/redis/main.go

@@ -59,7 +59,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.Entry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
 	response := item.Pair.Response.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
@@ -80,7 +80,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	if elapsedTime < 0 {
 		elapsedTime = 0
 	}
-	return &api.MizuEntry{
+	return &api.Entry{
 		Protocol: protocol,
 		Source: &api.TCP{
 			Name: resolvedSource,
@@ -106,25 +106,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 
 }
 
-func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
-	return &api.BaseEntryDetails{
-		Id:          entry.Id,
-		Protocol:    protocol,
-		Summary:     entry.Summary,
-		StatusCode:  entry.Status,
-		Method:      entry.Method,
-		Timestamp:   entry.Timestamp,
-		Source:      entry.Source,
-		Destination: entry.Destination,
-		IsOutgoing:  entry.IsOutgoing,
-		Latency:     entry.ElapsedTime,
-		Rules: api.ApplicableRules{
-			Latency: 0,
-			Status:  false,
-		},
-	}
-}
-
 func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
 	bodySize = 0
 	representation := make(map[string]interface{}, 0)

tap/passive_tapper.go

@@ -52,7 +52,7 @@ var tstype = flag.String("timestamp_type", "", "Type of timestamps to use")
 var promisc = flag.Bool("promisc", true, "Set promiscuous mode")
 var staleTimeoutSeconds = flag.Int("staletimout", 120, "Max time in seconds to keep connections which don't transmit data")
 var pids = flag.String("pids", "", "A comma separated list of PIDs to capture their network namespaces")
-var istio = flag.Bool("istio", false, "Record decrypted traffic if the cluster configured with istio and mtls")
+var servicemesh = flag.Bool("servicemesh", false, "Record decrypted traffic if the cluster is configured with a service mesh and with mtls")
 
 var memprofile = flag.String("memprofile", "", "Write memory profile")
 
@@ -179,7 +179,7 @@ func initializePacketSources() error {
 	}
 
 	var err error
-	if packetSourceManager, err = source.NewPacketSourceManager(*procfs, *pids, *fname, *iface, *istio, tapTargets, behaviour); err != nil {
+	if packetSourceManager, err = source.NewPacketSourceManager(*procfs, *pids, *fname, *iface, *servicemesh, tapTargets, behaviour); err != nil {
 		return err
 	} else {
 		packetSourceManager.ReadPackets(!*nodefrag, mainPacketInputChan)

ui/package-lock.json

@@ -1161,11 +1161,51 @@
       "resolved": "https://registry.npmjs.org/@csstools/normalize.css/-/normalize.css-10.1.0.tgz",
       "integrity": "sha512-ij4wRiunFfaJxjB0BdrYHIH8FxBJpOwNPhhAcunlmPdXudL1WQV1qoP9un6JsEBAgQH+7UXyyjh0g7jTxXK6tg=="
     },
+    "@emotion/cache": {
+      "version": "11.7.1",
+      "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.7.1.tgz",
+      "integrity": "sha512-r65Zy4Iljb8oyjtLeCuBH8Qjiy107dOYC6SJq7g7GV5UCQWMObY4SJDPGFjiiVpPrOJ2hmJOoBiYTC7hwx9E2A==",
+      "requires": {
+        "@emotion/memoize": "^0.7.4",
+        "@emotion/sheet": "^1.1.0",
+        "@emotion/utils": "^1.0.0",
+        "@emotion/weak-memoize": "^0.2.5",
+        "stylis": "4.0.13"
+      }
+    },
     "@emotion/hash": {
       "version": "0.8.0",
       "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.8.0.tgz",
       "integrity": "sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow=="
     },
+    "@emotion/is-prop-valid": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.1.1.tgz",
+      "integrity": "sha512-bW1Tos67CZkOURLc0OalnfxtSXQJMrAMV0jZTVGJUPSOd4qgjF3+tTD5CwJM13PHA8cltGW1WGbbvV9NpvUZPw==",
+      "requires": {
+        "@emotion/memoize": "^0.7.4"
+      }
+    },
+    "@emotion/memoize": {
+      "version": "0.7.5",
+      "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.7.5.tgz",
+      "integrity": "sha512-igX9a37DR2ZPGYtV6suZ6whr8pTFtyHL3K/oLUotxpSVO2ASaprmAe2Dkq7tBo7CRY7MMDrAa9nuQP9/YG8FxQ=="
+    },
+    "@emotion/sheet": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.1.0.tgz",
+      "integrity": "sha512-u0AX4aSo25sMAygCuQTzS+HsImZFuS8llY8O7b9MDRzbJM0kVJlAz6KNDqcG7pOuQZJmj/8X/rAW+66kMnMW+g=="
+    },
+    "@emotion/utils": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.0.0.tgz",
+      "integrity": "sha512-mQC2b3XLDs6QCW+pDQDiyO/EdGZYOygE8s5N5rrzjSI4M3IejPE/JPndCBwRT9z982aqQNi6beWs1UeayrQxxA=="
+    },
+    "@emotion/weak-memoize": {
+      "version": "0.2.5",
+      "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.2.5.tgz",
+      "integrity": "sha512-6U71C2Wp7r5XtFtQzYrW5iKFT67OixrSxjI4MptCHzdSVlgabczzqLe0ZSgnub/5Kp4hSbpDB1tMytZY9pwxxA=="
+    },
     "@eslint/eslintrc": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.0.tgz",
@@ -1898,6 +1938,335 @@
         "react-is": "^16.8.0 || ^17.0.0"
       }
     },
+    "@mui/base": {
+      "version": "5.0.0-alpha.62",
+      "resolved": "https://registry.npmjs.org/@mui/base/-/base-5.0.0-alpha.62.tgz",
+      "integrity": "sha512-ItmdSZwHKQbLbAsS3sWguR7OHqYqh2cYWahoVmHb13Kc6bMdmVUTY4x57IlDSU712B0yuA0Q/gPTq7xADKnFow==",
+      "requires": {
+        "@babel/runtime": "^7.16.3",
+        "@emotion/is-prop-valid": "^1.1.1",
+        "@mui/utils": "^5.2.3",
+        "@popperjs/core": "^2.4.4",
+        "clsx": "^1.1.1",
+        "prop-types": "^15.7.2",
+        "react-is": "^17.0.2"
+      },
+      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.16.5",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz",
+          "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        },
+        "react-is": {
+          "version": "17.0.2",
+          "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz",
+          "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w=="
+        }
+      }
+    },
+    "@mui/icons-material": {
+      "version": "5.2.5",
+      "resolved": "https://registry.npmjs.org/@mui/icons-material/-/icons-material-5.2.5.tgz",
+      "integrity": "sha512-uQiUz+l0xy+2jExyKyU19MkMAR2F7bQFcsQ5hdqAtsB14Jw2zlmIAD55mV6f0NxKCut7Rx6cA3ZpfzlzAfoK8Q==",
+      "requires": {
+        "@babel/runtime": "^7.16.3"
+      },
+      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.16.5",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz",
+          "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        }
+      }
+    },
+    "@mui/material": {
+      "version": "5.2.6",
+      "resolved": "https://registry.npmjs.org/@mui/material/-/material-5.2.6.tgz",
+      "integrity": "sha512-yF2bRqyJMo6bYXT7TPA9IU/XLaXHi47Xvmj8duQa5ha3bCpFMXLfGoZcAUl6ZDjjGEz1nCFS+c1qx219xD/aeQ==",
+      "requires": {
+        "@babel/runtime": "^7.16.3",
+        "@mui/base": "5.0.0-alpha.62",
+        "@mui/system": "^5.2.6",
+        "@mui/types": "^7.1.0",
+        "@mui/utils": "^5.2.3",
+        "@types/react-transition-group": "^4.4.4",
+        "clsx": "^1.1.1",
+        "csstype": "^3.0.10",
+        "hoist-non-react-statics": "^3.3.2",
+        "prop-types": "^15.7.2",
+        "react-is": "^17.0.2",
+        "react-transition-group": "^4.4.2"
+      },
+      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.16.5",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz",
+          "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        },
+        "@types/react-transition-group": {
+          "version": "4.4.4",
+          "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.4.tgz",
+          "integrity": "sha512-7gAPz7anVK5xzbeQW9wFBDg7G++aPLAFY0QaSMOou9rJZpbuI58WAuJrgu+qR92l61grlnCUe7AFX8KGahAgug==",
+          "requires": {
+            "@types/react": "*"
+          }
+        },
+        "csstype": {
+          "version": "3.0.10",
+          "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.10.tgz",
+          "integrity": "sha512-2u44ZG2OcNUO9HDp/Jl8C07x6pU/eTR3ncV91SiK3dhG9TWvRVsCoJw14Ckx5DgWkzGA3waZWO3d7pgqpUI/XA=="
+        },
+        "react-is": {
+          "version": "17.0.2",
+          "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz",
+          "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w=="
+        },
+        "react-transition-group": {
+          "version": "4.4.2",
+          "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.2.tgz",
+          "integrity": "sha512-/RNYfRAMlZwDSr6z4zNKV6xu53/e2BuaBbGhbyYIXTrmgu/bGHzmqOs7mJSJBHy9Ud+ApHx3QjrkKSp1pxvlFg==",
+          "requires": {
+            "@babel/runtime": "^7.5.5",
+            "dom-helpers": "^5.0.1",
+            "loose-envify": "^1.4.0",
+            "prop-types": "^15.6.2"
+          }
+        }
+      }
+    },
+    "@mui/private-theming": {
+      "version": "5.2.3",
+      "resolved": "https://registry.npmjs.org/@mui/private-theming/-/private-theming-5.2.3.tgz",
+      "integrity": "sha512-Lc1Cmu8lSsYZiXADi9PBb17Ho82ZbseHQujUFAcp6bCJ5x/d+87JYCIpCBMagPu/isRlFCwbziuXPmz7WOzJPQ==",
+      "requires": {
+        "@babel/runtime": "^7.16.3",
+        "@mui/utils": "^5.2.3",
+        "prop-types": "^15.7.2"
+      },
+      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.16.5",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz",
+          "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        }
+      }
+    },
+    "@mui/styled-engine": {
+      "version": "5.2.6",
+      "resolved": "https://registry.npmjs.org/@mui/styled-engine/-/styled-engine-5.2.6.tgz",
+      "integrity": "sha512-bqAhli8eGS6v2qxivy2/4K0Ag8o//jsu1G2G6QcieFiT6y7oIF/nd/6Tvw6OSm3roOTifVQWNKwkt1yFWhGS+w==",
+      "requires": {
+        "@babel/runtime": "^7.16.3",
+        "@emotion/cache": "^11.7.1",
+        "prop-types": "^15.7.2"
+      },
+      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.16.5",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz",
+          "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        }
+      }
+    },
+    "@mui/styles": {
+      "version": "5.2.3",
+      "resolved": "https://registry.npmjs.org/@mui/styles/-/styles-5.2.3.tgz",
+      "integrity": "sha512-Art4qjlEI9H2h34mLL8s+CE9nWZWZbuJLbNpievaIM6DGuayz3DYkJHcH5mXJYFPhTNoe9IQYbpyKofjE0YVag==",
+      "requires": {
+        "@babel/runtime": "^7.16.3",
+        "@emotion/hash": "^0.8.0",
+        "@mui/private-theming": "^5.2.3",
+        "@mui/types": "^7.1.0",
+        "@mui/utils": "^5.2.3",
+        "clsx": "^1.1.1",
+        "csstype": "^3.0.10",
+        "hoist-non-react-statics": "^3.3.2",
+        "jss": "^10.8.2",
+        "jss-plugin-camel-case": "^10.8.2",
+        "jss-plugin-default-unit": "^10.8.2",
+        "jss-plugin-global": "^10.8.2",
+        "jss-plugin-nested": "^10.8.2",
+        "jss-plugin-props-sort": "^10.8.2",
+        "jss-plugin-rule-value-function": "^10.8.2",
+        "jss-plugin-vendor-prefixer": "^10.8.2",
+        "prop-types": "^15.7.2"
+      },
+      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.16.5",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz",
+          "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        },
+        "csstype": {
+          "version": "3.0.10",
+          "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.10.tgz",
+          "integrity": "sha512-2u44ZG2OcNUO9HDp/Jl8C07x6pU/eTR3ncV91SiK3dhG9TWvRVsCoJw14Ckx5DgWkzGA3waZWO3d7pgqpUI/XA=="
+        },
+        "jss": {
+          "version": "10.9.0",
+          "resolved": "https://registry.npmjs.org/jss/-/jss-10.9.0.tgz",
+          "integrity": "sha512-YpzpreB6kUunQBbrlArlsMpXYyndt9JATbt95tajx0t4MTJJcCJdd4hdNpHmOIDiUJrF/oX5wtVFrS3uofWfGw==",
+          "requires": {
+            "@babel/runtime": "^7.3.1",
+            "csstype": "^3.0.2",
+            "is-in-browser": "^1.1.3",
+            "tiny-warning": "^1.0.2"
+          }
+        },
+        "jss-plugin-camel-case": {
+          "version": "10.9.0",
+          "resolved": "https://registry.npmjs.org/jss-plugin-camel-case/-/jss-plugin-camel-case-10.9.0.tgz",
+          "integrity": "sha512-UH6uPpnDk413/r/2Olmw4+y54yEF2lRIV8XIZyuYpgPYTITLlPOsq6XB9qeqv+75SQSg3KLocq5jUBXW8qWWww==",
+          "requires": {
+            "@babel/runtime": "^7.3.1",
+            "hyphenate-style-name": "^1.0.3",
+            "jss": "10.9.0"
+          }
+        },
+        "jss-plugin-default-unit": {
+          "version": "10.9.0",
+          "resolved": "https://registry.npmjs.org/jss-plugin-default-unit/-/jss-plugin-default-unit-10.9.0.tgz",
+          "integrity": "sha512-7Ju4Q9wJ/MZPsxfu4T84mzdn7pLHWeqoGd/D8O3eDNNJ93Xc8PxnLmV8s8ZPNRYkLdxZqKtm1nPQ0BM4JRlq2w==",
+          "requires": {
+            "@babel/runtime": "^7.3.1",
+            "jss": "10.9.0"
+          }
+        },
+        "jss-plugin-global": {
+          "version": "10.9.0",
+          "resolved": "https://registry.npmjs.org/jss-plugin-global/-/jss-plugin-global-10.9.0.tgz",
+          "integrity": "sha512-4G8PHNJ0x6nwAFsEzcuVDiBlyMsj2y3VjmFAx/uHk/R/gzJV+yRHICjT4MKGGu1cJq2hfowFWCyrr/Gg37FbgQ==",
+          "requires": {
+            "@babel/runtime": "^7.3.1",
+            "jss": "10.9.0"
+          }
+        },
+        "jss-plugin-nested": {
+          "version": "10.9.0",
+          "resolved": "https://registry.npmjs.org/jss-plugin-nested/-/jss-plugin-nested-10.9.0.tgz",
+          "integrity": "sha512-2UJnDrfCZpMYcpPYR16oZB7VAC6b/1QLsRiAutOt7wJaaqwCBvNsosLEu/fUyKNQNGdvg2PPJFDO5AX7dwxtoA==",
+          "requires": {
+            "@babel/runtime": "^7.3.1",
+            "jss": "10.9.0",
+            "tiny-warning": "^1.0.2"
+          }
+        },
+        "jss-plugin-props-sort": {
+          "version": "10.9.0",
+          "resolved": "https://registry.npmjs.org/jss-plugin-props-sort/-/jss-plugin-props-sort-10.9.0.tgz",
+          "integrity": "sha512-7A76HI8bzwqrsMOJTWKx/uD5v+U8piLnp5bvru7g/3ZEQOu1+PjHvv7bFdNO3DwNPC9oM0a//KwIJsIcDCjDzw==",
+          "requires": {
+            "@babel/runtime": "^7.3.1",
+            "jss": "10.9.0"
+          }
+        },
+        "jss-plugin-rule-value-function": {
+          "version": "10.9.0",
+          "resolved": "https://registry.npmjs.org/jss-plugin-rule-value-function/-/jss-plugin-rule-value-function-10.9.0.tgz",
+          "integrity": "sha512-IHJv6YrEf8pRzkY207cPmdbBstBaE+z8pazhPShfz0tZSDtRdQua5jjg6NMz3IbTasVx9FdnmptxPqSWL5tyJg==",
+          "requires": {
+            "@babel/runtime": "^7.3.1",
+            "jss": "10.9.0",
+            "tiny-warning": "^1.0.2"
+          }
+        },
+        "jss-plugin-vendor-prefixer": {
+          "version": "10.9.0",
+          "resolved": "https://registry.npmjs.org/jss-plugin-vendor-prefixer/-/jss-plugin-vendor-prefixer-10.9.0.tgz",
+          "integrity": "sha512-MbvsaXP7iiVdYVSEoi+blrW+AYnTDvHTW6I6zqi7JcwXdc6I9Kbm234nEblayhF38EftoenbM+5218pidmC5gA==",
+          "requires": {
+            "@babel/runtime": "^7.3.1",
+            "css-vendor": "^2.0.8",
+            "jss": "10.9.0"
+          }
+        }
+      }
+    },
+    "@mui/system": {
+      "version": "5.2.6",
+      "resolved": "https://registry.npmjs.org/@mui/system/-/system-5.2.6.tgz",
+      "integrity": "sha512-PZ7bmpWOLikWgqn2zWv9/Xa7lxnRBOmfjoMH7c/IVYJs78W3971brXJ3xV9MEWWQcoqiYQeXzUJaNf4rFbKCBA==",
+      "requires": {
+        "@babel/runtime": "^7.16.3",
+        "@mui/private-theming": "^5.2.3",
+        "@mui/styled-engine": "^5.2.6",
+        "@mui/types": "^7.1.0",
+        "@mui/utils": "^5.2.3",
+        "clsx": "^1.1.1",
+        "csstype": "^3.0.10",
+        "prop-types": "^15.7.2"
+      },
+      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.16.5",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz",
+          "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        },
+        "csstype": {
+          "version": "3.0.10",
+          "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.10.tgz",
+          "integrity": "sha512-2u44ZG2OcNUO9HDp/Jl8C07x6pU/eTR3ncV91SiK3dhG9TWvRVsCoJw14Ckx5DgWkzGA3waZWO3d7pgqpUI/XA=="
+        }
+      }
+    },
+    "@mui/types": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/@mui/types/-/types-7.1.0.tgz",
+      "integrity": "sha512-Hh7ALdq/GjfIwLvqH3XftuY3bcKhupktTm+S6qRIDGOtPtRuq2L21VWzOK4p7kblirK0XgGVH5BLwa6u8z/6QQ=="
+    },
+    "@mui/utils": {
+      "version": "5.2.3",
+      "resolved": "https://registry.npmjs.org/@mui/utils/-/utils-5.2.3.tgz",
+      "integrity": "sha512-sQujlajIS0zQKcGIS6tZR0L1R+ib26B6UtuEn+cZqwKHsPo3feuS+SkdscYBdcCdMbrZs4gj8WIJHl2z6tbSzQ==",
+      "requires": {
+        "@babel/runtime": "^7.16.3",
+        "@types/prop-types": "^15.7.4",
+        "@types/react-is": "^16.7.1 || ^17.0.0",
+        "prop-types": "^15.7.2",
+        "react-is": "^17.0.2"
+      },
+      "dependencies": {
+        "@babel/runtime": {
+          "version": "7.16.5",
+          "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz",
+          "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==",
+          "requires": {
+            "regenerator-runtime": "^0.13.4"
+          }
+        },
+        "@types/prop-types": {
+          "version": "15.7.4",
+          "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.4.tgz",
+          "integrity": "sha512-rZ5drC/jWjrArrS8BR6SIr4cWpW09RNTYt9AMZo3Jwwif+iacXAqgVjm0B0Bv/S1jhDXKHqRVNCbACkJ89RAnQ=="
+        },
+        "react-is": {
+          "version": "17.0.2",
+          "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz",
+          "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w=="
+        }
+      }
+    },
     "@nodelib/fs.scandir": {
       "version": "2.1.4",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.4.tgz",
@@ -1957,6 +2326,11 @@
         }
       }
     },
+    "@popperjs/core": {
+      "version": "2.11.0",
+      "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.0.tgz",
+      "integrity": "sha512-zrsUxjLOKAzdewIDRWy9nsV1GQsKBCWaGwsZQlCgr6/q+vjyZhFgqedLfFBuI9anTPEUT4APq9Mu0SZBTzIcGQ=="
+    },
     "@rollup/plugin-node-resolve": {
       "version": "7.1.3",
       "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-7.1.3.tgz",
@@ -2479,6 +2853,14 @@
         "@types/react": "*"
       }
     },
+    "@types/react-is": {
+      "version": "17.0.3",
+      "resolved": "https://registry.npmjs.org/@types/react-is/-/react-is-17.0.3.tgz",
+      "integrity": "sha512-aBTIWg1emtu95bLTLx0cpkxwGW3ueZv71nE2YFBpL8k/z5czEW8yYpOo8Dp+UUAFAtKwNaOsh/ioSeQnWlZcfw==",
+      "requires": {
+        "@types/react": "*"
+      }
+    },
     "@types/react-transition-group": {
       "version": "4.4.1",
       "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.1.tgz",
@@ -4355,6 +4737,11 @@
         }
       }
     },
+    "classnames": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.1.tgz",
+      "integrity": "sha512-OlQdbZ7gLfGarSqxesMesDa5uz7KFbID8Kpq/SxIoNGDqY8lSYs0D+hhtBXhcdB3rcbXArFr7vlHheLk1voeNA=="
+    },
     "clean-css": {
       "version": "4.2.3",
       "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-4.2.3.tgz",
@@ -10689,6 +11076,19 @@
         "object-visit": "^1.0.0"
       }
     },
+    "material-ui-popup-state": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/material-ui-popup-state/-/material-ui-popup-state-2.0.0.tgz",
+      "integrity": "sha512-1sbb9xpMs7OxG0SOGfGO0ZnwiLtqZSoXda0/AqqJkpouT4e0nADXutQtDJFMa9GUMNAODVDlYnNmfqM+MhFjsg==",
+      "requires": {
+        "@babel/runtime": "^7.12.5",
+        "@mui/icons-material": "^5.0.0",
+        "@mui/material": "^5.0.0",
+        "@mui/styles": "^5.0.0",
+        "classnames": "^2.2.6",
+        "prop-types": "^15.7.2"
+      }
+    },
     "md5.js": {
       "version": "1.3.5",
       "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz",
@@ -15733,6 +16133,11 @@
         }
       }
     },
+    "stylis": {
+      "version": "4.0.13",
+      "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.0.13.tgz",
+      "integrity": "sha512-xGPXiFVl4YED9Jh7Euv2V220mriG9u4B2TA6Ybjc1catrstKD2PpIdU3U0RKpkVBC2EhmL/F0sPCr9vrFTNRag=="
+    },
     "supports-color": {
       "version": "5.5.0",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
@@ -18279,4 +18684,4 @@
       "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="
     }
   }
-}
+}

ui/package.json

@@ -18,6 +18,7 @@
     "highlight.js": "^11.3.1",
     "json-beautify": "^1.1.1",
     "jsonpath": "^1.1.1",
+    "material-ui-popup-state": "^2.0.0",
     "moment": "^2.29.1",
     "node-sass": "^5.0.0",
     "numeral": "^2.0.6",

ui/public/index.html

@@ -29,6 +29,7 @@
       try {
         // Injected from server
         window.isEnt = __IS_STANDALONE__
+        window.isOasEnabled = __IS_OAS_ENABLED__
       }
       catch (e) {
       }

ui/src/App.sass

@@ -6,32 +6,3 @@ body
 .mizuApp
   color: $font-color
   width: 100%
-
-  .header
-    height: 60px
-    display: flex
-    align-items: center
-    padding: 5px 24px
-    justify-content: space-between
-
-    .title
-      letter-spacing: 2px
-
-      img
-        height: 45px
-
-    .description
-      margin-left: 10px
-      font-size: 11px
-      font-weight: bold
-      color: $light-blue-color
-
-  .centeredForm
-    max-width: 500px
-    text-align: center
-    display: flex
-    flex-direction: column
-    margin: 0 auto
-
-    .form-input, .form-button
-      margin-top: 20px

ui/src/App.tsx

@@ -1,71 +1,16 @@
-import React, {useEffect, useState} from 'react';
+import React, {useState} from 'react';
 import './App.sass';
-import {TrafficPage} from "./components/TrafficPage";
 import {TLSWarning} from "./components/TLSWarning/TLSWarning";
 import {Header} from "./components/Header/Header";
-import { ToastContainer, toast } from 'react-toastify';
-import 'react-toastify/dist/ReactToastify.css';
-import Api from "./helpers/api";
-import LoadingOverlay from './components/LoadingOverlay';
-import LoginPage from './components/LoginPage';
-import InstallPage from './components/InstallPage';
-
-const api = Api.getInstance();
-
- // TODO: move to state management
-export enum Page {
-    Traffic,
-    Setup,
-    Login
-}
-
-// TODO: move to state management
-export interface MizuContextModel {
-    page: Page;
-    setPage: (page: Page) => void;
-}
-
-// TODO: move to state management
-export const MizuContext = React.createContext<MizuContextModel>(null);
+import {TrafficPage} from "./components/TrafficPage";
 
 const App = () => {
-    const [isLoading, setIsLoading] = useState(true);
+
     const [analyzeStatus, setAnalyzeStatus] = useState(null);
     const [showTLSWarning, setShowTLSWarning] = useState(false);
     const [userDismissedTLSWarning, setUserDismissedTLSWarning] = useState(false);
     const [addressesWithTLS, setAddressesWithTLS] = useState(new Set<string>());
 
-    const [page, setPage] = useState(Page.Traffic); // TODO: move to state management
-
-    const determinePage = async () => { // TODO: move to state management
-        if (window['isEnt'] !== true) {
-            setPage(Page.Traffic);
-            setIsLoading(false);
-            return;
-        }
-
-        try {
-            const isInstallNeeded = await api.isInstallNeeded();
-            if (isInstallNeeded) {
-                setPage(Page.Setup);
-            } else {
-                const isAuthNeeded = await api.isAuthenticationNeeded();
-                if(isAuthNeeded) {
-                    setPage(Page.Login);
-                }
-            }
-        } catch (e) {
-            toast.error("Error occured while checking Mizu API status, see console for mode details");
-            console.error(e);
-        } finally {
-            setIsLoading(false);
-        }
-    }
-
-    useEffect(() => {
-        determinePage();
-    }, []);
-
     const onTLSDetected = (destAddress: string) => {
         addressesWithTLS.add(destAddress);
         setAddressesWithTLS(new Set(addressesWithTLS));
@@ -75,49 +20,16 @@ const App = () => {
         }
     };
 
-    let pageComponent: any;
-
-    switch (page) { // TODO: move to state management / proper routing
-        case Page.Traffic:
-            pageComponent = <TrafficPage setAnalyzeStatus={setAnalyzeStatus} onTLSDetected={onTLSDetected}/>;
-            break;
-        case Page.Setup:
-            pageComponent = <InstallPage/>;
-            break;
-        case Page.Login:
-            pageComponent = <LoginPage/>;
-            break;
-        default:
-            pageComponent = <div>Unknown Error</div>;
-    }
-
-    if (isLoading) {
-        return <LoadingOverlay/>;
-    }
-
     return (
         <div className="mizuApp">
-            <MizuContext.Provider value={{page, setPage}}>
-                <Header analyzeStatus={analyzeStatus}/>
-                {pageComponent}
-                <TLSWarning showTLSWarning={showTLSWarning}
-                            setShowTLSWarning={setShowTLSWarning}
-                            addressesWithTLS={addressesWithTLS}
-                            setAddressesWithTLS={setAddressesWithTLS}
-                            userDismissedTLSWarning={userDismissedTLSWarning}
-                            setUserDismissedTLSWarning={setUserDismissedTLSWarning}/>
-            </MizuContext.Provider>
-            <ToastContainer
-                position="bottom-right"
-                autoClose={5000}
-                hideProgressBar={false}
-                newestOnTop={false}
-                closeOnClick
-                rtl={false}
-                pauseOnFocusLoss
-                draggable
-                pauseOnHover
-            />
+            <Header analyzeStatus={analyzeStatus}/>
+            <TrafficPage setAnalyzeStatus={setAnalyzeStatus} onTLSDetected={onTLSDetected}/>
+            <TLSWarning showTLSWarning={showTLSWarning}
+                        setShowTLSWarning={setShowTLSWarning}
+                        addressesWithTLS={addressesWithTLS}
+                        setAddressesWithTLS={setAddressesWithTLS}
+                        userDismissedTLSWarning={userDismissedTLSWarning}
+                        setUserDismissedTLSWarning={setUserDismissedTLSWarning}/>
         </div>
     );
 }

ui/src/EntApp.tsx

@@ -0,0 +1,108 @@
+import React, {useEffect, useState} from 'react';
+import './App.sass';
+import {TrafficPage} from "./components/TrafficPage";
+import {TLSWarning} from "./components/TLSWarning/TLSWarning";
+import {EntHeader} from "./components/Header/EntHeader";
+import Api from "./helpers/api";
+import {toast} from "react-toastify";
+import InstallPage from "./components/InstallPage";
+import LoginPage from "./components/LoginPage";
+import LoadingOverlay from "./components/LoadingOverlay";
+import AuthPageBase from './components/AuthPageBase';
+
+const api = Api.getInstance();
+
+// TODO: move to state management
+export enum Page {
+    Traffic,
+    Setup,
+    Login
+}
+
+// TODO: move to state management
+export interface MizuContextModel {
+    page: Page;
+    setPage: (page: Page) => void;
+}
+
+// TODO: move to state management
+export const MizuContext = React.createContext<MizuContextModel>(null);
+
+const EntApp = () => {
+
+    const [isLoading, setIsLoading] = useState(true);
+    const [showTLSWarning, setShowTLSWarning] = useState(false);
+    const [userDismissedTLSWarning, setUserDismissedTLSWarning] = useState(false);
+    const [addressesWithTLS, setAddressesWithTLS] = useState(new Set<string>());
+    const [page, setPage] = useState(Page.Traffic); // TODO: move to state management
+    const [isFirstLogin, setIsFirstLogin] = useState(false);
+
+    const determinePage = async () => { // TODO: move to state management
+        try {
+            const isInstallNeeded = await api.isInstallNeeded();
+            if (isInstallNeeded) {
+                setPage(Page.Setup);
+            } else {
+                const isAuthNeeded = await api.isAuthenticationNeeded();
+                if(isAuthNeeded) {
+                    setPage(Page.Login);
+                }
+            }
+        } catch (e) {
+            toast.error("Error occured while checking Mizu API status, see console for mode details");
+            console.error(e);
+        } finally {
+            setIsLoading(false);
+        }
+    }
+
+    useEffect(() => {
+        determinePage();
+    }, []);
+
+    const onTLSDetected = (destAddress: string) => {
+        addressesWithTLS.add(destAddress);
+        setAddressesWithTLS(new Set(addressesWithTLS));
+
+        if (!userDismissedTLSWarning) {
+            setShowTLSWarning(true);
+        }
+    };
+
+    let pageComponent: any;
+
+    switch (page) { // TODO: move to state management / proper routing
+        case Page.Traffic:
+            pageComponent = <TrafficPage onTLSDetected={onTLSDetected}/>;
+            break;
+        case Page.Setup:
+            pageComponent = <AuthPageBase><InstallPage onFirstLogin={() => setIsFirstLogin(true)}/></AuthPageBase>;
+            break;
+        case Page.Login:
+            pageComponent = <AuthPageBase><LoginPage/></AuthPageBase>;
+            break;
+        default:
+            pageComponent = <div>Unknown Error</div>;
+    }
+
+    if (isLoading) {
+        return <LoadingOverlay/>;
+    }
+
+    return (
+        <div className="mizuApp">
+            <MizuContext.Provider value={{page, setPage}}>
+                {page === Page.Traffic && <EntHeader isFirstLogin={isFirstLogin} setIsFirstLogin={setIsFirstLogin}/>}
+                {pageComponent}
+                {page === Page.Traffic && <TLSWarning showTLSWarning={showTLSWarning}
+                            setShowTLSWarning={setShowTLSWarning}
+                            addressesWithTLS={addressesWithTLS}
+                            setAddressesWithTLS={setAddressesWithTLS}
+                            userDismissedTLSWarning={userDismissedTLSWarning}
+                            setUserDismissedTLSWarning={setUserDismissedTLSWarning}/>}
+            </MizuContext.Provider>
+        </div>
+    );
+}
+
+export default EntApp;

ui/src/components/AuthPageBase.tsx

@@ -0,0 +1,16 @@
+import React from "react";
+import background from "./assets/authBackground.png";
+import logo from './assets/MizuEntLogoFull.svg';
+import "./style/AuthBasePage.sass";
+
+
+export const AuthPageBase: React.FC = ({children}) => {
+    return <div className="authContainer" style={{background: `url(${background})`, backgroundSize: "cover"}}>
+            <div className="authHeader">
+                <img alt="logo" src={logo}/>
+            </div>
+            {children}
+    </div>;
+};
+
+export default AuthPageBase;

ui/src/components/EntriesList.tsx

@@ -1,4 +1,4 @@
-import React, {useCallback, useEffect, useMemo, useRef, useState} from "react";
+import React, {useCallback, useEffect, useMemo, useState} from "react";
 import styles from './style/EntriesList.module.sass';
 import ScrollableFeedVirtualized from "react-scrollable-feed-virtualized";
 import Moment from 'moment';
@@ -33,14 +33,14 @@ interface EntriesListProps {
     leftOffBottom: number;
     truncatedTimestamp: number;
     setTruncatedTimestamp: any;
+    scrollableRef: any;
 }
 
 const api = Api.getInstance();
 
-export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, query, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, setQueriedCurrent, queriedTotal, setQueriedTotal, startTime, noMoreDataTop, setNoMoreDataTop, focusedEntryId, setFocusedEntryId, updateQuery, leftOffTop, setLeftOffTop, isWebSocketConnectionClosed, ws, openWebSocket, leftOffBottom, truncatedTimestamp, setTruncatedTimestamp}) => {
+export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, query, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, setQueriedCurrent, queriedTotal, setQueriedTotal, startTime, noMoreDataTop, setNoMoreDataTop, focusedEntryId, setFocusedEntryId, updateQuery, leftOffTop, setLeftOffTop, isWebSocketConnectionClosed, ws, openWebSocket, leftOffBottom, truncatedTimestamp, setTruncatedTimestamp, scrollableRef}) => {
     const [loadMoreTop, setLoadMoreTop] = useState(false);
     const [isLoadingTop, setIsLoadingTop] = useState(false);
-    const scrollableRef = useRef(null);
 
     useEffect(() => {
         const list = document.getElementById('list').firstElementChild;
@@ -92,7 +92,7 @@ export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, qu
         if (scrollTo) {
             scrollableRef.current.scrollToIndex(data.data.length - 1);
         }
-    },[setLoadMoreTop, setIsLoadingTop, entries, setEntries, query, setNoMoreDataTop, leftOffTop, setLeftOffTop, queriedCurrent, setQueriedCurrent, setQueriedTotal, setTruncatedTimestamp]);
+    },[setLoadMoreTop, setIsLoadingTop, entries, setEntries, query, setNoMoreDataTop, leftOffTop, setLeftOffTop, queriedCurrent, setQueriedCurrent, setQueriedTotal, setTruncatedTimestamp, scrollableRef]);
 
     useEffect(() => {
         if(!isWebSocketConnectionClosed || !loadMoreTop || noMoreDataTop) return;

ui/src/components/EntryDetailed.tsx

@@ -69,9 +69,7 @@ const EntryTitle: React.FC<any> = ({protocol, data, bodySize, elapsedTime, updat
     </div>;
 };
 
-const EntrySummary: React.FC<any> = ({data, updateQuery}) => {
-    const entry = data.base;
-
+const EntrySummary: React.FC<any> = ({entry, updateQuery}) => {
     return <EntryItem
         key={`entry-${entry.id}`}
         entry={entry}
@@ -92,7 +90,7 @@ export const EntryDetailed: React.FC<EntryDetailedProps> = ({entryData, updateQu
             elapsedTime={entryData.data.elapsedTime}
             updateQuery={updateQuery}
         />
-        {entryData.data && <EntrySummary data={entryData.data} updateQuery={updateQuery}/>}
+        {entryData.data && <EntrySummary entry={entryData.data} updateQuery={updateQuery}/>}
         <>
             {entryData.data && <EntryViewer
                 representation={entryData.representation}

ui/src/components/EntryDetailed/EntryViewer.module.sass

@@ -51,6 +51,7 @@
         color: $blue-gray
         border-radius: 4px
         padding: 10px
+        position: relative
         .bodyHeader
             padding: 0 1rem
         .endpointURL

ui/src/components/EntryListItem/EntryListItem.tsx

@@ -20,11 +20,11 @@ interface TCPInterface {
 }
 
 interface Entry {
-    protocol: ProtocolInterface,
+    proto: ProtocolInterface,
     method?: string,
     summary: string,
     id: number,
-    statusCode?: number;
+    status?: number;
     timestamp: Date;
     src: TCPInterface,
     dst: TCPInterface,
@@ -53,7 +53,7 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
 
     const isSelected = focusedEntryId === entry.id.toString();
 
-    const classification = getClassification(entry.statusCode)
+    const classification = getClassification(entry.status)
     const numberOfRules = entry.rules.numberOfRules
     let ingoingIcon;
     let outgoingIcon;
@@ -123,7 +123,7 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
             break;
     }
 
-    const isStatusCodeEnabled = ((entry.protocol.name === "http" && "statusCode" in entry) || entry.statusCode !== 0);
+    const isStatusCodeEnabled = ((entry.proto.name === "http" && "status" in entry) || entry.status !== 0);
     var endpointServiceContainer = "10px";
     if (!isStatusCodeEnabled) endpointServiceContainer = "20px";
 
@@ -137,7 +137,7 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
                 setFocusedEntryId(entry.id.toString());
             }}
             style={{
-                border: isSelected ? `1px ${entry.protocol.backgroundColor} solid` : "1px transparent solid",
+                border: isSelected ? `1px ${entry.proto.backgroundColor} solid` : "1px transparent solid",
                 position: !headingMode ? "absolute" : "unset",
                 top: style['top'],
                 marginTop: !headingMode ? style['marginTop'] : "10px",
@@ -145,12 +145,12 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
             }}
         >
             {!headingMode ? <Protocol
-                protocol={entry.protocol}
+                protocol={entry.proto}
                 horizontal={false}
                 updateQuery={updateQuery}
             /> : null}
             {isStatusCodeEnabled && <div>
-                <StatusCode statusCode={entry.statusCode} updateQuery={updateQuery}/>
+                <StatusCode statusCode={entry.status} updateQuery={updateQuery}/>
             </div>}
             <div className={styles.endpointServiceContainer} style={{paddingLeft: endpointServiceContainer}}>
                 <Summary method={entry.method} summary={entry.summary} updateQuery={updateQuery}/>
@@ -161,7 +161,9 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
                         displayIconOnMouseOver={true}
                         flipped={true}
                         style={{marginTop: "-4px", overflow: "visible"}}
-                        iconStyle={!headingMode ? {marginTop: "4px", left: "68px", position: "absolute"} : {marginTop: "4px", left: "calc(50vw + 41px)", position: "absolute"}}
+                        iconStyle={!headingMode ? {marginTop: "4px", left: "68px", position: "absolute"} :
+                        entry.proto.name === "http" ? {marginTop: "4px", left: "calc(50vw + 41px)", position: "absolute"} :
+                        {marginTop: "4px", left: "calc(50vw - 9px)", position: "absolute"}}
                     >
                         <span
                             title="Source Name"
@@ -169,7 +171,7 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
                             {entry.src.name ? entry.src.name : "[Unresolved]"}
                         </span>
                     </Queryable>
-                    <SwapHorizIcon style={{color: entry.protocol.backgroundColor, marginTop: "-2px"}}></SwapHorizIcon>
+                    <SwapHorizIcon style={{color: entry.proto.backgroundColor, marginTop: "-2px"}}></SwapHorizIcon>
                     <Queryable
                         query={`dst.name == "${entry.dst.name}"`}
                         updateQuery={updateQuery}
@@ -214,7 +216,7 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
                         {entry.src.ip}
                     </span>
                 </Queryable>
-                <span className={`${styles.tcpInfo}`} style={{marginTop: "18px"}}>:</span>
+				<span className={`${styles.tcpInfo}`} style={{marginTop: "18px"}}>{entry.src.port ? ":" : ""}</span>
                 <Queryable
                         query={`src.port == "${entry.src.port}"`}
                         updateQuery={updateQuery}

ui/src/components/Filters.tsx

@@ -36,7 +36,7 @@ interface QueryFormProps {
     openWebSocket: (query: string, resetEntries: boolean) => void;
 }
 
-const style = {
+export const modalStyle = {
     position: 'absolute',
     top: '10%',
     left: '50%',
@@ -45,6 +45,7 @@ const style = {
     bgcolor: 'background.paper',
     borderRadius: '5px',
     boxShadow: 24,
+    outline: "none",
     p: 4,
     color: '#000',
 };
@@ -153,11 +154,11 @@ export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, background
             style={{overflow: 'auto'}}
         >
             <Fade in={openModal}>
-                <Box sx={style}>
+                <Box sx={modalStyle}>
                     <Typography id="modal-modal-title" variant="h5" component="h2" style={{textAlign: 'center'}}>
                         Filtering Guide (Cheatsheet)
                     </Typography>
-                    <Typography id="modal-modal-description">
+                    <Typography component={'span'} id="modal-modal-description">
                         <p>Mizu has a rich filtering syntax that let's you query the results both flexibly and efficiently.</p>
                         <p>Here are some examples that you can try;</p>
                     </Typography>
@@ -264,7 +265,7 @@ export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, background
                             </Typography>
                             <br></br>
                             <Typography id="modal-modal-description">
-                                true if the given selector's value starts with the string:
+                                true if the given selector's value starts with (similarly <code style={{fontSize: "14px"}}>endsWith</code>, <code style={{fontSize: "14px"}}>contains</code>) the string:
                             </Typography>
                             <SyntaxHighlighter
                                 showLineNumbers={false}
@@ -272,19 +273,19 @@ export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, background
                                 language="python"
                             />
                             <Typography id="modal-modal-description">
-                                true if the given selector's value ends with the string:
+                                a field that contains a JSON encoded string can be filtered based a JSONPath:
                             </Typography>
                             <SyntaxHighlighter
                                 showLineNumbers={false}
-                                code={`request.path.endsWith("something")`}
+                                code={`response.content.text.json().some.path == "somevalue"`}
                                 language="python"
                             />
                             <Typography id="modal-modal-description">
-                                true if the given selector's value contains the string:
+                                fields that contain sensitive information can be redacted:
                             </Typography>
                             <SyntaxHighlighter
                                 showLineNumbers={false}
-                                code={`request.path.contains("something")`}
+                                code={`and redact("request.path", "src.name")`}
                                 language="python"
                             />
                             <Typography id="modal-modal-description">

ui/src/components/Header/EntHeader.tsx

@@ -0,0 +1,78 @@
+import React, {useContext, useEffect, useState} from "react";
+import logo from '../assets/MizuEntLogo.svg';
+import './Header.sass';
+import userImg from '../assets/user-circle.svg';
+import settingImg from '../assets/settings.svg';
+import {Menu, MenuItem} from "@material-ui/core";
+import PopupState, {bindMenu, bindTrigger} from "material-ui-popup-state";
+import logoutIcon from '../assets/logout.png';
+import {SettingsModal} from "../SettingsModal/SettingModal";
+import Api from "../../helpers/api";
+import {toast} from "react-toastify";
+import {MizuContext, Page} from "../../EntApp";
+
+const api = Api.getInstance();
+
+interface EntHeaderProps {
+    isFirstLogin: boolean;
+    setIsFirstLogin: (flag: boolean) => void
+}
+
+export const EntHeader: React.FC<EntHeaderProps> = ({isFirstLogin, setIsFirstLogin}) => {
+
+    const [isSettingsModalOpen, setIsSettingsModalOpen] = useState(false);
+
+    useEffect(() => {
+        if(isFirstLogin) {
+            setIsSettingsModalOpen(true)
+        }
+    }, [isFirstLogin])
+
+    const onSettingsModalClose = () => {
+        setIsSettingsModalOpen(false);
+        setIsFirstLogin(false);
+    }
+
+    return <div className="header">
+        <div>
+            <div className="title">
+                <img style={{height: 55}} src={logo} alt="logo"/>
+            </div>
+        </div>
+        <div style={{display: "flex", alignItems: "center"}}>
+            <img className="headerIcon" alt="settings" src={settingImg} style={{marginRight: 25}} onClick={() => setIsSettingsModalOpen(true)}/>
+            <ProfileButton/>
+        </div>
+        <SettingsModal isOpen={isSettingsModalOpen} onClose={onSettingsModalClose} isFirstLogin={isFirstLogin}/>
+    </div>;
+}
+
+const ProfileButton = () => {
+
+    const {setPage} = useContext(MizuContext);
+
+    const logout = async (popupState) => {
+        try {
+            await api.logout();
+            setPage(Page.Login);
+        } catch (e) {
+            toast.error("Something went wrong, please check the console");
+            console.error(e);
+        }
+        popupState.close();
+    }
+
+    return (<PopupState variant="popover" popupId="demo-popup-menu">
+        {(popupState) => (
+            <React.Fragment>
+                <img className="headerIcon" alt="user" src={userImg} {...bindTrigger(popupState)}/>
+                <Menu {...bindMenu(popupState)}>
+                    <MenuItem style={{fontSize: 12, fontWeight: 600}} onClick={() => logout(popupState)}>
+                        <img alt="logout" src={logoutIcon} style={{marginRight: 5, height: 16}}/>
+                        Log Out
+                    </MenuItem>
+                </Menu>
+            </React.Fragment>
+        )}
+    </PopupState>);
+}

ui/src/components/Header/Header.sass

@@ -0,0 +1,23 @@
+@import '../../variables.module'
+
+.header
+  height: 60px
+  display: flex
+  align-items: center
+  padding: 5px 24px
+  justify-content: space-between
+
+  .title
+    letter-spacing: 2px
+
+    img
+      height: 45px
+
+  .description
+    margin-left: 10px
+    font-size: 11px
+    font-weight: bold
+    color: $light-blue-color
+
+  .headerIcon
+    cursor: pointer

ui/src/components/Header/Header.tsx

@@ -2,6 +2,7 @@ import React from "react";
 import {AuthPresentation} from "../AuthPresentation/AuthPresentation";
 import {AnalyzeButton} from "../AnalyzeButton/AnalyzeButton";
 import logo from '../assets/Mizu-logo.svg';
+import './Header.sass';
 
 interface HeaderProps {
     analyzeStatus: any

ui/src/components/InstallPage.tsx

@@ -1,15 +1,21 @@
-import { Button, TextField } from "@material-ui/core";
+import { Button } from "@material-ui/core";
 import React, { useContext, useState } from "react";
-import { MizuContext, Page } from "../App";
+import { MizuContext, Page } from "../EntApp";
 import { adminUsername } from "../consts";
 import Api, { FormValidationErrorType } from "../helpers/api";
 import { toast } from 'react-toastify';
 import LoadingOverlay from "./LoadingOverlay";
+import { useCommonStyles } from "../helpers/commonStyle";
 
 const api = Api.getInstance();
 
-export const InstallPage: React.FC = () => {
+interface InstallPageProps {
+    onFirstLogin: () => void;
+}
 
+export const InstallPage: React.FC<InstallPageProps> = ({onFirstLogin}) => {
+
+    const classes = useCommonStyles();
     const [isLoading, setIsLoading] = useState(false);
     const [password, setPassword] = useState("");
     const [passwordConfirm, setPasswordConfirm] = useState("");
@@ -29,8 +35,8 @@ export const InstallPage: React.FC = () => {
             setIsLoading(true);
             await api.register(adminUsername, password);
             if (!await api.isAuthenticationNeeded()) {
-                toast.success("admin user created successfully");
                 setPage(Page.Traffic);
+                onFirstLogin();
             }
         } catch (e) {
             if (e.type === FormValidationErrorType) {
@@ -47,13 +53,29 @@ export const InstallPage: React.FC = () => {
 
     }
 
-    return <div className="centeredForm">
+    const handleFormOnKeyPress = (e: React.KeyboardEvent<HTMLInputElement>) => {
+        if (e.key === "Enter") {
+            onFormSubmit();
+        }
+    };
+
+    return <div className="centeredForm" onKeyPress={handleFormOnKeyPress}>
             {isLoading && <LoadingOverlay/>}
-            <p>Welcome to Mizu, please set up the admin user to continue</p>
-            <TextField className="form-input" variant="standard" fullWidth value={adminUsername} disabled={true}/>
-            <TextField className="form-input" label="Password" variant="standard" type="password" fullWidth value={password} onChange={e => setPassword(e.target.value)}/>
-            <TextField className="form-input" label="Confirm Password" variant="standard" type="password" fullWidth value={passwordConfirm} onChange={e => setPasswordConfirm(e.target.value)}/>
-            <Button className="form-button" variant="contained" fullWidth onClick={onFormSubmit}>Finish</Button>
+            <div className="form-title left-text">Setup</div>
+            <span className="form-subtitle">Welcome to Mizu, please set up the admin user to continue</span>
+            <div className="form-input">
+                <label htmlFor="inputUsername">Username</label>
+                <input id="inputUsername" className={classes.textField} value={adminUsername} disabled={true} />
+            </div>
+            <div className="form-input">
+                <label htmlFor="inputUsername">Password</label>
+                <input id="inputUsername" className={classes.textField} value={password} type="password" onChange={(event) => setPassword(event.target.value)}/>    
+            </div>
+            <div className="form-input">
+                <label htmlFor="inputUsername">Confirm Password</label>
+                <input id="inputUsername" className={classes.textField} value={passwordConfirm} type="password" onChange={(event) => setPasswordConfirm(event.target.value)}/>    
+            </div>
+            <Button className={classes.button + " form-button"} variant="contained" fullWidth onClick={onFormSubmit}>Finish</Button>
     </div>;
 };
 

ui/src/components/LoadingOverlay.tsx

@@ -11,11 +11,17 @@ const LoadingOverlay: React.FC<LoadingOverlayProps> = ({delay}) => {
 
     const [isVisible, setIsVisible] = useState(false);
 
+    // @ts-ignore
     useEffect(() => {
+        let isRelevant = true;
+
         setTimeout(() => {
-            setIsVisible(true);
+            if(isRelevant)
+                setIsVisible(true);
         }, delay ?? SpinnerShowDelayMs);
-    }, []);
+
+        return () => isRelevant = false;
+    }, [delay]);
 
     return <div className="loading-overlay-container" hidden={!isVisible}>
         <div className="loading-overlay-spinner"/>

ui/src/components/LoginPage.tsx

@@ -1,16 +1,17 @@
-import { Button, TextField } from "@material-ui/core";
+import { Button } from "@material-ui/core";
 import React, { useContext, useState } from "react";
 import { toast } from "react-toastify";
-import { MizuContext, Page } from "../App";
+import { MizuContext, Page } from "../EntApp";
 import Api from "../helpers/api";
+import { useCommonStyles } from "../helpers/commonStyle";
 import LoadingOverlay from "./LoadingOverlay";
 
 const api = Api.getInstance();
 
 const LoginPage: React.FC = () => {
 
+    const classes = useCommonStyles();
     const [isLoading, setIsLoading] = useState(false);
-
     const [username, setUsername] = useState("");
     const [password, setPassword] = useState("");
 
@@ -22,7 +23,6 @@ const LoginPage: React.FC = () => {
         try {
             await api.login(username, password);
             if (!await api.isAuthenticationNeeded()) {
-                toast.success("Logged in successfully");
                 setPage(Page.Traffic);
             } else {
                 toast.error("Invalid credentials");
@@ -33,16 +33,27 @@ const LoginPage: React.FC = () => {
         } finally {
             setIsLoading(false);
         }
-
     }
 
+    const handleFormOnKeyPress = (e: React.KeyboardEvent<HTMLInputElement>) => {
+        if (e.key === "Enter") {
+            onFormSubmit();
+        }
+    };
 
-    return <div className="centeredForm">
+    return <div className="centeredForm" onKeyPress={handleFormOnKeyPress}>
             {isLoading && <LoadingOverlay/>}
-            <p>Welcome to Mizu, please login to continue</p>
-            <TextField className="form-input" label="Username" variant="standard" fullWidth value={username} onChange={e => setUsername(e.target.value)} />
-            <TextField className="form-input" label="Password" variant="standard" type="password" fullWidth value={password} onChange={e => setPassword(e.target.value)} />
-            <Button className="form-button" variant="contained" fullWidth onClick={onFormSubmit}>Login</Button>
+            <div className="form-title left-text">Login</div>
+            <div className="form-input">
+                <label htmlFor="inputUsername">Username</label>
+                <input id="inputUsername" autoFocus className={classes.textField} value={username} onChange={(event) => setUsername(event.target.value)}/>    
+            </div>
+            <div className="form-input">
+                <label htmlFor="inputPassword">Password</label>
+                <input id="inputPassword" className={classes.textField} value={password} type="password" onChange={(event) => setPassword(event.target.value)}/>
+            </div>
+            <Button className={classes.button + " form-button"} variant="contained" fullWidth onClick={onFormSubmit}>Log in</Button>
+            
     </div>;
 };
 

ui/src/components/SettingsModal/SettingModal.tsx

@@ -0,0 +1,152 @@
+import React, {useEffect, useMemo, useState} from "react";
+import {Modal, Backdrop, Fade, Box, Button} from "@material-ui/core";
+import {modalStyle} from "../Filters";
+import Checkbox from "../UI/Checkbox";
+import './SettingsModal.sass';
+import Api from "../../helpers/api";
+import spinner from "../assets/spinner.svg";
+import {useCommonStyles} from "../../helpers/commonStyle";
+import {toast} from "react-toastify";
+
+interface SettingsModalProps {
+    isOpen: boolean
+    onClose: () => void
+    isFirstLogin: boolean
+}
+
+const api = Api.getInstance();
+
+export const SettingsModal: React.FC<SettingsModalProps> = ({isOpen, onClose, isFirstLogin}) => {
+
+    const classes = useCommonStyles();
+    const [namespaces, setNamespaces] = useState({});
+    const [isLoading, setIsLoading] = useState(false);
+    const [searchValue, setSearchValue] = useState("");
+
+    useEffect(() => {
+        if(!isOpen) return;
+        (async () => {
+            try {
+                setSearchValue("");
+                setIsLoading(true);
+                const tapConfig = await api.getTapConfig()
+                if(isFirstLogin) {
+                    const namespacesObj = {...tapConfig?.tappedNamespaces}
+                    Object.keys(tapConfig?.tappedNamespaces ?? {}).forEach(namespace => {
+                        namespacesObj[namespace] = true;
+                    })
+                    setNamespaces(namespacesObj);
+                } else {
+                    setNamespaces(tapConfig?.tappedNamespaces);
+                }
+            } catch (e) {
+                console.error(e);
+            } finally {
+                setIsLoading(false);
+            }
+        })()
+    }, [isFirstLogin, isOpen])
+
+    const setAllNamespacesTappedValue = (isTap: boolean) => {
+        const newNamespaces = {};
+        Object.keys(namespaces).forEach(key => {
+            newNamespaces[key] = isTap;
+        })
+        setNamespaces(newNamespaces);
+    }
+
+    const updateTappingSettings = async () => {
+        try {
+            await api.setTapConfig(namespaces);
+            onClose();
+            toast.success("Saved successfully");
+        } catch (e) {
+            console.error(e);
+            toast.error("Something went wrong, changes may not have been saved.")
+        }
+    }
+
+    const toggleTapNamespace = (namespace) => {
+        const newNamespaces = {...namespaces};
+        newNamespaces[namespace] = !namespaces[namespace]
+        setNamespaces(newNamespaces);
+    }
+
+    const toggleAll = () => {
+        const isChecked = Object.values(namespaces).every(tap => tap === true);
+        setAllNamespacesTappedValue(!isChecked);
+    }
+
+    const filteredNamespaces = useMemo(() => {
+        return Object.keys(namespaces).filter((namespace) => namespace.includes(searchValue));
+    },[namespaces, searchValue])
+
+    const buildNamespacesTable = () => {
+        return <table cellPadding={5} style={{borderCollapse: "collapse"}}>
+            <thead>
+            <tr style={{borderBottomWidth: "2px"}}>
+                <th style={{width: 50}}><Checkbox checked={Object.values(namespaces).every(tap => tap === true)}
+                                                  onToggle={toggleAll}/></th>
+                <th>Namespace</th>
+            </tr>
+            </thead>
+            <tbody>
+            {filteredNamespaces?.map(namespace => {
+                    return <tr key={namespace}>
+                        <td style={{width: 50}}>
+                            <Checkbox checked={namespaces[namespace]} onToggle={() => toggleTapNamespace(namespace)}/>
+                        </td>
+                        <td>{namespace}</td>
+                    </tr>
+                }
+            )}
+            </tbody>
+        </table>
+    }
+
+    const onModalClose = (reason) => {
+        if(reason === 'backdropClick' && isFirstLogin) return;
+        onClose();
+    }
+
+    return <Modal
+        open={isOpen}
+        onClose={(event, reason) => onModalClose(reason)}
+        closeAfterTransition
+        BackdropComponent={Backdrop}
+        BackdropProps={{
+            timeout: 500,
+        }}
+        style={{overflow: 'auto'}}
+    >
+        <Fade in={isOpen}>
+            <Box sx={modalStyle} style={{width: "40vw", maxWidth: 600, height: "70vh", padding: 0, display: "flex", justifyContent: "space-between", flexDirection: "column"}}>
+                <div style={{padding: 32, paddingBottom: 0}}>
+                    <div className="settingsTitle">Tapping Settings</div>
+                    <div className="settingsSubtitle" style={{marginTop: 20}}>
+                        Please choose from below the namespaces for tapping, traffic for namespaces selected will be displayed
+                    </div>
+                    {isLoading ? <div style={{textAlign: "center", padding: 20}}>
+                        <img alt="spinner" src={spinner} style={{height: 35}}/>
+                    </div> : <>
+                        <div className="namespacesSettingsContainer">
+                            <div style={{margin: "10px 0"}}>
+                                <input className={classes.textField + " searchNamespace"} placeholder="Search" value={searchValue}
+                                       onChange={(event) => setSearchValue(event.target.value)}/></div>
+                            <div className="namespacesTable">
+                                {buildNamespacesTable()}
+                            </div>
+                        </div>
+                    </>}
+                </div>
+                <div className="settingsActionsContainer">
+                    {!isFirstLogin &&
+                    <Button style={{width: 100}} className={classes.outlinedButton} size={"small"}
+                            onClick={onClose} variant='outlined'>Cancel</Button>}
+                    <Button style={{width: 100, marginLeft: 20}} className={classes.button} size={"small"}
+                            onClick={updateTappingSettings}>OK</Button>
+                </div>
+            </Box>
+        </Fade>
+    </Modal>
+}