Bug/UI/tra 4169 apply qury by enter (#660)

* add shortcuts listeners and config

* key added

* listen for shortcut on input

* refactoring listensing to Enter Press

* comment for support

Co-authored-by: Leon <>

.github/workflows/pr_validation.yml

@@ -44,3 +44,18 @@ jobs:
 
       - name: Build Agent
         run: make agent
+
+  build-ui:
+    name: Build UI
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Node 14
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Build UI
+        run: make ui

README.md

@@ -1,5 +1,17 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
 
+<p align="center">
+    <a href="https://github.com/up9inc/mizu/releases/latest">
+        <img alt="GitHub Latest Release" src="https://img.shields.io/github/v/release/up9inc/mizu?logo=GitHub&style=flat-square">
+    </a>
+    <a href="https://github.com/up9inc/mizu/blob/main/LICENSE">
+        <img alt="GitHub License" src="https://img.shields.io/github/license/up9inc/mizu?logo=GitHub&style=flat-square">
+    </a>
+    <a href="https://join.slack.com/t/up9/shared_invite/zt-tfjnduli-QzlR8VV4Z1w3YnPIAJfhlQ">
+      <img alt="Slack" src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social">
+    </a>
+</p>
+
 # The API Traffic Viewer for Kubernetes
 
 A simple-yet-powerful API traffic viewer for Kubernetes enabling you to view all API communication between microservices to help your debug and troubleshoot regressions.

acceptanceTests/cypress.json

@@ -4,11 +4,17 @@
   "viewportHeight": 1080,
   "video": false,
   "screenshotOnRunFailure": false,
+
   "testFiles":
   ["tests/GuiPort.js",
   "tests/MultipleNamespaces.js",
+  "tests/Redact.js",
+    "tests/NoRedact.js",
   "tests/Regex.js"],
+
   "env": {
-    "testUrl": "http://localhost:8899/"
+    "testUrl": "http://localhost:8899/",
+    "redactHeaderContent": "User-Header[REDACTED]",
+    "redactBodyContent": "{ \"User\": \"[REDACTED]\" }"
   }
 }

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -0,0 +1,9 @@
+export function isValueExistsInElement(shouldInclude, content, domPathToContainer){
+    it(`should ${shouldInclude ? '' : 'not'} include '${content}'`, function () {
+        cy.get(domPathToContainer).then(htmlText => {
+            const allTextString = htmlText.text();
+            if (allTextString.includes(content) !== shouldInclude)
+                throw new Error(`One of the containers part contains ${content}`)
+        });
+    });
+}

acceptanceTests/cypress/integration/tests/MultipleNamespaces.js

@@ -1,4 +1,4 @@
-import {findLineAndCheck, getExpectedDetailsDict} from '../page_objects/StatusBar';
+import {findLineAndCheck, getExpectedDetailsDict} from '../testHelpers/StatusBarHelper';
 
 it('opening', function () {
     cy.visit(Cypress.env('testUrl'));

acceptanceTests/cypress/integration/tests/NoRedact.js

@@ -0,0 +1,8 @@
+import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+})
+
+isValueExistsInElement(false, Cypress.env('redactHeaderContent'), '#tbody-Headers');
+isValueExistsInElement(false, Cypress.env('redactBodyContent'), '.hljs');

acceptanceTests/cypress/integration/tests/Redact.js

@@ -0,0 +1,8 @@
+import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+})
+
+isValueExistsInElement(true, Cypress.env('redactHeaderContent'), '#tbody-Headers');
+isValueExistsInElement(true, Cypress.env('redactBodyContent'), '.hljs');

acceptanceTests/cypress/integration/tests/Regex.js

@@ -1,4 +1,4 @@
-import {getExpectedDetailsDict, checkLine} from '../page_objects/StatusBar';
+import {getExpectedDetailsDict, checkLine} from '../testHelpers/StatusBarHelper';
 
 
 it('opening', function () {

acceptanceTests/tap_test.go

@@ -3,7 +3,6 @@ package acceptanceTests
 import (
 	"archive/zip"
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -378,59 +377,7 @@ func TestTapRedact(t *testing.T) {
 		}
 	}
 
-	redactCheckFunc := func() error {
-		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
-
-		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
-		if err != nil {
-			return err
-		}
-		err = checkEntriesAtLeast(entries, 1)
-		if err != nil {
-			return err
-		}
-		firstEntry := entries[0]
-
-		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr := executeHttpGetRequest(entryUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entry, err: %v", requestErr)
-		}
-
-		entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		request := entry["request"].(map[string]interface{})
-
-		headers := request["_headers"].([]interface{})
-		for _, headerInterface := range headers {
-			header := headerInterface.(map[string]interface{})
-			if header["name"].(string) != "User-Header" {
-				continue
-			}
-
-			userHeader := header["value"].(string)
-			if userHeader != "[REDACTED]" {
-				return fmt.Errorf("unexpected result - user agent is not redacted")
-			}
-		}
-
-		postData := request["postData"].(map[string]interface{})
-		textDataStr := postData["text"].(string)
-
-		var textData map[string]string
-		if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
-			return fmt.Errorf("failed to parse text data, err: %v", parseErr)
-		}
-
-		if textData["User"] != "[REDACTED]" {
-			return fmt.Errorf("unexpected result - user in body is not redacted")
-		}
-
-		return nil
-	}
-	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Redact.js\""))
 }
 
 func TestTapNoRedact(t *testing.T) {
@@ -482,59 +429,7 @@ func TestTapNoRedact(t *testing.T) {
 		}
 	}
 
-	redactCheckFunc := func() error {
-		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
-
-		entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
-		if err != nil {
-			return err
-		}
-		err = checkEntriesAtLeast(entries, 1)
-		if err != nil {
-			return err
-		}
-		firstEntry := entries[0]
-
-		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
-		requestResult, requestErr := executeHttpGetRequest(entryUrl)
-		if requestErr != nil {
-			return fmt.Errorf("failed to get entry, err: %v", requestErr)
-		}
-
-		entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
-		request := entry["request"].(map[string]interface{})
-
-		headers := request["_headers"].([]interface{})
-		for _, headerInterface := range headers {
-			header := headerInterface.(map[string]interface{})
-			if header["name"].(string) != "User-Header" {
-				continue
-			}
-
-			userHeader := header["value"].(string)
-			if userHeader == "[REDACTED]" {
-				return fmt.Errorf("unexpected result - user agent is redacted")
-			}
-		}
-
-		postData := request["postData"].(map[string]interface{})
-		textDataStr := postData["text"].(string)
-
-		var textData map[string]string
-		if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
-			return fmt.Errorf("failed to parse text data, err: %v", parseErr)
-		}
-
-		if textData["User"] == "[REDACTED]" {
-			return fmt.Errorf("unexpected result - user in body is redacted")
-		}
-
-		return nil
-	}
-	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
-		t.Errorf("%v", err)
-		return
-	}
+	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/NoRedact.js\"")
 }
 
 func TestTapRegexMasking(t *testing.T) {

acceptanceTests/testsUtils.go

@@ -183,16 +183,16 @@ func tryExecuteFunc(executeFunc func() error) (err interface{}) {
 }
 
 func waitTapPodsReady(apiServerUrl string) error {
-	resolvingUrl := fmt.Sprintf("%v/status/tappersCount", apiServerUrl)
+	resolvingUrl := fmt.Sprintf("%v/status/connectedTappersCount", apiServerUrl)
 	tapPodsReadyFunc := func() error {
 		requestResult, requestErr := executeHttpGetRequest(resolvingUrl)
 		if requestErr != nil {
 			return requestErr
 		}
 
-		tappersCount := requestResult.(float64)
-		if tappersCount == 0 {
-			return fmt.Errorf("no tappers running")
+		connectedTappersCount := requestResult.(float64)
+		if connectedTappersCount == 0 {
+			return fmt.Errorf("no connected tappers running")
 		}
 		time.Sleep(waitAfterTapPodsReady)
 		return nil

agent/go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
+	github.com/chanced/openapi v0.0.6
 	github.com/djherbis/atime v1.0.0
 	github.com/getkin/kin-openapi v0.76.0
 	github.com/gin-contrib/static v0.0.1
@@ -12,6 +13,7 @@ require (
 	github.com/go-playground/universal-translator v0.17.0
 	github.com/go-playground/validator/v10 v10.5.0
 	github.com/google/martian v2.1.0+incompatible
+	github.com/google/uuid v1.1.2
 	github.com/gorilla/websocket v1.4.2
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231

agent/go.sum

@@ -82,6 +82,12 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
+github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a h1:zG6t+4krPXcCKtLbjFvAh+fKN1d0qfD+RaCj+680OU8=
+github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a/go.mod h1:yhcmlFk1hxuZ+5XZbupzT/cEm/eE4ZvWbmsW1+Q/aZE=
+github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44 h1:4NOJMtvZaOA6cI2gkIuXk/2b5KTOvm/R4zyPy/yLCM4=
+github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44/go.mod h1:XVNfXN5kgZST4PQ0W/oBAHJku2OteCeHxjAbvfd0ARM=
+github.com/chanced/openapi v0.0.6 h1:2giGS47+T8/7MN2hfRHSIUPx86Qksk+J/ciIWcAM7hY=
+github.com/chanced/openapi v0.0.6/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -119,6 +125,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
+github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -253,8 +261,10 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -316,6 +326,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -432,6 +443,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE=
+github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -465,7 +478,16 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.12.0 h1:61wEp/qfvFnqKH/WCI3M8HuRut+mHT6Mr82QrFmM2SY=
+github.com/tidwall/gjson v1.12.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.3 h1:5+deguEhHSEjmuICXZ21uSSsXotWMA0orU783+Z7Cp8=
+github.com/tidwall/sjson v1.2.3/go.mod h1:5WdjKx3AQMvCJ4RG6/2UYT7dLrGvJUV1x4jdTAyGvZs=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
@@ -477,6 +499,8 @@ github.com/up9inc/basenine/client/go v0.0.0-20220110083745-04fbc6c2068d/go.mod h
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=
+github.com/wI2L/jsondiff v0.1.1/go.mod h1:bAbJSAJXZtfOCZ5y3v7Mfb6UQa3DGdGFjQj1cNv8EcM=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
@@ -863,5 +887,6 @@ sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

agent/main.go

@@ -11,6 +11,7 @@ import (
 	"mizuserver/pkg/controllers"
 	"mizuserver/pkg/middlewares"
 	"mizuserver/pkg/models"
+	"mizuserver/pkg/oas"
 	"mizuserver/pkg/routes"
 	"mizuserver/pkg/up9"
 	"mizuserver/pkg/utils"
@@ -120,14 +121,14 @@ func main() {
 
 		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-
+		enableExpFeatureIfNeeded()
 		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
 		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
 
 		syncEntriesConfig := getSyncEntriesConfig()
 		if syncEntriesConfig != nil {
 			if err := up9.SyncEntries(syncEntriesConfig); err != nil {
-				panic(fmt.Sprintf("Error syncing entries, err: %v", err))
+				logger.Log.Error("Error syncing entries, err: %v", err)
 			}
 		}
 
@@ -148,6 +149,12 @@ func main() {
 	logger.Log.Info("Exiting")
 }
 
+func enableExpFeatureIfNeeded() {
+	if config.Config.OAS {
+		oas.GetOasGeneratorInstance().Start()
+	}
+}
+
 func configureBasenineServer(host string, port string) {
 	if !wait.New(
 		wait.WithProto("tcp"),
@@ -233,7 +240,7 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 
 	app.Use(DisableRootStaticCache())
 
-	if err := setUIMode(); err != nil {
+	if err := setUIFlags(); err != nil {
 		logger.Log.Errorf("Error setting ui mode, err: %v", err)
 	}
 	app.Use(static.ServeRoot("/", "./site"))
@@ -248,12 +255,15 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 		routes.InstallRoutes(app)
 	}
 
+	if config.Config.OAS {
+		routes.OASRoutes(app)
+	}
+
 	routes.QueryRoutes(app)
 	routes.EntriesRoutes(app)
 	routes.MetadataRoutes(app)
 	routes.StatusRoutes(app)
 	routes.NotFoundRoute(app)
-
 	utils.StartServer(app)
 }
 
@@ -268,13 +278,14 @@ func DisableRootStaticCache() gin.HandlerFunc {
 	}
 }
 
-func setUIMode() error {
+func setUIFlags() error {
 	read, err := ioutil.ReadFile(uiIndexPath)
 	if err != nil {
 		return err
 	}
 
 	replacedContent := strings.Replace(string(read), "__IS_STANDALONE__", strconv.FormatBool(config.Config.StandaloneMode), 1)
+	replacedContent = strings.Replace(replacedContent, "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS), 1)
 
 	err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
 	if err != nil {

agent/pkg/api/main.go

@@ -19,6 +19,7 @@ import (
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	"mizuserver/pkg/models"
+	"mizuserver/pkg/oas"
 	"mizuserver/pkg/resolver"
 	"mizuserver/pkg/utils"
 
@@ -136,6 +137,8 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
 				mizuEntry.Rules = rules
 			}
+
+			oas.GetOasGeneratorInstance().PushEntry(harEntry)
 		}
 
 		data, err := json.Marshal(mizuEntry)

agent/pkg/api/socket_server_handlers.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/providers"
-	"mizuserver/pkg/providers/tappersCount"
+	"mizuserver/pkg/providers/tappers"
 	"mizuserver/pkg/up9"
 	"sync"
 
@@ -30,7 +30,7 @@ func init() {
 func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
-		tappersCount.Add()
+		tappers.Connected()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
 		socketListLock.Lock()
@@ -42,7 +42,7 @@ func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
-		tappersCount.Remove()
+		tappers.Disconnected()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
 		socketListLock.Lock()

agent/pkg/controllers/config_controller.go

@@ -13,7 +13,7 @@ import (
 	"mizuserver/pkg/providers"
 	"mizuserver/pkg/providers/tapConfig"
 	"mizuserver/pkg/providers/tappedPods"
-	"mizuserver/pkg/providers/tappersStatus"
+	"mizuserver/pkg/providers/tappers"
 	"net/http"
 	"regexp"
 	"time"
@@ -33,7 +33,7 @@ func PostTapConfig(c *gin.Context) {
 		cancelTapperSyncer()
 
 		tappedPods.Set([]*shared.PodInfo{})
-		tappersStatus.Reset()
+		tappers.ResetStatus()
 
 		broadcastTappedPodsStatus()
 	}
@@ -141,7 +141,7 @@ func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, t
 					return
 				}
 
-				tappersStatus.Set(&tapperStatus)
+				tappers.SetStatus(&tapperStatus)
 				broadcastTappedPodsStatus()
 			case <-ctx.Done():
 				logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")

agent/pkg/controllers/oas_controller.go

@@ -0,0 +1,62 @@
+package controllers
+
+import (
+	"github.com/chanced/openapi"
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/oas"
+	"net/http"
+)
+
+func GetOASServers(c *gin.Context) {
+	m := make([]string, 0)
+	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
+		m = append(m, key.(string))
+		return true
+	})
+
+	c.JSON(http.StatusOK, m)
+}
+
+func GetOASSpec(c *gin.Context) {
+	res, ok := oas.GetOasGeneratorInstance().ServiceSpecs.Load(c.Param("id"))
+	if !ok {
+		c.JSON(http.StatusNotFound, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       "Service not found among specs",
+		})
+		return // exit
+	}
+
+	gen := res.(*oas.SpecGen)
+	spec, err := gen.GetSpec()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       err,
+		})
+		return // exit
+	}
+
+	c.JSON(http.StatusOK, spec)
+}
+
+func GetOASAllSpecs(c *gin.Context) {
+	res := map[string]*openapi.OpenAPI{}
+	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
+		svc := key.(string)
+		gen := value.(*oas.SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			logger.Log.Warningf("Failed to obtain spec for service %s: %s", svc, err)
+			return true
+		}
+		res[svc] = spec
+		return true
+	})
+	c.JSON(http.StatusOK, res)
+}

agent/pkg/controllers/oas_controller_test.go

@@ -0,0 +1,43 @@
+package controllers
+
+import (
+	"github.com/gin-gonic/gin"
+	"mizuserver/pkg/oas"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestGetOASServers(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+
+	GetOASServers(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+	return
+}
+
+func TestGetOASAllSpecs(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+
+	GetOASAllSpecs(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+	return
+}
+
+func TestGetOASSpec(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetOasGeneratorInstance().Start()
+	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+
+	c.Params = []gin.Param{{Key: "id", Value: "some"}}
+
+	GetOASSpec(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+	return
+}

agent/pkg/controllers/status_controller.go

@@ -9,23 +9,22 @@ import (
 	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
 	"mizuserver/pkg/providers/tappedPods"
-	"mizuserver/pkg/providers/tappersCount"
-	"mizuserver/pkg/providers/tappersStatus"
+	"mizuserver/pkg/providers/tappers"
 	"mizuserver/pkg/up9"
 	"mizuserver/pkg/validation"
 	"net/http"
 )
 
 func HealthCheck(c *gin.Context) {
-	tappers := make([]*shared.TapperStatus, 0)
-	for _, value := range tappersStatus.Get() {
-		tappers = append(tappers, value)
+	tappersStatus := make([]*shared.TapperStatus, 0)
+	for _, value := range tappers.GetStatus() {
+		tappersStatus = append(tappersStatus, value)
 	}
 
 	response := shared.HealthResponse{
-		TappedPods:    tappedPods.Get(),
-		TappersCount:  tappersCount.Get(),
-		TappersStatus: tappers,
+		TappedPods:            tappedPods.Get(),
+		ConnectedTappersCount: tappers.GetConnectedCount(),
+		TappersStatus:         tappersStatus,
 	}
 	c.JSON(http.StatusOK, response)
 }
@@ -66,12 +65,12 @@ func PostTapperStatus(c *gin.Context) {
 	}
 
 	logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus)
-	tappersStatus.Set(tapperStatus)
+	tappers.SetStatus(tapperStatus)
 	broadcastTappedPodsStatus()
 }
 
-func GetTappersCount(c *gin.Context) {
-	c.JSON(http.StatusOK, tappersCount.Get())
+func GetConnectedTappersCount(c *gin.Context) {
+	c.JSON(http.StatusOK, tappers.GetConnectedCount())
 }
 
 func GetAuthStatus(c *gin.Context) {

agent/pkg/oas/feeder_test.go

@@ -0,0 +1,158 @@
+package oas
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func getFiles(baseDir string) (result []string, err error) {
+	result = make([]string, 0, 0)
+	logger.Log.Infof("Reading files from tree: %s", baseDir)
+
+	// https://yourbasic.org/golang/list-files-in-directory/
+	err = filepath.Walk(baseDir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			ext := strings.ToLower(filepath.Ext(path))
+			if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
+				result = append(result, path)
+			}
+
+			return nil
+		})
+
+	sort.SliceStable(result, func(i, j int) bool {
+		return fileSize(result[i]) < fileSize(result[j])
+	})
+
+	logger.Log.Infof("Got files: %d", len(result))
+	return result, err
+}
+
+func fileSize(fname string) int64 {
+	fi, err := os.Stat(fname)
+	if err != nil {
+		panic(err)
+	}
+
+	return fi.Size()
+}
+
+func feedEntries(fromFiles []string) (err error) {
+	for _, file := range fromFiles {
+		logger.Log.Info("Processing file: " + file)
+		ext := strings.ToLower(filepath.Ext(file))
+		switch ext {
+		case ".har":
+			err = feedFromHAR(file)
+			if err != nil {
+				logger.Log.Warning("Failed processing file: " + err.Error())
+				continue
+			}
+		case ".ldjson":
+			err = feedFromLDJSON(file)
+			if err != nil {
+				logger.Log.Warning("Failed processing file: " + err.Error())
+				continue
+			}
+		default:
+			return errors.New("Unsupported file extension: " + ext)
+		}
+	}
+
+	return nil
+}
+
+func feedFromHAR(file string) error {
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		return err
+	}
+
+	var harDoc har.HAR
+	err = json.Unmarshal(data, &harDoc)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range harDoc.Log.Entries {
+		GetOasGeneratorInstance().PushEntry(entry)
+	}
+
+	return nil
+}
+
+func feedFromLDJSON(file string) error {
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	reader := bufio.NewReader(fd)
+
+	var meta map[string]interface{}
+
+	buf := strings.Builder{}
+	for {
+		substr, isPrefix, err := reader.ReadLine()
+		if err == io.EOF {
+			break
+		}
+
+		buf.WriteString(string(substr))
+		if isPrefix {
+			continue
+		}
+
+		line := buf.String()
+		buf.Reset()
+
+		if meta == nil {
+			err := json.Unmarshal([]byte(line), &meta)
+			if err != nil {
+				return err
+			}
+		} else {
+			var entry har.Entry
+			err := json.Unmarshal([]byte(line), &entry)
+			if err != nil {
+				logger.Log.Warningf("Failed decoding entry: %s", line)
+			}
+			GetOasGeneratorInstance().PushEntry(&entry)
+		}
+	}
+
+	return nil
+}
+
+func TestFilesList(t *testing.T) {
+	res, err := getFiles("./test_artifacts/")
+	t.Log(len(res))
+	t.Log(res)
+	if err != nil || len(res) != 2 {
+		t.Logf("Should return 2 files but returned %d", len(res))
+		t.FailNow()
+	}
+}

agent/pkg/oas/gibberish.go

@@ -0,0 +1,108 @@
+package oas
+
+import (
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+var (
+	patBase64   = regexp.MustCompile(`^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$`)
+	patUuid4    = regexp.MustCompile(`(?i)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
+	patEmail    = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
+	patHexLower = regexp.MustCompile(`(0x)?[0-9a-f]{6,}`)
+	patHexUpper = regexp.MustCompile(`(0x)?[0-9A-F]{6,}`)
+)
+
+func IsGibberish(str string) bool {
+	if patBase64.MatchString(str) && len(str) > 32 {
+		return true
+	}
+
+	if patUuid4.MatchString(str) {
+		return true
+	}
+
+	if patEmail.MatchString(str) {
+		return true
+	}
+
+	if patHexLower.MatchString(str) || patHexUpper.MatchString(str) {
+		return true
+	}
+
+	noise := noiseLevel(str)
+	if noise >= 0.2 {
+		return true
+	}
+
+	return false
+}
+
+func noiseLevel(str string) (score float64) {
+	// opinionated algo of certain char pairs marking the non-human strings
+	prev := *new(rune)
+	cnt := 0.0
+	for _, char := range str {
+		cnt += 1
+		if prev > 0 {
+			switch {
+			// continued class of upper/lower/digit adds no noise
+			case unicode.IsUpper(prev) && unicode.IsUpper(char):
+			case unicode.IsLower(prev) && unicode.IsLower(char):
+			case unicode.IsDigit(prev) && unicode.IsDigit(char):
+
+			// upper =>
+			case unicode.IsUpper(prev) && unicode.IsLower(char):
+				score += 0.25
+			case unicode.IsUpper(prev) && unicode.IsDigit(char):
+				score += 0.25
+
+			// lower =>
+			case unicode.IsLower(prev) && unicode.IsUpper(char):
+				score += 0.75
+			case unicode.IsLower(prev) && unicode.IsDigit(char):
+				score += 0.25
+
+			// digit =>
+			case unicode.IsDigit(prev) && unicode.IsUpper(char):
+				score += 0.75
+			case unicode.IsDigit(prev) && unicode.IsLower(char):
+				score += 0.75
+
+			// the rest is 100% noise
+			default:
+				score += 1
+			}
+		}
+		prev = char
+	}
+
+	score /= cnt // weigh it
+
+	return score
+}
+
+func IsVersionString(component string) bool {
+	if component == "" {
+		return false
+	}
+
+	hasV := false
+	if strings.HasPrefix(component, "v") {
+		component = component[1:]
+		hasV = true
+	}
+
+	for _, c := range component {
+		if string(c) != "." && !unicode.IsDigit(c) {
+			return false
+		}
+	}
+
+	if !hasV && strings.Contains(component, ".") {
+		return false
+	}
+
+	return true
+}

agent/pkg/oas/gibberish_test.go

@@ -0,0 +1,71 @@
+package oas
+
+import "testing"
+
+func TestNegative(t *testing.T) {
+	cases := []string{
+		"",
+		"b", // can be valid hexadecimal
+		"GetUniversalVariableUser",
+		"callback",
+		"runs",
+		"tcfv2",
+		"StartUpCheckout",
+		"GetCart",
+	}
+
+	for _, str := range cases {
+		if IsGibberish(str) {
+			t.Errorf("Mistakenly true: %s", str)
+		}
+	}
+}
+
+func TestPositive(t *testing.T) {
+	cases := []string{
+		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
+		"952bea17-3776-11ea-9341-42010a84012a",
+		"456795af-b48f-4a8d-9b37-3e932622c2f0",
+		"0a0d0174-b338-4520-a1c3-24f7e3d5ec50.html",
+		"6120c057c7a97b03f6986f1b",
+		"610bc3fd5a77a7fa25033fb0",
+		"610bd0315a77a7fa25034368",
+		"610bd0315a77a7fa25034368zh",
+		"710a462e",
+		"1554507871",
+		"qwerqwerasdfqwer@protonmai.com",
+		"john.dow.1981@protonmail.com",
+		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
+		"11ca096cbc224a67360493d44a9903",
+		"c738338322370b47a79251f7510dd", // prefixed hex
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
+		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
+		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
+		"730970532670-compute@developer.gserviceaccount.com",
+		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
+		"AAAA028295945",
+		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
+		"n63nd45qsj",
+		"n9z9QGNiz",
+		"proxy.3d2100fd7107262ecb55ce6847f01fa5.html",
+		"r-ext-5579e00a95c90",
+		"r-ext-5579e8b12f11e",
+		"r-v4-5c92513c9e04a",
+		"r-v4-5c92513c9e04a-0-c.monitoring.json",
+		"segments-1563566437171.639994",
+		"t_52d94268-8810-4a7e-ba87-ffd657a6752f",
+		"timeouts-1563566437171.639994",
+
+		// TODO
+		// "fb6cjraf9cejut2a",
+		// "Fxvd1timk", // questionable
+		// "JEHJW4BKVFDRTMTUQLHKK5WVAU",
+	}
+
+	for _, str := range cases {
+		if !IsGibberish(str) {
+			t.Errorf("Mistakenly false: %s", str)
+		}
+	}
+}

agent/pkg/oas/ignores.go

@@ -0,0 +1,75 @@
+package oas
+
+import "strings"
+
+var ignoredExtensions = []string{"gif", "svg", "css", "png", "ico", "js", "woff2", "woff", "jpg", "jpeg", "swf", "ttf", "map", "webp", "otf", "mp3"}
+
+var ignoredCtypePrefixes = []string{"image/", "font/", "video/", "audio/", "text/javascript"}
+var ignoredCtypes = []string{"application/javascript", "application/x-javascript", "text/css", "application/font-woff2", "application/font-woff", "application/x-font-woff"}
+
+var ignoredHeaders = []string{
+	"a-im", "accept",
+	"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-type", "cookie",
+	"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
+	"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
+	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests",
+	"server", "user-agent", "via", "warning", "strict-transport-security",
+	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data",
+	"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
+	"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
+	"x-wap-profile", "x-scheme",
+	"newrelic", "x-cloud-trace-context", "sentry-trace",
+	"expires", "set-cookie", "p3p", "location", "content-security-policy", "content-security-policy-report-only",
+	"last-modified", "content-language",
+	"keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by",
+	"age", "allow", "www-authenticate",
+}
+
+var ignoredHeaderPrefixes = []string{
+	":", "accept-", "access-control-", "if-", "sec-", "grpc-",
+	"x-forwarded-", "x-original-",
+	"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
+	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-",
+}
+
+func isCtypeIgnored(ctype string) bool {
+	for _, prefix := range ignoredCtypePrefixes {
+		if strings.HasPrefix(ctype, prefix) {
+			return true
+		}
+	}
+
+	for _, toIgnore := range ignoredCtypes {
+		if ctype == toIgnore {
+			return true
+		}
+	}
+	return false
+}
+
+func isExtIgnored(path string) bool {
+	for _, extIgn := range ignoredExtensions {
+		if strings.HasSuffix(path, "."+extIgn) {
+			return true
+		}
+	}
+	return false
+}
+
+func isHeaderIgnored(name string) bool {
+	name = strings.ToLower(name)
+
+	for _, ignore := range ignoredHeaders {
+		if name == ignore {
+			return true
+		}
+	}
+
+	for _, prefix := range ignoredHeaderPrefixes {
+		if strings.HasPrefix(name, prefix) {
+			return true
+		}
+	}
+
+	return false
+}

agent/pkg/oas/oas_generator.go

@@ -0,0 +1,107 @@
+package oas
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"net/url"
+	"sync"
+)
+
+var (
+	syncOnce sync.Once
+	instance *oasGenerator
+)
+
+func GetOasGeneratorInstance() *oasGenerator {
+	syncOnce.Do(func() {
+		instance = newOasGenerator()
+		logger.Log.Debug("Oas Generator Initialized")
+	})
+	return instance
+}
+
+func (g *oasGenerator) Start() {
+	if g.started {
+		return
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	g.cancel = cancel
+	g.ctx = ctx
+	g.entriesChan = make(chan har.Entry, 100) // buffer up to 100 entries for OAS processing
+	g.ServiceSpecs = &sync.Map{}
+	g.started = true
+	go instance.runGeneretor()
+}
+
+func (g *oasGenerator) runGeneretor() {
+	for {
+		select {
+		case <-g.ctx.Done():
+			logger.Log.Infof("OAS Generator was canceled")
+			return
+
+		case entry, ok := <-g.entriesChan:
+			if !ok {
+				logger.Log.Infof("OAS Generator - entries channel closed")
+				break
+			}
+			u, err := url.Parse(entry.Request.URL)
+			if err != nil {
+				logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
+			}
+
+			val, found := g.ServiceSpecs.Load(u.Host)
+			var gen *SpecGen
+			if !found {
+				gen = NewGen(u.Scheme + "://" + u.Host)
+				g.ServiceSpecs.Store(u.Host, gen)
+			} else {
+				gen = val.(*SpecGen)
+			}
+
+			opId, err := gen.feedEntry(entry)
+			if err != nil {
+				txt, suberr := json.Marshal(entry)
+				if suberr == nil {
+					logger.Log.Debugf("Problematic entry: %s", txt)
+				}
+
+				logger.Log.Warningf("Failed processing entry: %s", err)
+				continue
+			}
+
+			logger.Log.Debugf("Handled entry %s as opId: %s", entry.Request.URL, opId) // TODO: set opId back to entry?
+		}
+	}
+}
+
+func (g *oasGenerator) PushEntry(entry *har.Entry) {
+	if !g.started {
+		return
+	}
+	select {
+	case g.entriesChan <- *entry:
+	default:
+		logger.Log.Warningf("OAS Generator - entry wasn't sent to channel because the channel has no buffer or there is no receiver")
+	}
+}
+
+func newOasGenerator() *oasGenerator {
+	return &oasGenerator{
+		started:      false,
+		ctx:          nil,
+		cancel:       nil,
+		ServiceSpecs: nil,
+		entriesChan:  nil,
+	}
+}
+
+type oasGenerator struct {
+	started      bool
+	ctx          context.Context
+	cancel       context.CancelFunc
+	ServiceSpecs *sync.Map
+	entriesChan  chan har.Entry
+}

agent/pkg/oas/specgen.go

@@ -0,0 +1,497 @@
+package oas
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/google/uuid"
+	"github.com/up9inc/mizu/shared/logger"
+	"mime"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+type reqResp struct { // hello, generics in Go
+	Req  *har.Request
+	Resp *har.Response
+}
+
+type SpecGen struct {
+	oas  *openapi.OpenAPI
+	tree *Node
+	lock sync.Mutex
+}
+
+func NewGen(server string) *SpecGen {
+	spec := new(openapi.OpenAPI)
+	spec.Version = "3.1.0"
+
+	info := openapi.Info{Title: server}
+	info.Version = "0.0"
+	spec.Info = &info
+	spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
+
+	spec.Servers = make([]*openapi.Server, 0)
+	spec.Servers = append(spec.Servers, &openapi.Server{URL: server})
+
+	gen := SpecGen{oas: spec, tree: new(Node)}
+	return &gen
+}
+
+func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
+	g.oas = oas
+	for pathStr, pathObj := range oas.Paths.Items {
+		pathSplit := strings.Split(string(pathStr), "/")
+		g.tree.getOrSet(pathSplit, pathObj)
+	}
+}
+
+func (g *SpecGen) feedEntry(entry har.Entry) (string, error) {
+	g.lock.Lock()
+	defer g.lock.Unlock()
+
+	opId, err := g.handlePathObj(&entry)
+	if err != nil {
+		return "", err
+	}
+
+	// NOTE: opId can be empty for some failed entries
+	return opId, err
+}
+
+func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
+	g.lock.Lock()
+	defer g.lock.Unlock()
+
+	g.tree.compact()
+
+	for _, pathop := range g.tree.listOps() {
+		if pathop.op.Summary == "" {
+			pathop.op.Summary = pathop.path
+		}
+	}
+
+	// put paths back from tree into OAS
+	g.oas.Paths = g.tree.listPaths()
+
+	suggestTags(g.oas)
+
+	// to make a deep copy, no better idea than marshal+unmarshal
+	specText, err := json.MarshalIndent(g.oas, "", "\t")
+	if err != nil {
+		return nil, err
+	}
+
+	spec := new(openapi.OpenAPI)
+	err = json.Unmarshal(specText, spec)
+	if err != nil {
+		return nil, err
+	}
+
+	return spec, err
+}
+
+func suggestTags(oas *openapi.OpenAPI) {
+	paths := getPathsKeys(oas.Paths.Items)
+	for len(paths) > 0 {
+		group := make([]string, 0)
+		group = append(group, paths[0])
+		paths = paths[1:]
+
+		pathsClone := append(paths[:0:0], paths...)
+		for _, path := range pathsClone {
+			if getSimilarPrefix([]string{group[0], path}) != "" {
+				group = append(group, path)
+				paths = deleteFromSlice(paths, path)
+			}
+		}
+
+		common := getSimilarPrefix(group)
+
+		if len(group) > 1 {
+			for _, path := range group {
+				pathObj := oas.Paths.Items[openapi.PathValue(path)]
+				for _, op := range getOps(pathObj) {
+					if op.Tags == nil {
+						op.Tags = make([]string, 0)
+					}
+					// only add tags if not present
+					if len(op.Tags) == 0 {
+						op.Tags = append(op.Tags, common)
+					}
+				}
+			}
+		}
+
+		//groups[common] = group
+	}
+}
+
+func getSimilarPrefix(strs []string) string {
+	chunked := make([][]string, 0)
+	for _, item := range strs {
+		chunked = append(chunked, strings.Split(item, "/"))
+	}
+
+	cmn := longestCommonXfix(chunked, true)
+	res := make([]string, 0)
+	for _, chunk := range cmn {
+		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
+			res = append(res, chunk)
+		}
+	}
+	return strings.Join(res[1:], ".")
+}
+
+func deleteFromSlice(s []string, val string) []string {
+	temp := s[:0]
+	for _, x := range s {
+		if x != val {
+			temp = append(temp, x)
+		}
+	}
+	return temp
+}
+
+func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
+	keys := make([]string, len(mymap))
+
+	i := 0
+	for k := range mymap {
+		keys[i] = string(k)
+		i++
+	}
+	return keys
+}
+
+func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
+	urlParsed, err := url.Parse(entry.Request.URL)
+	if err != nil {
+		return "", err
+	}
+
+	if isExtIgnored(urlParsed.Path) {
+		logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
+	}
+
+	ctype := getRespCtype(entry.Response)
+	if isCtypeIgnored(ctype) {
+		logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
+	}
+
+	if entry.Response.Status < 100 {
+		logger.Log.Debugf("Dropped traffic entry due to status<100: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	if entry.Response.Status == 301 || entry.Response.Status == 308 {
+		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	split := strings.Split(urlParsed.Path, "/")
+	node := g.tree.getOrSet(split, new(openapi.PathObj))
+	opObj, err := handleOpObj(entry, node.ops)
+
+	if opObj != nil {
+		return opObj.OperationID, err
+	}
+
+	return "", err
+}
+
+func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation, error) {
+	isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
+	opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
+	if err != nil {
+		return nil, err
+	}
+
+	if !isSuccess && wasMissing {
+		logger.Log.Debugf("Dropped traffic entry due to failed status and no known endpoint at: %s", entry.StartedDateTime)
+		return nil, nil
+	}
+
+	err = handleRequest(entry.Request, opObj, isSuccess)
+	if err != nil {
+		return nil, err
+	}
+
+	err = handleResponse(entry.Response, opObj, isSuccess)
+	if err != nil {
+		return nil, err
+	}
+
+	return opObj, nil
+}
+
+func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error {
+	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
+
+	qstrGW := nvParams{
+		In: openapi.InQuery,
+		Pairs: func() []NVPair {
+			return qstrToNVP(req.QueryString)
+		},
+		IsIgnored:      func(name string) bool { return false },
+		GeneralizeName: func(name string) string { return name },
+	}
+	handleNameVals(qstrGW, &opObj.Parameters)
+
+	hdrGW := nvParams{
+		In: openapi.InHeader,
+		Pairs: func() []NVPair {
+			return hdrToNVP(req.Headers)
+		},
+		IsIgnored:      isHeaderIgnored,
+		GeneralizeName: strings.ToLower,
+	}
+	handleNameVals(hdrGW, &opObj.Parameters)
+
+	if req.PostData != nil && req.PostData.Text != "" && isSuccess {
+		reqBody, err := getRequestBody(req, opObj, isSuccess)
+		if err != nil {
+			return err
+		}
+
+		if reqBody != nil {
+			reqCtype := getReqCtype(req)
+			reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, err)
+			if err != nil {
+				return err
+			}
+
+			_ = reqMedia
+		}
+	}
+	return nil
+}
+
+func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool) error {
+	// TODO: we don't support "default" response
+	respObj, err := getResponseObj(resp, opObj, isSuccess)
+	if err != nil {
+		return err
+	}
+
+	handleRespHeaders(resp.Headers, respObj)
+
+	respCtype := getRespCtype(resp)
+	respContent := respObj.Content
+	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, err)
+	if err != nil {
+		return err
+	}
+	_ = respMedia
+	return nil
+}
+
+func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj) {
+	visited := map[string]*openapi.HeaderObj{}
+	for _, pair := range reqHeaders {
+		if isHeaderIgnored(pair.Name) {
+			continue
+		}
+
+		nameGeneral := strings.ToLower(pair.Name)
+
+		initHeaders(respObj)
+		objHeaders := respObj.Headers
+		param := findHeaderByName(&respObj.Headers, pair.Name)
+		if param == nil {
+			param = createHeader(openapi.TypeString)
+			objHeaders[nameGeneral] = param
+		}
+		exmp := &param.Examples
+		err := fillParamExample(&exmp, pair.Value)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+		visited[nameGeneral] = param
+	}
+
+	// maintain "required" flag
+	if respObj.Headers != nil {
+		for name, param := range respObj.Headers {
+			paramObj, err := param.ResolveHeader(headerResolver)
+			if err != nil {
+				logger.Log.Warningf("Failed to resolve param: %s", err)
+				continue
+			}
+
+			_, ok := visited[strings.ToLower(name)]
+			if !ok {
+				flag := false
+				paramObj.Required = &flag
+			}
+		}
+	}
+
+	return
+}
+
+func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err error) (*openapi.MediaType, error) {
+	content, found := respContent[ctype]
+	if !found {
+		respContent[ctype] = &openapi.MediaType{}
+		content = respContent[ctype]
+	}
+
+	var text string
+	if reqResp.Req != nil {
+		text = reqResp.Req.PostData.Text
+	} else {
+		text = decRespText(reqResp.Resp.Content)
+	}
+
+	var exampleMsg []byte
+	// try treating it as json
+	any, isJSON := anyJSON(text)
+	if isJSON {
+		// re-marshal with forced indent
+		exampleMsg, err = json.MarshalIndent(any, "", "\t")
+		if err != nil {
+			panic("Failed to re-marshal value, super-strange")
+		}
+	} else {
+		exampleMsg, err = json.Marshal(text)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	content.Example = exampleMsg
+	return respContent[ctype], nil
+}
+
+func decRespText(content *har.Content) (res string) {
+	res = string(content.Text)
+	if content.Encoding == "base64" {
+		data, err := base64.StdEncoding.DecodeString(res)
+		if err != nil {
+			logger.Log.Warningf("error decoding response text as base64: %s", err)
+		} else {
+			res = string(data)
+		}
+	}
+	return
+}
+
+func getRespCtype(resp *har.Response) string {
+	var ctype string
+	ctype = resp.Content.MimeType
+	for _, hdr := range resp.Headers {
+		if strings.ToLower(hdr.Name) == "content-type" {
+			ctype = hdr.Value
+		}
+	}
+
+	mediaType, _, err := mime.ParseMediaType(ctype)
+	if err != nil {
+		return ""
+	}
+	return mediaType
+}
+
+func getReqCtype(req *har.Request) string {
+	var ctype string
+	ctype = req.PostData.MimeType
+	for _, hdr := range req.Headers {
+		if strings.ToLower(hdr.Name) == "content-type" {
+			ctype = hdr.Value
+		}
+	}
+
+	mediaType, _, err := mime.ParseMediaType(ctype)
+	if err != nil {
+		return ""
+	}
+	return mediaType
+}
+
+func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool) (*openapi.ResponseObj, error) {
+	statusStr := strconv.Itoa(resp.Status)
+
+	var response openapi.Response
+	response, found := opObj.Responses[statusStr]
+	if !found {
+		if opObj.Responses == nil {
+			opObj.Responses = map[string]openapi.Response{}
+		}
+
+		opObj.Responses[statusStr] = &openapi.ResponseObj{Content: map[string]*openapi.MediaType{}}
+		response = opObj.Responses[statusStr]
+	}
+
+	resResponse, err := response.ResolveResponse(responseResolver)
+	if err != nil {
+		return nil, err
+	}
+
+	if isSuccess {
+		resResponse.Description = "Successful call with status " + statusStr
+	} else {
+		resResponse.Description = "Failed call with status " + statusStr
+	}
+	return resResponse, nil
+}
+
+func getRequestBody(req *har.Request, opObj *openapi.Operation, isSuccess bool) (*openapi.RequestBodyObj, error) {
+	if opObj.RequestBody == nil {
+		opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
+	}
+
+	reqBody, err := opObj.RequestBody.ResolveRequestBody(reqBodyResolver)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: maintain required flag for it, but only consider successful responses
+	//reqBody.Content[]
+
+	return reqBody, nil
+}
+
+func getOpObj(pathObj *openapi.PathObj, method string, createIfNone bool) (*openapi.Operation, bool, error) {
+	method = strings.ToLower(method)
+	var op **openapi.Operation
+
+	switch method {
+	case "get":
+		op = &pathObj.Get
+	case "put":
+		op = &pathObj.Put
+	case "post":
+		op = &pathObj.Post
+	case "delete":
+		op = &pathObj.Delete
+	case "options":
+		op = &pathObj.Options
+	case "head":
+		op = &pathObj.Head
+	case "patch":
+		op = &pathObj.Patch
+	case "trace":
+		op = &pathObj.Trace
+	default:
+		return nil, false, errors.New("unsupported HTTP method: " + method)
+	}
+
+	isMissing := false
+	if *op == nil {
+		isMissing = true
+		if createIfNone {
+			*op = &openapi.Operation{Responses: map[string]openapi.Response{}}
+			newUUID := uuid.New().String()
+			(**op).OperationID = newUUID
+		} else {
+			return nil, isMissing, nil
+		}
+	}
+
+	return *op, isMissing, nil
+}

agent/pkg/oas/specgen_test.go

@@ -0,0 +1,197 @@
+package oas
+
+import (
+	"encoding/json"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+)
+
+// if started via env, write file into subdir
+func writeFiles(label string, spec *openapi.OpenAPI) {
+	if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
+		path := "./oas-samples"
+		err := os.MkdirAll(path, 0o755)
+		if err != nil {
+			panic(err)
+		}
+		content, err := json.MarshalIndent(spec, "", "\t")
+		if err != nil {
+			panic(err)
+		}
+		err = ioutil.WriteFile(path+"/"+label+".json", content, 0644)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+func TestEntries(t *testing.T) {
+	files, err := getFiles("./test_artifacts/")
+	// files, err = getFiles("/media/bigdisk/UP9")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	GetOasGeneratorInstance().Start()
+
+	if err := feedEntries(files); err != nil {
+		t.Log(err)
+		t.Fail()
+	}
+
+	loadStartingOAS()
+
+	svcs := strings.Builder{}
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+		gen := val.(*SpecGen)
+		svc := key.(string)
+		svcs.WriteString(svc + ",")
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+			return false
+		}
+
+		err = spec.Validate()
+		if err != nil {
+			specText, _ := json.MarshalIndent(spec, "", "\t")
+			t.Log(string(specText))
+			t.Log(err)
+			t.FailNow()
+		}
+
+		return true
+	})
+
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+		svc := key.(string)
+		gen := val.(*SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		specText, _ := json.MarshalIndent(spec, "", "\t")
+		t.Logf("%s", string(specText))
+
+		err = spec.Validate()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+		writeFiles(svc, spec)
+
+		return true
+	})
+
+}
+
+func TestFileLDJSON(t *testing.T) {
+	GetOasGeneratorInstance().Start()
+	file := "test_artifacts/output_rdwtyeoyrj.har.ldjson"
+	err := feedFromLDJSON(file)
+	if err != nil {
+		logger.Log.Warning("Failed processing file: " + err.Error())
+		t.Fail()
+	}
+
+	loadStartingOAS()
+
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(_, val interface{}) bool {
+		gen := val.(*SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		specText, _ := json.MarshalIndent(spec, "", "\t")
+		t.Logf("%s", string(specText))
+
+		err = spec.Validate()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		return true
+	})
+}
+
+func loadStartingOAS() {
+	file := "test_artifacts/catalogue.json"
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		panic(err)
+	}
+
+	var doc *openapi.OpenAPI
+	err = json.Unmarshal(data, &doc)
+	if err != nil {
+		panic(err)
+	}
+
+	gen := NewGen("catalogue")
+	gen.StartFromSpec(doc)
+
+	GetOasGeneratorInstance().ServiceSpecs.Store("catalogue", gen)
+
+	return
+}
+
+func TestEntriesNegative(t *testing.T) {
+	files := []string{"invalid"}
+	err := feedEntries(files)
+	if err == nil {
+		t.Logf("Should have failed")
+		t.Fail()
+	}
+}
+
+func TestLoadValidHAR(t *testing.T) {
+	inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}`
+	var entry *har.Entry
+	var err = json.Unmarshal([]byte(inp), &entry)
+	if err != nil {
+		t.Logf("Failed to decode entry: %s", err)
+		// t.FailNow() demonstrates the problem of library
+	}
+}
+
+func TestLoadValid3_1(t *testing.T) {
+	fd, err := os.Open("test_artifacts/catalogue.json")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	var oas openapi.OpenAPI
+	err = json.Unmarshal(data, &oas)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+	return
+}

agent/pkg/oas/test_artifacts/catalogue.json

@@ -0,0 +1,51 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "Preloaded",
+    "version": "0.1",
+    "description": "Test file for loading pre-existing OAS"
+  },
+  "paths": {
+    "/catalogue/{id}": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ],
+      "get": {
+        "parameters": [        {
+          "name": "non-required-header",
+          "in": "header",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+        ]
+      }
+    },
+    "/catalogue/{id}/details": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "style": "simple",
+          "required": true,
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ]
+    }
+  }
+}

agent/pkg/oas/test_artifacts/output_rdwtyeoyrj.har.ldjson

@@ -0,0 +1,13 @@
+{"messageType": "http", "_source": null, "firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78415179Z", "time": 13, "request": {"method": "GET", "url": "http://catalogue/catalogue/size?tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "x-some", "value": "demo val"},{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [{"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "22"}], "content": {"size": 22, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInNpemUiOjl9", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 22}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 13}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.784918698Z", "time": 19, "request": {"method": "GET", "url": "http://catalogue/catalogue?page=1&size=6&tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "page", "value": "1"}, {"name": "size", "value": "6"}, {"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "1927"}], "content": {"size": 1927, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19LHsiaWQiOiI4MzdhYjE0MS0zOTllLTRjMWYtOWFiYy1iYWNlNDAyOTZiYWMiLCJuYW1lIjoiQ2F0IHNvY2tzIiwiZGVzY3JpcHRpb24iOiJjb25zZXF1YXQgYW1ldCBjdXBpZGF0YXQgbWluaW0gbGFib3J1bSB0ZW1wb3IgZWxpdCBleCBjb25zZXF1YXQgaW4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2NrczIuanBnIl0sInByaWNlIjoxNSwiY291bnQiOjE3NSwidGFnIjpbImJyb3duIiwiZm9ybWFsIiwiZ3JlZW4iXX1dCg==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1927}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 19}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78418182Z", "time": 7, "request": {"method": "GET", "url": "http://catalogue/tags", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "107"}], "content": {"size": 107, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInRhZ3MiOlsiYnJvd24iLCJnZWVrIiwiZm9ybWFsIiwiYmx1ZSIsInNraW4iLCJyZWQiLCJhY3Rpb24iLCJzcG9ydCIsImJsYWNrIiwibWFnaWMiLCJncmVlbiJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 107}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 7}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:18.131501482Z", "time": 5, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}, {"name": "x-some", "value": "demoval"}], ",queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 5}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:18.379836908Z", "time": 14, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 14}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.920540124Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/808a2de1-1aaa-4c25-a9b9-6612e8f29a38", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "275"}], "content": {"size": 275, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NzM4LCJkZXNjcmlwdGlvbiI6IkEgbWF0dXJlIHNvY2ssIGNyb3NzZWQsIHdpdGggYW4gYWlyIG9mIG5vbmNoYWxhbmNlLiIsImlkIjoiODA4YTJkZTEtMWFhYS00YzI1LWE5YjktNjYxMmU4ZjI5YTM4IiwiaW1hZ2VVcmwiOlsiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMS5qcGVnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMi5qcGVnIl0sIm5hbWUiOiJDcm9zc2VkIiwicHJpY2UiOjE3LjMyLCJ0YWciOlsiYmx1ZSIsInJlZCIsImFjdGlvbiIsImZvcm1hbCJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 275}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.921609501Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?sort=id&size=3&tags=blue", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "3"}, {"name": "tags", "value": "blue"}, {"name": "sort", "value": "id"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "789"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}], "content": {"size": 789, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJuYW1lIjoiQ29sb3VyZnVsIiwiZGVzY3JpcHRpb24iOiJwcm9pZGVudCBvY2NhZWNhdCBpcnVyZSBldCBleGNlcHRldXIgbGFib3JlIG1pbmltIG5pc2kgYW1ldCBpcnVyZSIsImltYWdlVXJsIjpbIi9jYXRhbG9ndWUvaW1hZ2VzL2NvbG91cmZ1bF9zb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIl0sInByaWNlIjoxOCwiY291bnQiOjQzOCwidGFnIjpbImJsdWUiXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiYmx1ZSJdfV0K", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 789}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.923197848Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:23.175549218Z", "time": 26, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 26}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.239777333Z", "time": 10, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Transfer-Encoding", "value": "chunked"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "X-Application-Context", "value": "carts:80"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 10}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.725866772Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?size=5", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "5"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "1643"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}], "content": {"size": 1643, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19XQo=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1643}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.729303217Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}

agent/pkg/oas/tree.go

@@ -0,0 +1,205 @@
+package oas
+
+import (
+	"github.com/chanced/openapi"
+	"github.com/up9inc/mizu/shared/logger"
+	"strconv"
+	"strings"
+)
+
+type NodePath = []string
+
+type Node struct {
+	constant *string
+	param    *openapi.ParameterObj
+	ops      *openapi.PathObj
+	parent   *Node
+	children []*Node
+}
+
+func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Node) {
+	if pathObjToSet == nil {
+		panic("Invalid function call")
+	}
+
+	pathChunk := path[0]
+	chunkIsParam := strings.HasPrefix(pathChunk, "{") && strings.HasSuffix(pathChunk, "}")
+	chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
+
+	var paramObj *openapi.ParameterObj
+	if chunkIsParam && pathObjToSet != nil {
+		paramObj = findParamByName(pathObjToSet.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
+	}
+
+	if paramObj == nil {
+		node = n.searchInConstants(pathChunk)
+	}
+
+	if node == nil {
+		node = n.searchInParams(paramObj, chunkIsGibberish)
+	}
+
+	// still no node found, should create it
+	if node == nil {
+		node = new(Node)
+		node.parent = n
+		n.children = append(n.children, node)
+
+		if paramObj != nil {
+			node.param = paramObj
+		} else if chunkIsGibberish {
+			initParams(&pathObjToSet.Parameters)
+
+			newParam := n.createParam()
+			node.param = newParam
+
+			appended := append(*pathObjToSet.Parameters, newParam)
+			pathObjToSet.Parameters = &appended
+		} else {
+			node.constant = &pathChunk
+		}
+	}
+
+	// add example if it's a param
+	if node.param != nil && !chunkIsParam {
+		exmp := &node.param.Examples
+		err := fillParamExample(&exmp, pathChunk)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+	}
+
+	// TODO: eat up trailing slash, in a smart way: node.ops!=nil && path[1]==""
+	if len(path) > 1 {
+		return node.getOrSet(path[1:], pathObjToSet)
+	} else if node.ops == nil {
+		node.ops = pathObjToSet
+	}
+
+	return node
+}
+
+func (n *Node) createParam() *openapi.ParameterObj {
+	name := "param"
+
+	// REST assumption, not always correct
+	if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
+		name = *n.constant
+		name = name[:len(name)-2] + "Id"
+	} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
+		name = *n.constant
+		name = name[:len(name)-1] + "Id"
+	}
+
+	newParam := createSimpleParam(name, "path", "string")
+	x := n.countParentParams()
+	if x > 1 {
+		newParam.Name = newParam.Name + strconv.Itoa(x)
+	}
+
+	return newParam
+}
+
+func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunkIsGibberish bool) *Node {
+	// look among params
+	if paramObj != nil || chunkIsGibberish {
+		for _, subnode := range n.children {
+			if subnode.constant != nil {
+				continue
+			}
+
+			// TODO: check the regex pattern of param? for exceptions etc
+
+			if paramObj != nil {
+				// TODO: mergeParam(subnode.param, paramObj)
+				return subnode
+			} else {
+				return subnode
+			}
+		}
+	}
+	return nil
+}
+
+func (n *Node) searchInConstants(pathChunk string) *Node {
+	// look among constants
+	for _, subnode := range n.children {
+		if subnode.constant == nil {
+			continue
+		}
+
+		if *subnode.constant == pathChunk {
+			return subnode
+		}
+	}
+	return nil
+}
+
+func (n *Node) compact() {
+	// TODO
+}
+
+func (n *Node) listPaths() *openapi.Paths {
+	paths := &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
+
+	var strChunk string
+	if n.constant != nil {
+		strChunk = *n.constant
+	} else if n.param != nil {
+		strChunk = "{" + n.param.Name + "}"
+	} else {
+		// this is the root node
+	}
+
+	// add self
+	if n.ops != nil {
+		paths.Items[openapi.PathValue(strChunk)] = n.ops
+	}
+
+	// recurse into children
+	for _, child := range n.children {
+		subPaths := child.listPaths()
+		for path, pathObj := range subPaths.Items {
+			var concat string
+			if n.parent == nil {
+				concat = string(path)
+			} else {
+				concat = strChunk + "/" + string(path)
+			}
+			paths.Items[openapi.PathValue(concat)] = pathObj
+		}
+	}
+
+	return paths
+}
+
+type PathAndOp struct {
+	path string
+	op   *openapi.Operation
+}
+
+func (n *Node) listOps() []PathAndOp {
+	res := make([]PathAndOp, 0)
+	for path, pathObj := range n.listPaths().Items {
+		for _, op := range getOps(pathObj) {
+			res = append(res, PathAndOp{path: string(path), op: op})
+		}
+	}
+	return res
+}
+
+func (n *Node) countParentParams() int {
+	res := 0
+	node := n
+	for {
+		if node.param != nil {
+			res++
+		}
+
+		if node.parent == nil {
+			break
+		}
+		node = node.parent
+	}
+	return res
+}

agent/pkg/oas/utils.go

@@ -0,0 +1,344 @@
+package oas
+
+import (
+	"encoding/json"
+	"errors"
+	"github.com/chanced/openapi"
+	"github.com/google/martian/har"
+	"github.com/up9inc/mizu/shared/logger"
+	"strconv"
+	"strings"
+)
+
+func exampleResolver(ref string) (*openapi.ExampleObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func responseResolver(ref string) (*openapi.ResponseObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func reqBodyResolver(ref string) (*openapi.RequestBodyObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func paramResolver(ref string) (*openapi.ParameterObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func headerResolver(ref string) (*openapi.HeaderObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func initParams(obj **openapi.ParameterList) {
+	if *obj == nil {
+		var params openapi.ParameterList
+		params = make([]openapi.Parameter, 0)
+		*obj = &params
+	}
+}
+
+func initHeaders(respObj *openapi.ResponseObj) {
+	if respObj.Headers == nil {
+		var created openapi.Headers
+		created = map[string]openapi.Header{}
+		respObj.Headers = created
+	}
+}
+
+func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *openapi.ParameterObj {
+	if name == "" {
+		panic("Cannot create parameter with empty name")
+	}
+	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
+	schema := new(openapi.SchemaObj)
+	schema.Type = make(openapi.Types, 0)
+	schema.Type = append(schema.Type, ptype)
+
+	style := openapi.StyleSimple
+	if in == openapi.InQuery {
+		style = openapi.StyleForm
+	}
+
+	newParam := openapi.ParameterObj{
+		Name:     name,
+		In:       in,
+		Style:    string(style),
+		Examples: map[string]openapi.Example{},
+		Schema:   schema,
+		Required: &required,
+	}
+	return &newParam
+}
+
+func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (pathParam *openapi.ParameterObj) {
+	caseInsensitive := in == openapi.InHeader
+	for _, param := range *params {
+		paramObj, err := param.ResolveParameter(paramResolver)
+		if err != nil {
+			logger.Log.Warningf("Failed to resolve reference: %s", err)
+			continue
+		}
+
+		if paramObj.In != in {
+			continue
+		}
+
+		if paramObj.Name == name || (caseInsensitive && strings.ToLower(paramObj.Name) == strings.ToLower(name)) {
+			pathParam = paramObj
+			break
+		}
+	}
+	return pathParam
+}
+
+func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj {
+	for hname, param := range *headers {
+		hdrObj, err := param.ResolveHeader(headerResolver)
+		if err != nil {
+			logger.Log.Warningf("Failed to resolve reference: %s", err)
+			continue
+		}
+
+		if strings.ToLower(hname) == strings.ToLower(name) {
+			return hdrObj
+		}
+	}
+	return nil
+}
+
+type NVPair struct {
+	Name  string
+	Value string
+}
+
+type nvParams struct {
+	In             openapi.In
+	Pairs          func() []NVPair
+	IsIgnored      func(name string) bool
+	GeneralizeName func(name string) string
+}
+
+func qstrToNVP(list []har.QueryString) []NVPair {
+	res := make([]NVPair, len(list))
+	for idx, val := range list {
+		res[idx] = NVPair{Name: val.Name, Value: val.Value}
+	}
+	return res
+}
+
+func hdrToNVP(list []har.Header) []NVPair {
+	res := make([]NVPair, len(list))
+	for idx, val := range list {
+		res[idx] = NVPair{Name: val.Name, Value: val.Value}
+	}
+	return res
+}
+
+func handleNameVals(gw nvParams, params **openapi.ParameterList) {
+	visited := map[string]*openapi.ParameterObj{}
+	for _, pair := range gw.Pairs() {
+		if gw.IsIgnored(pair.Name) {
+			continue
+		}
+
+		nameGeneral := gw.GeneralizeName(pair.Name)
+
+		initParams(params)
+		param := findParamByName(*params, gw.In, pair.Name)
+		if param == nil {
+			param = createSimpleParam(nameGeneral, gw.In, openapi.TypeString)
+			appended := append(**params, param)
+			*params = &appended
+		}
+		exmp := &param.Examples
+		err := fillParamExample(&exmp, pair.Value)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+		visited[nameGeneral] = param
+	}
+
+	// maintain "required" flag
+	if *params != nil {
+		for _, param := range **params {
+			paramObj, err := param.ResolveParameter(paramResolver)
+			if err != nil {
+				logger.Log.Warningf("Failed to resolve param: %s", err)
+				continue
+			}
+			if paramObj.In != gw.In {
+				continue
+			}
+
+			_, ok := visited[strings.ToLower(paramObj.Name)]
+			if !ok {
+				flag := false
+				paramObj.Required = &flag
+			}
+		}
+	}
+}
+
+func createHeader(ptype openapi.SchemaType) *openapi.HeaderObj {
+	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
+	schema := new(openapi.SchemaObj)
+	schema.Type = make(openapi.Types, 0)
+	schema.Type = append(schema.Type, ptype)
+
+	style := openapi.StyleSimple
+	newParam := openapi.HeaderObj{
+		Style:    string(style),
+		Examples: map[string]openapi.Example{},
+		Schema:   schema,
+		Required: &required,
+	}
+	return &newParam
+}
+
+func fillParamExample(param **openapi.Examples, exampleValue string) error {
+	if **param == nil {
+		**param = map[string]openapi.Example{}
+	}
+
+	cnt := 0
+	for _, example := range **param {
+		cnt++
+		exampleObj, err := example.ResolveExample(exampleResolver)
+		if err != nil {
+			continue
+		}
+
+		var value string
+		err = json.Unmarshal(exampleObj.Value, &value)
+		if err != nil {
+			logger.Log.Warningf("Failed decoding parameter example into string: %s", err)
+			continue
+		}
+
+		if value == exampleValue || cnt > 5 { // 5 examples is enough
+			return nil
+		}
+	}
+
+	valMsg, err := json.Marshal(exampleValue)
+	if err != nil {
+		return err
+	}
+
+	themap := **param
+	themap["example #"+strconv.Itoa(cnt)] = &openapi.ExampleObj{Value: valMsg}
+
+	return nil
+}
+
+func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.com/jpillora/longestcommon
+	empty := make([]string, 0)
+	//short-circuit empty list
+	if len(strs) == 0 {
+		return empty
+	}
+	xfix := strs[0]
+	//short-circuit single-element list
+	if len(strs) == 1 {
+		return xfix
+	}
+	//compare first to rest
+	for _, str := range strs[1:] {
+		xfixl := len(xfix)
+		strl := len(str)
+		//short-circuit empty strings
+		if xfixl == 0 || strl == 0 {
+			return empty
+		}
+		//maximum possible length
+		maxl := xfixl
+		if strl < maxl {
+			maxl = strl
+		}
+		//compare letters
+		if pre {
+			//prefix, iterate left to right
+			for i := 0; i < maxl; i++ {
+				if xfix[i] != str[i] {
+					xfix = xfix[:i]
+					break
+				}
+			}
+		} else {
+			//suffix, iternate right to left
+			for i := 0; i < maxl; i++ {
+				xi := xfixl - i - 1
+				si := strl - i - 1
+				if xfix[xi] != str[si] {
+					xfix = xfix[xi+1:]
+					break
+				}
+			}
+		}
+	}
+	return xfix
+}
+
+// returns all non-nil ops in PathObj
+func getOps(pathObj *openapi.PathObj) []*openapi.Operation {
+	ops := []**openapi.Operation{&pathObj.Get, &pathObj.Patch, &pathObj.Put, &pathObj.Options, &pathObj.Post, &pathObj.Trace, &pathObj.Head, &pathObj.Delete}
+	res := make([]*openapi.Operation, 0)
+	for _, opp := range ops {
+		if *opp == nil {
+			continue
+		}
+		res = append(res, *opp)
+	}
+	return res
+}
+
+// parses JSON into any possible value
+func anyJSON(text string) (anyVal interface{}, isJSON bool) {
+	isJSON = true
+	asMap := map[string]interface{}{}
+	err := json.Unmarshal([]byte(text), &asMap)
+	if err == nil && asMap != nil {
+		return asMap, isJSON
+	}
+
+	asArray := make([]interface{}, 0)
+	err = json.Unmarshal([]byte(text), &asArray)
+	if err == nil && asArray != nil {
+		return asArray, isJSON
+	}
+
+	asString := ""
+	sPtr := &asString
+	err = json.Unmarshal([]byte(text), &sPtr)
+	if err == nil && sPtr != nil {
+		return asString, isJSON
+	}
+
+	asInt := 0
+	intPtr := &asInt
+	err = json.Unmarshal([]byte(text), &intPtr)
+	if err == nil && intPtr != nil {
+		return asInt, isJSON
+	}
+
+	asFloat := 0.0
+	floatPtr := &asFloat
+	err = json.Unmarshal([]byte(text), &floatPtr)
+	if err == nil && floatPtr != nil {
+		return asFloat, isJSON
+	}
+
+	asBool := false
+	boolPtr := &asBool
+	err = json.Unmarshal([]byte(text), &boolPtr)
+	if err == nil && boolPtr != nil {
+		return asBool, isJSON
+	}
+
+	if text == "null" {
+		return nil, isJSON
+	}
+
+	return nil, false
+}

agent/pkg/oas/utils_test.go

@@ -0,0 +1,35 @@
+package oas
+
+import (
+	"testing"
+)
+
+func TestAnyJSON(t *testing.T) {
+	testCases := []struct {
+		inp    string
+		isJSON bool
+		out    interface{}
+	}{
+		{`{"key": 1, "keyNull": null}`, true, nil},
+		{`[{"key": "val"}, ["subarray"], "string", 1, 2.2, true, null]`, true, nil},
+		{`"somestring"`, true, "somestring"},
+		{"0", true, 0},
+		{"0.5", true, 0.5},
+		{"true", true, true},
+		{"null", true, nil},
+		{"sabbra cadabra", false, nil},
+		{"0.1.2.3", false, nil},
+	}
+	for _, tc := range testCases {
+		any, isJSON := anyJSON(tc.inp)
+		if isJSON != tc.isJSON {
+			t.Errorf("Parse flag mismatch: %t != %t", tc.isJSON, isJSON)
+		} else if isJSON && tc.out != nil && tc.out != any {
+			t.Errorf("%s != %s", any, tc.out)
+		} else if tc.inp == "null" && any != nil {
+			t.Errorf("null has to parse as nil (but got %s)", any)
+		} else {
+			t.Logf("%s => %s", tc.inp, any)
+		}
+	}
+}

agent/pkg/providers/tapConfig/tap_config_provider.go

@@ -1,40 +1,32 @@
 package tapConfig
 
 import (
-	"encoding/json"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
-	"io/ioutil"
 	"mizuserver/pkg/models"
+	"mizuserver/pkg/utils"
 	"os"
 	"sync"
 )
 
 const FilePath = shared.DataDirPath + "tap-config.json"
 
-var lock = &sync.Mutex{}
-
-var config *models.TapConfig
+var (
+	lock     = &sync.Mutex{}
+	syncOnce sync.Once
+	config   *models.TapConfig
+)
 
 func Get() *models.TapConfig {
-	if config == nil {
-		lock.Lock()
-		defer lock.Unlock()
+	syncOnce.Do(func() {
+		if err := utils.ReadJsonFile(FilePath, &config); err != nil {
+			config = &models.TapConfig{TappedNamespaces: make(map[string]bool)}
 
-		if config == nil {
-			if content, err := ioutil.ReadFile(FilePath); err != nil {
-				config = &models.TapConfig{TappedNamespaces: make(map[string]bool)}
-				if !os.IsNotExist(err) {
-					logger.Log.Errorf("Error loading tap config from file, err: %v", err)
-				}
-			} else {
-				if err = json.Unmarshal(content, &config); err != nil {
-					config = &models.TapConfig{TappedNamespaces: make(map[string]bool)}
-					logger.Log.Errorf("Error while unmarshal tap config, err: %v", err)
-				}
+			if !os.IsNotExist(err) {
+				logger.Log.Errorf("Error reading tap config from file, err: %v", err)
 			}
 		}
-	}
+	})
 
 	return config
 }
@@ -44,11 +36,7 @@ func Save(tapConfigToSave *models.TapConfig) {
 	defer lock.Unlock()
 
 	config = tapConfigToSave
-	if data, err := json.Marshal(config); err != nil {
-		logger.Log.Errorf("Error while marshal tap config, err: %v", err)
-	} else {
-		if err := ioutil.WriteFile(FilePath, data, 0644); err != nil {
-			logger.Log.Errorf("Error writing tap config to file, err: %v", err)
-		}
+	if err := utils.SaveJsonFile(FilePath, config); err != nil {
+		logger.Log.Errorf("Error saving tap config, err: %v", err)
 	}
 }

agent/pkg/providers/tappedPods/tapped_pods_provider.go

@@ -2,25 +2,49 @@ package tappedPods
 
 import (
 	"github.com/up9inc/mizu/shared"
-	"mizuserver/pkg/providers/tappersStatus"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/providers/tappers"
+	"mizuserver/pkg/utils"
+	"os"
 	"strings"
+	"sync"
 )
 
-var tappedPods []*shared.PodInfo
+const FilePath = shared.DataDirPath + "tapped-pods.json"
+
+var (
+	lock       = &sync.Mutex{}
+	syncOnce   sync.Once
+	tappedPods []*shared.PodInfo
+)
 
 func Get() []*shared.PodInfo {
+	syncOnce.Do(func() {
+		if err := utils.ReadJsonFile(FilePath, &tappedPods); err != nil {
+			if !os.IsNotExist(err) {
+				logger.Log.Errorf("Error reading tapped pods from file, err: %v", err)
+			}
+		}
+	})
+
 	return tappedPods
 }
 
 func Set(tappedPodsToSet []*shared.PodInfo) {
+	lock.Lock()
+	defer lock.Unlock()
+
 	tappedPods = tappedPodsToSet
+	if err := utils.SaveJsonFile(FilePath, tappedPods); err != nil {
+		logger.Log.Errorf("Error saving tapped pods, err: %v", err)
+	}
 }
 
 func GetTappedPodsStatus() []shared.TappedPodStatus {
 	tappedPodsStatus := make([]shared.TappedPodStatus, 0)
 	for _, pod := range Get() {
 		var status string
-		if tapperStatus, ok := tappersStatus.Get()[pod.NodeName]; ok {
+		if tapperStatus, ok := tappers.GetStatus()[pod.NodeName]; ok {
 			status = strings.ToLower(tapperStatus.Status)
 		}
 

agent/pkg/providers/tappers/tappers_provider.go

@@ -0,0 +1,82 @@
+package tappers
+
+import (
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/utils"
+	"os"
+	"sync"
+)
+
+const FilePath = shared.DataDirPath + "tappers-status.json"
+
+var (
+	lockStatus = &sync.Mutex{}
+	syncOnce   sync.Once
+	status     map[string]*shared.TapperStatus
+
+	lockConnectedCount = &sync.Mutex{}
+	connectedCount     int
+)
+
+func GetStatus() map[string]*shared.TapperStatus {
+	initStatus()
+
+	return status
+}
+
+func SetStatus(tapperStatus *shared.TapperStatus) {
+	initStatus()
+
+	lockStatus.Lock()
+	defer lockStatus.Unlock()
+
+	status[tapperStatus.NodeName] = tapperStatus
+
+	saveStatus()
+}
+
+func ResetStatus() {
+	lockStatus.Lock()
+	defer lockStatus.Unlock()
+
+	status = make(map[string]*shared.TapperStatus)
+
+	saveStatus()
+}
+
+func GetConnectedCount() int {
+	return connectedCount
+}
+
+func Connected() {
+	lockConnectedCount.Lock()
+	defer lockConnectedCount.Unlock()
+
+	connectedCount++
+}
+
+func Disconnected() {
+	lockConnectedCount.Lock()
+	defer lockConnectedCount.Unlock()
+
+	connectedCount--
+}
+
+func initStatus() {
+	syncOnce.Do(func() {
+		if err := utils.ReadJsonFile(FilePath, &status); err != nil {
+			status = make(map[string]*shared.TapperStatus)
+
+			if !os.IsNotExist(err) {
+				logger.Log.Errorf("Error reading tappers status from file, err: %v", err)
+			}
+		}
+	})
+}
+
+func saveStatus() {
+	if err := utils.SaveJsonFile(FilePath, status); err != nil {
+		logger.Log.Errorf("Error saving tappers status, err: %v", err)
+	}
+}

agent/pkg/providers/tappersCount/tappers_count_provider.go

@@ -1,25 +0,0 @@
-package tappersCount
-
-import "sync"
-
-var lock = &sync.Mutex{}
-
-var tappersCount int
-
-func Add() {
-	lock.Lock()
-	defer lock.Unlock()
-
-	tappersCount++
-}
-
-func Remove() {
-	lock.Lock()
-	defer lock.Unlock()
-
-	tappersCount--
-}
-
-func Get() int {
-	return tappersCount
-}

agent/pkg/providers/tappersStatus/tappers_status_provider.go

@@ -1,25 +0,0 @@
-package tappersStatus
-
-import "github.com/up9inc/mizu/shared"
-
-var tappersStatus map[string]*shared.TapperStatus
-
-func Get() map[string]*shared.TapperStatus {
-	if tappersStatus == nil {
-		tappersStatus = make(map[string]*shared.TapperStatus)
-	}
-
-	return tappersStatus
-}
-
-func Set(tapperStatus *shared.TapperStatus) {
-	if tappersStatus == nil {
-		tappersStatus = make(map[string]*shared.TapperStatus)
-	}
-
-	tappersStatus[tapperStatus.NodeName] = tapperStatus
-}
-
-func Reset() {
-	tappersStatus = make(map[string]*shared.TapperStatus)
-}

agent/pkg/routes/oas_routes.go

@@ -0,0 +1,18 @@
+package routes
+
+import (
+	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/middlewares"
+
+	"github.com/gin-gonic/gin"
+)
+
+// OASRoutes methods to access OAS spec
+func OASRoutes(ginApp *gin.Engine) {
+	routeGroup := ginApp.Group("/oas")
+	routeGroup.Use(middlewares.RequiresAuth())
+
+	routeGroup.GET("/", controllers.GetOASServers)     // list of servers in OAS map
+	routeGroup.GET("/all", controllers.GetOASAllSpecs) // list of servers in OAS map
+	routeGroup.GET("/:id", controllers.GetOASSpec)     // get OAS spec for given server
+}

agent/pkg/routes/status_routes.go

@@ -15,7 +15,7 @@ func StatusRoutes(ginApp *gin.Engine) {
 
 	routeGroup.POST("/tappedPods", controllers.PostTappedPods)
 	routeGroup.POST("/tapperStatus", controllers.PostTapperStatus)
-	routeGroup.GET("/tappersCount", controllers.GetTappersCount)
+	routeGroup.GET("/connectedTappersCount", controllers.GetConnectedTappersCount)
 	routeGroup.GET("/tap", controllers.GetTappingStatus)
 
 	routeGroup.GET("/auth", controllers.GetAuthStatus)

agent/pkg/utils/utils.go

@@ -2,7 +2,9 @@ package utils
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -59,3 +61,27 @@ func SetHostname(address, newHostname string) string {
 	return replacedUrl.String()
 
 }
+
+func ReadJsonFile(filePath string, value interface{}) error {
+	if content, err := ioutil.ReadFile(filePath); err != nil {
+		return err
+	} else {
+		if err = json.Unmarshal(content, value); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func SaveJsonFile(filePath string, value interface{}) error {
+	if data, err := json.Marshal(value); err != nil {
+		return err
+	} else {
+		if err = ioutil.WriteFile(filePath, data, 0644); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cli/README.md.TEMPLATE

@@ -1,6 +1,7 @@
 # Mizu release _SEM_VER_
+Full changelog for stable release see in [docs](https://github.com/up9inc/mizu/blob/main/docs/CHANGELOG.md)
 
-Download Mizu for your platform
+## Download Mizu for your platform
 
 **Mac** (Intel)
 ```

cli/cmd/installRunner.go

@@ -101,7 +101,8 @@ func watchApiServerPodReady(ctx context.Context, kubernetesProvider *kubernetes.
 	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
 	eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper)
 
-	timeAfter := time.After(1 * time.Minute)
+	apiServerTimeoutSec := config.GetIntEnvConfig(config.ApiServerTimeoutSec, 120)
+	timeAfter := time.After(time.Duration(apiServerTimeoutSec) * time.Second)
 	for {
 		select {
 		case wEvent, ok := <-eventChan:

cli/cmd/tapRunner.go

@@ -306,7 +306,9 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
 	eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper)
 	isPodReady := false
-	timeAfter := time.After(25 * time.Second)
+
+	apiServerTimeoutSec := config.GetIntEnvConfig(config.ApiServerTimeoutSec, 120)
+	timeAfter := time.After(time.Duration(apiServerTimeoutSec) * time.Second)
 	for {
 		select {
 		case wEvent, ok := <-eventChan:

cli/config/configStruct.go

@@ -34,8 +34,8 @@ type ConfigStruct struct {
 	ConfigFilePath         string                      `yaml:"config-path,omitempty" readonly:""`
 	HeadlessMode           bool                        `yaml:"headless" default:"false"`
 	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
-	ServiceMap             bool                        `yaml:"service-map" default:"false" readonly:""`
-	OAS                    bool                        `yaml:"oas" default:"false" readonly:""`
+	ServiceMap             bool                        `yaml:"service-map,omitempty" default:"false" readonly:""`
+	OAS                    bool                        `yaml:"oas,omitempty" default:"false" readonly:""`
 }
 
 func (config *ConfigStruct) validate() error {

cli/config/envConfig.go

@@ -7,6 +7,7 @@ import (
 
 const (
 	ApiServerRetries = "API_SERVER_RETRIES"
+	ApiServerTimeoutSec = "API_SERVER_TIMEOUT_SEC"
 )
 
 func GetIntEnvConfig(key string, defaultValue int) int {

debug.Dockerfile

@@ -3,11 +3,12 @@ FROM node:14-slim AS site-build
 
 WORKDIR /app/ui-build
 
-COPY ui .
+COPY ui/package.json .
+COPY ui/package-lock.json .
 RUN npm i
+COPY ui .
 RUN npm run build
 
-
 FROM golang:1.16-alpine AS builder
 # Set necessary environment variables needed for our image.
 ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
@@ -34,15 +35,16 @@ ARG SEM_VER=0.0.0
 COPY shared ../shared
 COPY tap ../tap
 COPY agent .
+# Include gcflags for debugging
 RUN go build -gcflags="all=-N -l" -o mizuagent .
 
 COPY devops/build_extensions_debug.sh ..
 RUN cd .. && /bin/bash build_extensions_debug.sh
 
-
 FROM golang:1.16-alpine
 
-RUN apk add bash libpcap-dev
+# Set necessary environment variables needed for our image.
+RUN apk add bash libpcap-dev gcc g++
 
 WORKDIR /app
 
@@ -50,9 +52,16 @@ WORKDIR /app
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
 COPY --from=builder ["/app/agent/build/extensions", "extensions"]
 COPY --from=site-build ["/app/ui-build/build", "site"]
+RUN mkdir /app/data/
 
-# install remote debugging tool
+# install delve
+ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 RUN go get github.com/go-delve/delve/cmd/dlv
 
+ENV GIN_MODE=debug
+
+# delve ports
+EXPOSE 2345 2346
+
+# this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
 ENTRYPOINT "/app/mizuagent"
-#CMD ["sh", "-c", "dlv --headless=true --listen=:2345 --log --api-version=2 --accept-multiclient exec ./mizuagent -- --api-server"]

docs/CHANGELOG.md

@@ -0,0 +1,43 @@
+# CHANGELOG
+This document summarizes main and fixes changes published in stable (aka `main`) branch of this project.
+Ongoing work and development releases are under `develop` branch.
+
+## 0.22.0
+
+### main features
+* Service Mesh support -- mizu is now capable to tap mTLS traffic between pods connected by Istio service mesh
+  * Use `--service-mesh` option to enable this feature
+* New installation option - have the same Mizu functionality as long living pods in your cluster, with password protection
+  * To install use `mizu install` command
+  * To access use `mizu view` or `kubectl -n mizu port-forward svc/mizu-api-server`
+  * To uninstall run `mizu clean`
+* At first login
+  * Set admin password as prompted, use it to login to mizu later on.
+  * After login, user should select cluster namespaces to tap: by default all namespaces in the cluster are selected, user can select/unselect according to their needs. These settings are retained and can be modified at any time via Settings menu (cog icon on the top-right)
+
+
+### improvements
+* improved Mizu permissions/roles logic to support clusters with strict PodSecurityPolicy (PSP) -- see [PERMISSIONS](PERMISSIONS.md) doc for more details
+ 
+### notable bug fixes
+* mizu now works properly when API service is exposed via HTTPS url
+* mizu now properly displays KAFKA message body 
+
+
+
+
+## 0.21.0
+
+### main features
+* New traffic search & stream exprience
+* Rich query language with full-text search capabilities on headers & body
+* Distinct live-streaming vs paging/browsing modes, all with filter applied
+
+### improvements
+* GUI - source and destination IP addresses & service names for each traffic item
+* GUI - Mizu health - display warning sign in top bar when not all requested pods are successfully tapped
+* GUI - pod tapping status reflected in the list (ok or problem)
+* Mizu telemetry - report platform type
+
+### fixes
+* Request duration and body size properly shown in GUI (instead of -1)

docs/SERVICE_MESH.md

@@ -7,7 +7,7 @@ This document describe how Mizu tapper handles workloads configured with mtls, m
 The list of service meshes supported by Mizu include:
 
 - Istio
-- Linkerd
+- Linkerd (beta)
 
 ## Installation
 

shared/models.go

@@ -122,9 +122,9 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc
 }
 
 type HealthResponse struct {
-	TappedPods    []*PodInfo      `json:"tappedPods"`
-	TappersCount  int             `json:"tappersCount"`
-	TappersStatus []*TapperStatus `json:"tappersStatus"`
+	TappedPods            []*PodInfo      `json:"tappedPods"`
+	ConnectedTappersCount int             `json:"connectedTappersCount"`
+	TappersStatus         []*TapperStatus `json:"tappersStatus"`
 }
 
 type VersionResponse struct {

tap/extensions/http/helpers.go

@@ -3,14 +3,15 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"strconv"
 
 	"github.com/up9inc/mizu/tap/api"
 )
 
 func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}) {
 	newMap = make(map[string]interface{})
-	for _, header := range mapSlice {
-		h := header.(map[string]interface{})
+	for _, item := range mapSlice {
+		h := item.(map[string]interface{})
 		newMap[h["name"].(string)] = h["value"]
 	}
 
@@ -19,8 +20,8 @@ func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}
 
 func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (representation string) {
 	var table []api.TableData
-	for _, header := range mapSlice {
-		h := header.(map[string]interface{})
+	for _, item := range mapSlice {
+		h := item.(map[string]interface{})
 		selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, h["name"].(string))
 		table = append(table, api.TableData{
 			Name:     h["name"].(string),
@@ -33,3 +34,19 @@ func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (re
 	representation = string(obj)
 	return
 }
+
+func representSliceAsTable(slice []interface{}, selectorPrefix string) (representation string) {
+	var table []api.TableData
+	for i, item := range slice {
+		selector := fmt.Sprintf("%s[%d]", selectorPrefix, i)
+		table = append(table, api.TableData{
+			Name:     strconv.Itoa(i),
+			Value:    item.(interface{}),
+			Selector: selector,
+		})
+	}
+
+	obj, _ := json.Marshal(table)
+	representation = string(obj)
+	return
+}

tap/extensions/http/main.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/up9inc/mizu/tap/api"
@@ -209,6 +210,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	request["url"] = reqDetails["url"].(string)
 	reqDetails["targetUri"] = reqDetails["url"]
 	reqDetails["path"] = path
+	reqDetails["pathSegments"] = strings.Split(path, "/")[1:]
 	reqDetails["summary"] = path
 
 	// Rearrange the maps for the querying
@@ -296,6 +298,15 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
 		Data:  string(details),
 	})
 
+	pathSegments := request["pathSegments"].([]interface{})
+	if len(pathSegments) > 1 {
+		repRequest = append(repRequest, api.SectionData{
+			Type:  api.TABLE,
+			Title: "Path Segments",
+			Data:  representSliceAsTable(pathSegments, `request.pathSegments`),
+		})
+	}
+
 	repRequest = append(repRequest, api.SectionData{
 		Type:  api.TABLE,
 		Title: "Headers",

tap/extensions/kafka/go.mod

@@ -3,6 +3,8 @@ module github.com/up9inc/mizu/tap/extensions/kafka
 go 1.16
 
 require (
+	github.com/fatih/camelcase v1.0.0
+	github.com/ohler55/ojg v1.12.12
 	github.com/segmentio/kafka-go v0.4.17
 	github.com/up9inc/mizu/tap/api v0.0.0
 )

tap/extensions/kafka/go.sum

@@ -1,6 +1,8 @@
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
+github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
@@ -16,6 +18,8 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/ohler55/ojg v1.12.12 h1:hepbQFn7GHAecTPmwS3j5dCiOLsOpzPLvhiqnlAVAoE=
+github.com/ohler55/ojg v1.12.12/go.mod h1:LBbIVRAgoFbYBXQhRhuEpaJIqq+goSO63/FQ+nyJU88=
 github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=
 github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

tap/extensions/kafka/helpers.go

@@ -3,8 +3,13 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"reflect"
 	"strconv"
+	"strings"
 
+	"github.com/fatih/camelcase"
+	"github.com/ohler55/ojg/jp"
+	"github.com/ohler55/ojg/oj"
 	"github.com/up9inc/mizu/tap/api"
 )
 
@@ -289,17 +294,12 @@ func representProduceRequest(data map[string]interface{}) []interface{} {
 	rep = representRequestHeader(data, rep)
 
 	payload := data["payload"].(map[string]interface{})
-	topicData := ""
-	_topicData := payload["topicData"]
-	if _topicData != nil {
-		x, _ := json.Marshal(_topicData.([]interface{}))
-		topicData = string(x)
-	}
+	topicData := payload["topicData"]
 	transactionalID := ""
 	if payload["transactionalID"] != nil {
 		transactionalID = payload["transactionalID"].(string)
 	}
-	repPayload, _ := json.Marshal([]api.TableData{
+	repTransactionDetails, _ := json.Marshal([]api.TableData{
 		{
 			Name:     "Transactional ID",
 			Value:    transactionalID,
@@ -315,18 +315,73 @@ func representProduceRequest(data map[string]interface{}) []interface{} {
 			Value:    fmt.Sprintf("%d", int(payload["timeout"].(float64))),
 			Selector: `request.payload.timeout`,
 		},
-		{
-			Name:     "Topic Data",
-			Value:    topicData,
-			Selector: `request.payload.topicData`,
-		},
 	})
 	rep = append(rep, api.SectionData{
 		Type:  api.TABLE,
-		Title: "Payload",
-		Data:  string(repPayload),
+		Title: "Transaction Details",
+		Data:  string(repTransactionDetails),
 	})
 
+	if topicData != nil {
+		for _, _topic := range topicData.([]interface{}) {
+			topic := _topic.(map[string]interface{})
+			topicName := topic["topic"].(string)
+			partitions := topic["partitions"].(map[string]interface{})
+			partitionsJson, err := json.Marshal(partitions)
+			if err != nil {
+				return rep
+			}
+
+			repPartitions, _ := json.Marshal([]api.TableData{
+				{
+					Name:     "Length",
+					Value:    partitions["length"],
+					Selector: `request.payload.transactionalID`,
+				},
+			})
+			rep = append(rep, api.SectionData{
+				Type:  api.TABLE,
+				Title: fmt.Sprintf("Partitions (topic: %s)", topicName),
+				Data:  string(repPartitions),
+			})
+
+			obj, err := oj.ParseString(string(partitionsJson))
+			recordBatchPath, err := jp.ParseString(`partitionData.records.recordBatch`)
+			recordBatchresults := recordBatchPath.Get(obj)
+			if len(recordBatchresults) > 0 {
+				rep = append(rep, api.SectionData{
+					Type:  api.TABLE,
+					Title: fmt.Sprintf("Record Batch (topic: %s)", topicName),
+					Data:  representMapAsTable(recordBatchresults[0].(map[string]interface{}), `request.payload.topicData.partitions.partitionData.records.recordBatch`, []string{"record"}),
+				})
+			}
+
+			recordsPath, err := jp.ParseString(`partitionData.records.recordBatch.record`)
+			recordsResults := recordsPath.Get(obj)
+			if len(recordsResults) > 0 {
+				records := recordsResults[0].([]interface{})
+				for i, _record := range records {
+					record := _record.(map[string]interface{})
+					value := record["value"]
+					delete(record, "value")
+
+					rep = append(rep, api.SectionData{
+						Type:  api.TABLE,
+						Title: fmt.Sprintf("Record [%d] Details (topic: %s)", i, topicName),
+						Data:  representMapAsTable(record, fmt.Sprintf(`request.payload.topicData.partitions.partitionData.records.recordBatch.record[%d]`, i), []string{"value"}),
+					})
+
+					rep = append(rep, api.SectionData{
+						Type:     api.BODY,
+						Title:    fmt.Sprintf("Record [%d] Value", i),
+						Data:     value.(string),
+						Selector: fmt.Sprintf(`request.payload.topicData.partitions.partitionData.records.recordBatch.record[%d].value`, i),
+					})
+				}
+			}
+		}
+	}
+
 	return rep
 }
 
@@ -336,21 +391,12 @@ func representProduceResponse(data map[string]interface{}) []interface{} {
 	rep = representResponseHeader(data, rep)
 
 	payload := data["payload"].(map[string]interface{})
-	responses := ""
-	if payload["responses"] != nil {
-		_responses, _ := json.Marshal(payload["responses"].([]interface{}))
-		responses = string(_responses)
-	}
+	responses := payload["responses"]
 	throttleTimeMs := ""
 	if payload["throttleTimeMs"] != nil {
 		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
 	}
 	repPayload, _ := json.Marshal([]api.TableData{
-		{
-			Name:     "Responses",
-			Value:    string(responses),
-			Selector: `response.payload.responses`,
-		},
 		{
 			Name:     "Throttle Time (ms)",
 			Value:    throttleTimeMs,
@@ -359,10 +405,31 @@ func representProduceResponse(data map[string]interface{}) []interface{} {
 	})
 	rep = append(rep, api.SectionData{
 		Type:  api.TABLE,
-		Title: "Payload",
+		Title: "Transaction Details",
 		Data:  string(repPayload),
 	})
 
+	if responses != nil {
+		for i, _response := range responses.([]interface{}) {
+			response := _response.(map[string]interface{})
+
+			rep = append(rep, api.SectionData{
+				Type:  api.TABLE,
+				Title: fmt.Sprintf("Response [%d]", i),
+				Data:  representMapAsTable(response, fmt.Sprintf(`response.payload.responses[%d]`, i), []string{"partitionResponses"}),
+			})
+
+			for j, _partitionResponse := range response["partitionResponses"].([]interface{}) {
+				partitionResponse := _partitionResponse.(map[string]interface{})
+				rep = append(rep, api.SectionData{
+					Type:  api.TABLE,
+					Title: fmt.Sprintf("Response [%d] Partition Response [%d]", i, j),
+					Data:  representMapAsTable(partitionResponse, fmt.Sprintf(`response.payload.responses[%d].partitionResponses[%d]`, i, j), []string{}),
+				})
+			}
+		}
+	}
+
 	return rep
 }
 
@@ -372,11 +439,7 @@ func representFetchRequest(data map[string]interface{}) []interface{} {
 	rep = representRequestHeader(data, rep)
 
 	payload := data["payload"].(map[string]interface{})
-	topics := ""
-	if payload["topics"] != nil {
-		_topics, _ := json.Marshal(payload["topics"].([]interface{}))
-		topics = string(_topics)
-	}
+	topics := payload["topics"]
 	replicaId := ""
 	if payload["replicaId"] != nil {
 		replicaId = fmt.Sprintf("%d", int(payload["replicaId"].(float64)))
@@ -442,11 +505,6 @@ func representFetchRequest(data map[string]interface{}) []interface{} {
 			Value:    sessionEpoch,
 			Selector: `request.payload.sessionEpoch`,
 		},
-		{
-			Name:     "Topics",
-			Value:    topics,
-			Selector: `request.payload.topics`,
-		},
 		{
 			Name:     "Forgotten Topics Data",
 			Value:    forgottenTopicsData,
@@ -460,10 +518,26 @@ func representFetchRequest(data map[string]interface{}) []interface{} {
 	})
 	rep = append(rep, api.SectionData{
 		Type:  api.TABLE,
-		Title: "Payload",
+		Title: "Transaction Details",
 		Data:  string(repPayload),
 	})
 
+	if topics != nil {
+		for i, _topic := range topics.([]interface{}) {
+			topic := _topic.(map[string]interface{})
+			topicName := topic["topic"].(string)
+			for j, _partition := range topic["partitions"].([]interface{}) {
+				partition := _partition.(map[string]interface{})
+
+				rep = append(rep, api.SectionData{
+					Type:  api.TABLE,
+					Title: fmt.Sprintf("Partition [%d] (topic: %s)", j, topicName),
+					Data:  representMapAsTable(partition, fmt.Sprintf(`request.payload.topics[%d].partitions[%d]`, i, j), []string{}),
+				})
+			}
+		}
+	}
+
 	return rep
 }
 
@@ -473,11 +547,7 @@ func representFetchResponse(data map[string]interface{}) []interface{} {
 	rep = representResponseHeader(data, rep)
 
 	payload := data["payload"].(map[string]interface{})
-	responses := ""
-	if payload["responses"] != nil {
-		_responses, _ := json.Marshal(payload["responses"].([]interface{}))
-		responses = string(_responses)
-	}
+	responses := payload["responses"]
 	throttleTimeMs := ""
 	if payload["throttleTimeMs"] != nil {
 		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
@@ -506,18 +576,56 @@ func representFetchResponse(data map[string]interface{}) []interface{} {
 			Value:    sessionId,
 			Selector: `response.payload.sessionId`,
 		},
-		{
-			Name:     "Responses",
-			Value:    responses,
-			Selector: `response.payload.responses`,
-		},
 	})
 	rep = append(rep, api.SectionData{
 		Type:  api.TABLE,
-		Title: "Payload",
+		Title: "Transaction Details",
 		Data:  string(repPayload),
 	})
 
+	if responses != nil {
+		for i, _response := range responses.([]interface{}) {
+			response := _response.(map[string]interface{})
+			topicName := response["topic"].(string)
+
+			for j, _partitionResponse := range response["partitionResponses"].([]interface{}) {
+				partitionResponse := _partitionResponse.(map[string]interface{})
+				recordSet := partitionResponse["recordSet"].(map[string]interface{})
+
+				rep = append(rep, api.SectionData{
+					Type:  api.TABLE,
+					Title: fmt.Sprintf("Response [%d] Partition Response [%d] (topic: %s)", i, j, topicName),
+					Data:  representMapAsTable(partitionResponse, fmt.Sprintf(`response.payload.responses[%d].partitionResponses[%d]`, i, j), []string{"recordSet"}),
+				})
+
+				recordBatch := recordSet["recordBatch"].(map[string]interface{})
+				rep = append(rep, api.SectionData{
+					Type:  api.TABLE,
+					Title: fmt.Sprintf("Response [%d] Partition Response [%d] Record Batch (topic: %s)", i, j, topicName),
+					Data:  representMapAsTable(recordBatch, fmt.Sprintf(`response.payload.responses[%d].partitionResponses[%d].recordSet.recordBatch`, i, j), []string{"record"}),
+				})
+
+				for k, _record := range recordBatch["record"].([]interface{}) {
+					record := _record.(map[string]interface{})
+					value := record["value"]
+
+					rep = append(rep, api.SectionData{
+						Type:  api.TABLE,
+						Title: fmt.Sprintf("Response [%d] Partition Response [%d] Record [%d] (topic: %s)", i, j, k, topicName),
+						Data:  representMapAsTable(record, fmt.Sprintf(`response.payload.responses[%d].partitionResponses[%d].recordSet.recordBatch.record[%d]`, i, j, k), []string{"value"}),
+					})
+
+					rep = append(rep, api.SectionData{
+						Type:     api.BODY,
+						Title:    fmt.Sprintf("Response [%d] Partition Response [%d] Record [%d] Value (topic: %s)", i, j, k, topicName),
+						Data:     value.(string),
+						Selector: fmt.Sprintf(`response.payload.responses[%d].partitionResponses[%d].recordSet.recordBatch.record[%d].value`, i, j, k),
+					})
+				}
+			}
+		}
+	}
+
 	return rep
 }
 
@@ -591,17 +699,11 @@ func representCreateTopicsRequest(data map[string]interface{}) []interface{} {
 	rep = representRequestHeader(data, rep)
 
 	payload := data["payload"].(map[string]interface{})
-	topics, _ := json.Marshal(payload["topics"].([]interface{}))
 	validateOnly := ""
 	if payload["validateOnly"] != nil {
 		validateOnly = strconv.FormatBool(payload["validateOnly"].(bool))
 	}
 	repPayload, _ := json.Marshal([]api.TableData{
-		{
-			Name:     "Topics",
-			Value:    string(topics),
-			Selector: `request.payload.topics`,
-		},
 		{
 			Name:     "Timeout (ms)",
 			Value:    fmt.Sprintf("%d", int(payload["timeoutMs"].(float64))),
@@ -615,10 +717,20 @@ func representCreateTopicsRequest(data map[string]interface{}) []interface{} {
 	})
 	rep = append(rep, api.SectionData{
 		Type:  api.TABLE,
-		Title: "Payload",
+		Title: "Transaction Details",
 		Data:  string(repPayload),
 	})
 
+	for i, _topic := range payload["topics"].([]interface{}) {
+		topic := _topic.(map[string]interface{})
+
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: fmt.Sprintf("Topic [%d]", i),
+			Data:  representMapAsTable(topic, fmt.Sprintf(`request.payload.topics[%d]`, i), []string{}),
+		})
+	}
+
 	return rep
 }
 
@@ -628,7 +740,6 @@ func representCreateTopicsResponse(data map[string]interface{}) []interface{} {
 	rep = representResponseHeader(data, rep)
 
 	payload := data["payload"].(map[string]interface{})
-	topics, _ := json.Marshal(payload["topics"].([]interface{}))
 	throttleTimeMs := ""
 	if payload["throttleTimeMs"] != nil {
 		throttleTimeMs = fmt.Sprintf("%d", int(payload["throttleTimeMs"].(float64)))
@@ -639,18 +750,23 @@ func representCreateTopicsResponse(data map[string]interface{}) []interface{} {
 			Value:    throttleTimeMs,
 			Selector: `response.payload.throttleTimeMs`,
 		},
-		{
-			Name:     "Topics",
-			Value:    string(topics),
-			Selector: `response.payload.topics`,
-		},
 	})
 	rep = append(rep, api.SectionData{
 		Type:  api.TABLE,
-		Title: "Payload",
+		Title: "Transaction Details",
 		Data:  string(repPayload),
 	})
 
+	for i, _topic := range payload["topics"].([]interface{}) {
+		topic := _topic.(map[string]interface{})
+
+		rep = append(rep, api.SectionData{
+			Type:  api.TABLE,
+			Title: fmt.Sprintf("Topic [%d]", i),
+			Data:  representMapAsTable(topic, fmt.Sprintf(`response.payload.topics[%d]`, i), []string{}),
+		})
+	}
+
 	return rep
 }
 
@@ -727,3 +843,42 @@ func representDeleteTopicsResponse(data map[string]interface{}) []interface{} {
 
 	return rep
 }
+
+func contains(s []string, str string) bool {
+	for _, v := range s {
+		if v == str {
+			return true
+		}
+	}
+
+	return false
+}
+
+func representMapAsTable(mapData map[string]interface{}, selectorPrefix string, ignoreKeys []string) (representation string) {
+	var table []api.TableData
+	for key, value := range mapData {
+		if contains(ignoreKeys, key) {
+			continue
+		}
+		switch reflect.ValueOf(value).Kind() {
+		case reflect.Map:
+			fallthrough
+		case reflect.Slice:
+			x, err := json.Marshal(value)
+			value = string(x)
+			if err != nil {
+				continue
+			}
+		}
+		selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, key)
+		table = append(table, api.TableData{
+			Name:     strings.Join(camelcase.Split(strings.Title(key)), " "),
+			Value:    value,
+			Selector: selector,
+		})
+	}
+
+	obj, _ := json.Marshal(table)
+	representation = string(obj)
+	return
+}

tap/extensions/redis/helpers.go

@@ -42,11 +42,6 @@ func representGeneric(generic map[string]interface{}, selectorPrefix string) (re
 			Value:    generic["key"].(string),
 			Selector: fmt.Sprintf("%skey", selectorPrefix),
 		},
-		{
-			Name:     "Value",
-			Value:    generic["value"].(string),
-			Selector: fmt.Sprintf("%svalue", selectorPrefix),
-		},
 		{
 			Name:     "Keyword",
 			Value:    generic["keyword"].(string),
@@ -59,5 +54,12 @@ func representGeneric(generic map[string]interface{}, selectorPrefix string) (re
 		Data:  string(details),
 	})
 
+	representation = append(representation, api.SectionData{
+		Type:     api.BODY,
+		Title:    "Value",
+		Data:     generic["value"].(string),
+		Selector: fmt.Sprintf("%svalue", selectorPrefix),
+	})
+
 	return
 }

ui/package-lock.json

@@ -7856,6 +7856,11 @@
         "pify": "^4.0.1"
       }
     },
+    "hamt_plus": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/hamt_plus/-/hamt_plus-1.0.2.tgz",
+      "integrity": "sha1-4hwlKWjH4zsg9qGwlM2FeHomVgE="
+    },
     "handle-thing": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz",
@@ -14229,6 +14234,14 @@
         "picomatch": "^2.2.1"
       }
     },
+    "recoil": {
+      "version": "0.5.2",
+      "resolved": "https://registry.npmjs.org/recoil/-/recoil-0.5.2.tgz",
+      "integrity": "sha512-Edibzpu3dbUMLy6QRg73WL8dvMl9Xqhp+kU+f2sJtXxsaXvAlxU/GcnDE8HXPkprXrhHF2e6SZozptNvjNF5fw==",
+      "requires": {
+        "hamt_plus": "1.0.2"
+      }
+    },
     "recursive-readdir": {
       "version": "2.2.2",
       "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.2.tgz",

ui/package.json

@@ -31,6 +31,7 @@
     "react-scrollable-feed-virtualized": "^1.4.9",
     "react-syntax-highlighter": "^15.4.3",
     "react-toastify": "^8.0.3",
+    "recoil": "^0.5.2",
     "typescript": "^4.2.4",
     "web-vitals": "^1.1.1",
     "xml-formatter": "^2.6.0"

ui/public/index.html

@@ -29,6 +29,7 @@
       try {
         // Injected from server
         window.isEnt = __IS_STANDALONE__
+        window.isOasEnabled = __IS_OAS_ENABLED__
       }
       catch (e) {
       }

ui/src/EntApp.tsx

@@ -1,4 +1,4 @@
-import React, {useEffect, useState} from 'react';
+import React, {useCallback, useEffect, useState} from 'react';
 import './App.sass';
 import {TrafficPage} from "./components/TrafficPage";
 import {TLSWarning} from "./components/TLSWarning/TLSWarning";
@@ -9,43 +9,29 @@ import InstallPage from "./components/InstallPage";
 import LoginPage from "./components/LoginPage";
 import LoadingOverlay from "./components/LoadingOverlay";
 import AuthPageBase from './components/AuthPageBase';
+import entPageAtom, {Page} from "./recoil/entPage";
+import {useRecoilState} from "recoil";
 
 const api = Api.getInstance();
 
-// TODO: move to state management
-export enum Page {
-    Traffic,
-    Setup,
-    Login
-}
-
-// TODO: move to state management
-export interface MizuContextModel {
-    page: Page;
-    setPage: (page: Page) => void;
-}
-
-// TODO: move to state management
-export const MizuContext = React.createContext<MizuContextModel>(null);
-
 const EntApp = () => {
 
     const [isLoading, setIsLoading] = useState(true);
     const [showTLSWarning, setShowTLSWarning] = useState(false);
     const [userDismissedTLSWarning, setUserDismissedTLSWarning] = useState(false);
     const [addressesWithTLS, setAddressesWithTLS] = useState(new Set<string>());
-    const [page, setPage] = useState(Page.Traffic); // TODO: move to state management
+    const [entPage, setEntPage] = useRecoilState(entPageAtom);
     const [isFirstLogin, setIsFirstLogin] = useState(false);
 
-    const determinePage = async () => { // TODO: move to state management
+    const determinePage =  useCallback(async () => { // TODO: move to state management
         try {
             const isInstallNeeded = await api.isInstallNeeded();
             if (isInstallNeeded) {
-                setPage(Page.Setup);
+                setEntPage(Page.Setup);
             } else {
                 const isAuthNeeded = await api.isAuthenticationNeeded();
                 if(isAuthNeeded) {
-                    setPage(Page.Login);
+                    setEntPage(Page.Login);
                 }
             }
         } catch (e) {
@@ -54,11 +40,11 @@ const EntApp = () => {
         } finally {
             setIsLoading(false);
         }
-    }
+    },[setEntPage]);
 
     useEffect(() => {
         determinePage();
-    }, []);
+    }, [determinePage]);
 
     const onTLSDetected = (destAddress: string) => {
         addressesWithTLS.add(destAddress);
@@ -71,7 +57,7 @@ const EntApp = () => {
 
     let pageComponent: any;
 
-    switch (page) { // TODO: move to state management / proper routing
+    switch (entPage) { // TODO: move to state management / proper routing
         case Page.Traffic:
             pageComponent = <TrafficPage onTLSDetected={onTLSDetected}/>;
             break;
@@ -91,16 +77,14 @@ const EntApp = () => {
 
     return (
         <div className="mizuApp">
-            <MizuContext.Provider value={{page, setPage}}>
-                {page === Page.Traffic && <EntHeader isFirstLogin={isFirstLogin} setIsFirstLogin={setIsFirstLogin}/>}
-                {pageComponent}
-                {page === Page.Traffic && <TLSWarning showTLSWarning={showTLSWarning}
-                            setShowTLSWarning={setShowTLSWarning}
-                            addressesWithTLS={addressesWithTLS}
-                            setAddressesWithTLS={setAddressesWithTLS}
-                            userDismissedTLSWarning={userDismissedTLSWarning}
-                            setUserDismissedTLSWarning={setUserDismissedTLSWarning}/>}
-            </MizuContext.Provider>
+            {entPage === Page.Traffic && <EntHeader isFirstLogin={isFirstLogin} setIsFirstLogin={setIsFirstLogin}/>}
+            {pageComponent}
+            {entPage === Page.Traffic && <TLSWarning showTLSWarning={showTLSWarning}
+                        setShowTLSWarning={setShowTLSWarning}
+                        addressesWithTLS={addressesWithTLS}
+                        setAddressesWithTLS={setAddressesWithTLS}
+                        userDismissedTLSWarning={userDismissedTLSWarning}
+                        setUserDismissedTLSWarning={setUserDismissedTLSWarning}/>}
         </div>
     );
 }

ui/src/components/EntriesList.tsx

@@ -6,11 +6,12 @@ import {EntryItem} from "./EntryListItem/EntryListItem";
 import down from "./assets/downImg.svg";
 import spinner from './assets/spinner.svg';
 import Api from "../helpers/api";
+import {useRecoilState, useRecoilValue} from "recoil";
+import entriesAtom from "../recoil/entries";
+import wsConnectionAtom, {WsConnectionStatus} from "../recoil/wsConnection";
+import queryAtom from "../recoil/query";
 
 interface EntriesListProps {
-    entries: any[];
-    setEntries: any;
-    query: string;
     listEntryREF: any;
     onSnapBrokenEvent: () => void;
     isSnappedToBottom: boolean;
@@ -22,12 +23,8 @@ interface EntriesListProps {
     startTime: number;
     noMoreDataTop: boolean;
     setNoMoreDataTop: (flag: boolean) => void;
-    focusedEntryId: string;
-    setFocusedEntryId: (id: string) => void;
-    updateQuery: any;
     leftOffTop: number;
     setLeftOffTop: (leftOffTop: number) => void;
-    isWebSocketConnectionClosed: boolean;
     ws: any;
     openWebSocket: (query: string, resetEntries: boolean) => void;
     leftOffBottom: number;
@@ -38,7 +35,13 @@ interface EntriesListProps {
 
 const api = Api.getInstance();
 
-export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, query, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, setQueriedCurrent, queriedTotal, setQueriedTotal, startTime, noMoreDataTop, setNoMoreDataTop, focusedEntryId, setFocusedEntryId, updateQuery, leftOffTop, setLeftOffTop, isWebSocketConnectionClosed, ws, openWebSocket, leftOffBottom, truncatedTimestamp, setTruncatedTimestamp, scrollableRef}) => {
+export const EntriesList: React.FC<EntriesListProps> = ({listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, setQueriedCurrent, queriedTotal, setQueriedTotal, startTime, noMoreDataTop, setNoMoreDataTop, leftOffTop, setLeftOffTop, ws, openWebSocket, leftOffBottom, truncatedTimestamp, setTruncatedTimestamp, scrollableRef}) => {
+
+    const [entries, setEntries] = useRecoilState(entriesAtom);
+    const wsConnection = useRecoilValue(wsConnectionAtom);
+    const query = useRecoilValue(queryAtom);
+    const isWsConnectionClosed = wsConnection === WsConnectionStatus.Closed;
+
     const [loadMoreTop, setLoadMoreTop] = useState(false);
     const [isLoadingTop, setIsLoadingTop] = useState(false);
 
@@ -95,9 +98,9 @@ export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, qu
     },[setLoadMoreTop, setIsLoadingTop, entries, setEntries, query, setNoMoreDataTop, leftOffTop, setLeftOffTop, queriedCurrent, setQueriedCurrent, setQueriedTotal, setTruncatedTimestamp, scrollableRef]);
 
     useEffect(() => {
-        if(!isWebSocketConnectionClosed || !loadMoreTop || noMoreDataTop) return;
+        if(!isWsConnectionClosed || !loadMoreTop || noMoreDataTop) return;
         getOldEntries();
-    }, [loadMoreTop, noMoreDataTop, getOldEntries, isWebSocketConnectionClosed]);
+    }, [loadMoreTop, noMoreDataTop, getOldEntries, isWsConnectionClosed]);
 
     const scrollbarVisible = scrollableRef.current?.childWrapperRef.current.clientHeight > scrollableRef.current?.wrapperRef.current.clientHeight;
 
@@ -113,10 +116,7 @@ export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, qu
                         {memoizedEntries.map(entry => <EntryItem
                             key={`entry-${entry.id}`}
                             entry={entry}
-                            focusedEntryId={focusedEntryId}
-                            setFocusedEntryId={setFocusedEntryId}
                             style={{}}
-                            updateQuery={updateQuery}
                             headingMode={false}
                         />)}
                     </ScrollableFeedVirtualized>
@@ -131,9 +131,9 @@ export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, qu
                     </button>
                     <button type="button"
                         title="Snap to bottom"
-                        className={`${styles.btnLive} ${isSnappedToBottom && !isWebSocketConnectionClosed ? styles.hideButton : styles.showButton}`}
+                        className={`${styles.btnLive} ${isSnappedToBottom && !isWsConnectionClosed ? styles.hideButton : styles.showButton}`}
                         onClick={(_) => {
-                            if (isWebSocketConnectionClosed) {
+                            if (isWsConnectionClosed) {
                                 if (query) {
                                     openWebSocket(`(${query}) and leftOff(${leftOffBottom})`, false);
                                 } else {
@@ -148,7 +148,7 @@ export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, qu
                 </div>
 
                 <div className={styles.footer}>
-                    <div>Displaying <b>{entries?.length}</b> results out of <b>{queriedTotal}</b> total</div>
+                    <div>Displaying <b id="entries-length">{entries?.length}</b> results out of <b id="total-entries">{queriedTotal}</b> total</div>
                     {startTime !== 0 && <div>Started listening at <span style={{marginRight: 5, fontWeight: 600, fontSize: 13}}>{Moment(truncatedTimestamp ? truncatedTimestamp : startTime).utc().format('MM/DD/YYYY, h:mm:ss.SSS A')}</span></div>}
                 </div>
             </div>

ui/src/components/EntryDetailed.tsx

@@ -1,9 +1,13 @@
-import React from "react";
+import React, {useEffect, useState} from "react";
 import EntryViewer from "./EntryDetailed/EntryViewer";
 import {EntryItem} from "./EntryListItem/EntryListItem";
 import {makeStyles} from "@material-ui/core";
 import Protocol from "./UI/Protocol"
 import Queryable from "./UI/Queryable";
+import {toast} from "react-toastify";
+import {useRecoilValue} from "recoil";
+import focusedEntryIdAtom from "../recoil/focusedEntryId";
+import Api from "../helpers/api";
 
 const useStyles = makeStyles(() => ({
     entryTitle: {
@@ -27,23 +31,17 @@ const useStyles = makeStyles(() => ({
     }
 }));
 
-interface EntryDetailedProps {
-    entryData: any
-    updateQuery: any
-}
-
 export const formatSize = (n: number) => n > 1000 ? `${Math.round(n / 1000)}KB` : `${n} B`;
 
-const EntryTitle: React.FC<any> = ({protocol, data, bodySize, elapsedTime, updateQuery}) => {
+const EntryTitle: React.FC<any> = ({protocol, data, bodySize, elapsedTime}) => {
     const classes = useStyles();
     const response = data.response;
 
     return <div className={classes.entryTitle}>
-        <Protocol protocol={protocol} horizontal={true} updateQuery={updateQuery}/>
+        <Protocol protocol={protocol} horizontal={true}/>
         <div style={{right: "30px", position: "absolute", display: "flex"}}>
             {response && <Queryable
                 query={`response.bodySize == ${bodySize}`}
-                updateQuery={updateQuery}
                 style={{margin: "0 18px"}}
                 displayIconOnMouseOver={true}
             >
@@ -55,7 +53,6 @@ const EntryTitle: React.FC<any> = ({protocol, data, bodySize, elapsedTime, updat
             </Queryable>}
             {response && <Queryable
                 query={`elapsedTime >= ${elapsedTime}`}
-                updateQuery={updateQuery}
                 style={{marginRight: 18}}
                 displayIconOnMouseOver={true}
             >
@@ -69,30 +66,58 @@ const EntryTitle: React.FC<any> = ({protocol, data, bodySize, elapsedTime, updat
     </div>;
 };
 
-const EntrySummary: React.FC<any> = ({entry, updateQuery}) => {
+const EntrySummary: React.FC<any> = ({entry}) => {
     return <EntryItem
         key={`entry-${entry.id}`}
         entry={entry}
-        focusedEntryId={null}
-        setFocusedEntryId={null}
         style={{}}
-        updateQuery={updateQuery}
         headingMode={true}
     />;
 };
 
-export const EntryDetailed: React.FC<EntryDetailedProps> = ({entryData, updateQuery}) => {
+const api = Api.getInstance();
+
+export const EntryDetailed = () => {
+
+    const focusedEntryId = useRecoilValue(focusedEntryIdAtom);
+    const [entryData, setEntryData] = useState(null);
+
+    useEffect(() => {
+        if (!focusedEntryId) return;
+        setEntryData(null);
+        (async () => {
+            try {
+                const entryData = await api.getEntry(focusedEntryId);
+                setEntryData(entryData);
+            } catch (error) {
+                if (error.response?.data?.type) {
+                    toast[error.response.data.type](`Entry[${focusedEntryId}]: ${error.response.data.msg}`, {
+                        position: "bottom-right",
+                        theme: "colored",
+                        autoClose: error.response.data.autoClose,
+                        hideProgressBar: false,
+                        closeOnClick: true,
+                        pauseOnHover: true,
+                        draggable: true,
+                        progress: undefined,
+                    });
+                }
+                console.error(error);
+            }
+        })();
+        // eslint-disable-next-line
+    }, [focusedEntryId]);
+
     return <>
-        <EntryTitle
+        {entryData && <EntryTitle
             protocol={entryData.protocol}
             data={entryData.data}
             bodySize={entryData.bodySize}
             elapsedTime={entryData.data.elapsedTime}
-            updateQuery={updateQuery}
-        />
-        {entryData.data && <EntrySummary entry={entryData.data} updateQuery={updateQuery}/>}
+        />}
+        {entryData && <EntrySummary entry={entryData.data}/>}
         <>
-            {entryData.data && <EntryViewer
+            {entryData && <EntryViewer
                 representation={entryData.representation}
                 isRulesEnabled={entryData.isRulesEnabled}
                 rulesMatched={entryData.rulesMatched}
@@ -102,7 +127,6 @@ export const EntryDetailed: React.FC<EntryDetailedProps> = ({entryData, updateQu
                 contractContent={entryData.data.contractContent}
                 elapsedTime={entryData.data.elapsedTime}
                 color={entryData.protocol.backgroundColor}
-                updateQuery={updateQuery}
             />}
         </>
     </>

ui/src/components/EntryDetailed/EntrySections.tsx

@@ -12,14 +12,13 @@ import {default as xmlBeautify} from "xml-formatter";
 interface EntryViewLineProps {
     label: string;
     value: number | string;
-    updateQuery?: any;
     selector?: string;
     overrideQueryValue?: string;
     displayIconOnMouseOver?: boolean;
     useTooltip?: boolean;
 }
 
-const EntryViewLine: React.FC<EntryViewLineProps> = ({label, value, updateQuery = null, selector = "", overrideQueryValue = "", displayIconOnMouseOver = true, useTooltip = true}) => {
+const EntryViewLine: React.FC<EntryViewLineProps> = ({label, value, selector = "", overrideQueryValue = "", displayIconOnMouseOver = true, useTooltip = true}) => {
     let query: string;
     if (!selector) {
         query = "";
@@ -34,7 +33,6 @@ const EntryViewLine: React.FC<EntryViewLineProps> = ({label, value, updateQuery
                     <td className={`${styles.dataKey}`}>
                         <Queryable
                             query={query}
-                            updateQuery={updateQuery}
                             style={{float: "right", height: "18px"}}
                             iconStyle={{marginRight: "20px"}}
                             flipped={true}
@@ -63,10 +61,9 @@ interface EntrySectionCollapsibleTitleProps {
     expanded: boolean,
     setExpanded: any,
     query?: string,
-    updateQuery?: any,
 }
 
-const EntrySectionCollapsibleTitle: React.FC<EntrySectionCollapsibleTitleProps> = ({title, color, expanded, setExpanded, query = "", updateQuery = null}) => {
+const EntrySectionCollapsibleTitle: React.FC<EntrySectionCollapsibleTitleProps> = ({title, color, expanded, setExpanded, query = ""}) => {
     return <div className={styles.title}>
         <div
             className={`${styles.button} ${expanded ? styles.expanded : ''}`}
@@ -79,9 +76,8 @@ const EntrySectionCollapsibleTitle: React.FC<EntrySectionCollapsibleTitleProps>
         </div>
         <Queryable
             query={query}
-            updateQuery={updateQuery}
-            useTooltip={updateQuery ? true : false}
-            displayIconOnMouseOver={updateQuery ? true : false}
+            useTooltip={!!query}
+            displayIconOnMouseOver={!!query}
         >
             <span>{title}</span>
         </Queryable>
@@ -92,32 +88,31 @@ interface EntrySectionContainerProps {
     title: string,
     color: string,
     query?: string,
-    updateQuery?: any,
 }
 
-export const EntrySectionContainer: React.FC<EntrySectionContainerProps> = ({title, color, children, query = "", updateQuery = null}) => {
+export const EntrySectionContainer: React.FC<EntrySectionContainerProps> = ({title, color, children, query = ""}) => {
     const [expanded, setExpanded] = useState(true);
     return <CollapsibleContainer
         className={styles.collapsibleContainer}
         expanded={expanded}
-        title={<EntrySectionCollapsibleTitle title={title} color={color} expanded={expanded} setExpanded={setExpanded} query={query} updateQuery={updateQuery}/>}
+        title={<EntrySectionCollapsibleTitle title={title} color={color} expanded={expanded} setExpanded={setExpanded} query={query}/>}
     >
         {children}
     </CollapsibleContainer>
 }
 
 interface EntryBodySectionProps {
+    title: string,
     content: any,
     color: string,
-    updateQuery: any,
     encoding?: string,
     contentType?: string,
     selector?: string,
 }
 
 export const EntryBodySection: React.FC<EntryBodySectionProps> = ({
+    title,
     color,
-    updateQuery,
     content,
     encoding,
     contentType,
@@ -167,10 +162,9 @@ export const EntryBodySection: React.FC<EntryBodySectionProps> = ({
 
     return <React.Fragment>
         {content && content?.length > 0 && <EntrySectionContainer
-                                                title='Body'
+                                                title={title}
                                                 color={color}
                                                 query={`${selector} == r".*"`}
-                                                updateQuery={updateQuery}
                                             >
             <div style={{display: 'flex', alignItems: 'center', alignContent: 'center', margin: "5px 0"}}>
                 {supportsPrettying && <div style={{paddingTop: 3}}>
@@ -201,21 +195,33 @@ interface EntrySectionProps {
     title: string,
     color: string,
     arrayToIterate: any[],
-    updateQuery: any,
 }
 
-export const EntryTableSection: React.FC<EntrySectionProps> = ({title, color, arrayToIterate, updateQuery}) => {
+export const EntryTableSection: React.FC<EntrySectionProps> = ({title, color, arrayToIterate}) => {
+    let arrayToIterateSorted: any[];
+    if (arrayToIterate) {
+        arrayToIterateSorted = arrayToIterate.sort((a, b) => {
+            if (a.name > b.name) {
+                return 1;
+            }
+
+            if (a.name < b.name) {
+                return -1;
+            }
+
+            return 0;
+        });
+    }
     return <React.Fragment>
         {
             arrayToIterate && arrayToIterate.length > 0 ?
                 <EntrySectionContainer title={title} color={color}>
                     <table>
-                        <tbody>
-                            {arrayToIterate.map(({name, value, selector}, index) => <EntryViewLine
+                        <tbody id={`tbody-${title}`}>
+                            {arrayToIterateSorted.map(({name, value, selector}, index) => <EntryViewLine
                                 key={index}
                                 label={name}
                                 value={value}
-                                updateQuery={updateQuery}
                                 selector={selector}
                             />)}
                         </tbody>

ui/src/components/EntryDetailed/EntryViewer.tsx

@@ -8,7 +8,7 @@ enum SectionTypes {
     SectionBody = "body",
 }
 
-const SectionsRepresentation: React.FC<any> = ({data, color, updateQuery}) => {
+const SectionsRepresentation: React.FC<any> = ({data, color}) => {
     const sections = []
 
     if (data) {
@@ -16,12 +16,12 @@ const SectionsRepresentation: React.FC<any> = ({data, color, updateQuery}) => {
             switch (row.type) {
                 case SectionTypes.SectionTable:
                     sections.push(
-                        <EntryTableSection key={i} title={row.title} color={color} arrayToIterate={JSON.parse(row.data)} updateQuery={updateQuery}/>
+                        <EntryTableSection key={i} title={row.title} color={color} arrayToIterate={JSON.parse(row.data)}/>
                     )
                     break;
                 case SectionTypes.SectionBody:
                     sections.push(
-                        <EntryBodySection key={i} color={color} content={row.data} updateQuery={updateQuery} encoding={row.encoding} contentType={row.mimeType} selector={row.selector}/>
+                        <EntryBodySection key={i} title={row.title} color={color} content={row.data} encoding={row.encoding} contentType={row.mimeType} selector={row.selector}/>
                     )
                     break;
                 default:
@@ -33,7 +33,7 @@ const SectionsRepresentation: React.FC<any> = ({data, color, updateQuery}) => {
     return <>{sections}</>;
 }
 
-const AutoRepresentation: React.FC<any> = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color, updateQuery}) => {
+const AutoRepresentation: React.FC<any> = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color}) => {
     var TABS = [
         {
             tab: 'Request'
@@ -48,9 +48,9 @@ const AutoRepresentation: React.FC<any> = ({representation, isRulesEnabled, rule
 
     const {request, response} = JSON.parse(representation);
 
-    var responseTabIndex = 0;
-    var rulesTabIndex = 0;
-    var contractTabIndex = 0;
+    let responseTabIndex = 0;
+    let rulesTabIndex = 0;
+    let contractTabIndex = 0;
 
     if (response) {
         TABS.push(
@@ -85,10 +85,10 @@ const AutoRepresentation: React.FC<any> = ({representation, isRulesEnabled, rule
                 <Tabs tabs={TABS} currentTab={currentTab} color={color} onChange={setCurrentTab} leftAligned/>
             </div>
             {currentTab === TABS[0].tab && <React.Fragment>
-                <SectionsRepresentation data={request} color={color} updateQuery={updateQuery}/>
+                <SectionsRepresentation data={request} color={color}/>
             </React.Fragment>}
             {response && currentTab === TABS[responseTabIndex].tab && <React.Fragment>
-                <SectionsRepresentation data={response} color={color} updateQuery={updateQuery}/>
+                <SectionsRepresentation data={response} color={color}/>
             </React.Fragment>}
             {isRulesEnabled && currentTab === TABS[rulesTabIndex].tab && <React.Fragment>
                 <EntryTablePolicySection title={'Rule'} color={color} latency={elapsedTime} arrayToIterate={rulesMatched ? rulesMatched : []}/>
@@ -110,10 +110,9 @@ interface Props {
     contractContent: string;
     color: string;
     elapsedTime: number;
-    updateQuery: any;
 }
 
-const EntryViewer: React.FC<Props> = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color, updateQuery}) => {
+const EntryViewer: React.FC<Props> = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color}) => {
     return <AutoRepresentation
         representation={representation}
         isRulesEnabled={isRulesEnabled}
@@ -124,7 +123,6 @@ const EntryViewer: React.FC<Props> = ({representation, isRulesEnabled, rulesMatc
         contractContent={contractContent}
         elapsedTime={elapsedTime}
         color={color}
-        updateQuery={updateQuery}
     />
 };
 

ui/src/components/EntryListItem/EntryListItem.module.sass

@@ -74,7 +74,6 @@
 .separatorRight
   display: flex
   border-right: 1px solid $data-background-color
-  padding: 4px
   padding-right: 12px
 
 .separatorLeft

ui/src/components/EntryListItem/EntryListItem.tsx

@@ -12,6 +12,9 @@ import ingoingIconNeutral from "../assets/ingoing-traffic-neutral.svg"
 import outgoingIconSuccess from "../assets/outgoing-traffic-success.svg"
 import outgoingIconFailure from "../assets/outgoing-traffic-failure.svg"
 import outgoingIconNeutral from "../assets/outgoing-traffic-neutral.svg"
+import {useRecoilState} from "recoil";
+import focusedEntryIdAtom from "../../recoil/focusedEntryId";
+import queryAtom from "../../recoil/query";
 
 interface TCPInterface {
     ip: string
@@ -42,15 +45,14 @@ interface Rules {
 
 interface EntryProps {
     entry: Entry;
-    focusedEntryId: string;
-    setFocusedEntryId: (id: string) => void;
     style: object;
-    updateQuery: any;
     headingMode: boolean;
 }
 
-export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocusedEntryId, style, updateQuery, headingMode}) => {
+export const EntryItem: React.FC<EntryProps> = ({entry, style, headingMode}) => {
 
+    const [focusedEntryId, setFocusedEntryId] = useRecoilState(focusedEntryIdAtom);
+    const [queryState, setQuery] = useRecoilState(queryAtom);
     const isSelected = focusedEntryId === entry.id.toString();
 
     const classification = getClassification(entry.status)
@@ -103,8 +105,8 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
         }
     }
 
-    var contractEnabled = true;
-    var contractText = "";
+    let contractEnabled = true;
+    let contractText = "";
     switch (entry.contractStatus) {
         case 0:
             contractEnabled = false;
@@ -123,8 +125,9 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
             break;
     }
 
+
     const isStatusCodeEnabled = ((entry.proto.name === "http" && "status" in entry) || entry.status !== 0);
-    var endpointServiceContainer = "10px";
+    let endpointServiceContainer = "10px";
     if (!isStatusCodeEnabled) endpointServiceContainer = "20px";
 
     return <>
@@ -147,17 +150,15 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
             {!headingMode ? <Protocol
                 protocol={entry.proto}
                 horizontal={false}
-                updateQuery={updateQuery}
             /> : null}
             {isStatusCodeEnabled && <div>
-                <StatusCode statusCode={entry.status} updateQuery={updateQuery}/>
+                <StatusCode statusCode={entry.status}/>
             </div>}
             <div className={styles.endpointServiceContainer} style={{paddingLeft: endpointServiceContainer}}>
-                <Summary method={entry.method} summary={entry.summary} updateQuery={updateQuery}/>
+                <Summary method={entry.method} summary={entry.summary}/>
                 <div className={styles.resolvedName}>
                     <Queryable
                         query={`src.name == "${entry.src.name}"`}
-                        updateQuery={updateQuery}
                         displayIconOnMouseOver={true}
                         flipped={true}
                         style={{marginTop: "-4px", overflow: "visible"}}
@@ -174,7 +175,6 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
                     <SwapHorizIcon style={{color: entry.proto.backgroundColor, marginTop: "-2px"}}></SwapHorizIcon>
                     <Queryable
                         query={`dst.name == "${entry.dst.name}"`}
-                        updateQuery={updateQuery}
                         displayIconOnMouseOver={true}
                         style={{marginTop: "-4px"}}
                         iconStyle={{marginTop: "4px", marginLeft: "-2px"}}
@@ -204,7 +204,6 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
             <div className={styles.separatorRight}>
                 <Queryable
                         query={`src.ip == "${entry.src.ip}"`}
-                        updateQuery={updateQuery}
                         displayIconOnMouseOver={true}
                         flipped={true}
                         iconStyle={{marginRight: "16px"}}
@@ -219,7 +218,6 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
 				<span className={`${styles.tcpInfo}`} style={{marginTop: "18px"}}>{entry.src.port ? ":" : ""}</span>
                 <Queryable
                         query={`src.port == "${entry.src.port}"`}
-                        updateQuery={updateQuery}
                         displayIconOnMouseOver={true}
                         flipped={true}
                         iconStyle={{marginTop: "28px"}}
@@ -234,7 +232,6 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
                 {entry.isOutgoing ?
                     <Queryable
                             query={`outgoing == true`}
-                            updateQuery={updateQuery}
                             displayIconOnMouseOver={true}
                             flipped={true}
                             iconStyle={{marginTop: "28px"}}
@@ -248,7 +245,6 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
                     :
                     <Queryable
                             query={`outgoing == true`}
-                            updateQuery={updateQuery}
                             displayIconOnMouseOver={true}
                             flipped={true}
                             iconStyle={{marginTop: "28px"}}
@@ -258,14 +254,14 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
                             alt="Outgoing traffic"
                             title="Outgoing"
                             onClick={() => {
-                                updateQuery(`outgoing == false`)
+                                const query = `outgoing == false`;
+                                setQuery(queryState ? `${queryState} and ${query}` : query);
                             }}
                         />
                     </Queryable>
                 }
                 <Queryable
                         query={`dst.ip == "${entry.dst.ip}"`}
-                        updateQuery={updateQuery}
                         displayIconOnMouseOver={true}
                         flipped={false}
                         iconStyle={{marginTop: "28px"}}
@@ -280,7 +276,6 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
                 <span className={`${styles.tcpInfo}`} style={{marginTop: "18px"}}>:</span>
                 <Queryable
                         query={`dst.port == "${entry.dst.port}"`}
-                        updateQuery={updateQuery}
                         displayIconOnMouseOver={true}
                         flipped={false}
                 >
@@ -295,7 +290,6 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
             <div className={styles.timestamp}>
                 <Queryable
                         query={`timestamp >= datetime("${Moment(+entry.timestamp)?.utc().format('MM/DD/YYYY, h:mm:ss.SSS A')}")`}
-                        updateQuery={updateQuery}
                         displayIconOnMouseOver={true}
                         flipped={false}
                 >

ui/src/components/Filters.tsx

@@ -7,20 +7,20 @@ import {SyntaxHighlighter} from "./UI/SyntaxHighlighter/index";
 import filterUIExample1 from "./assets/filter-ui-example-1.png"
 import filterUIExample2 from "./assets/filter-ui-example-2.png"
 import variables from '../variables.module.scss';
+import {useRecoilState} from "recoil";
+import queryAtom from "../recoil/query";
+import useKeyPress from "../hooks/useKeyPress"
+import shortcutsKeyboard from "../configs/shortcutsKeyboard"
 
 interface FiltersProps {
-    query: string
-    setQuery: any
     backgroundColor: string
     ws: any
     openWebSocket: (query: string, resetEntries: boolean) => void;
 }
 
-export const Filters: React.FC<FiltersProps> = ({query, setQuery, backgroundColor, ws, openWebSocket}) => {
+export const Filters: React.FC<FiltersProps> = ({backgroundColor, ws, openWebSocket}) => {
     return <div className={styles.container}>
         <QueryForm
-            query={query}
-            setQuery={setQuery}
             backgroundColor={backgroundColor}
             ws={ws}
             openWebSocket={openWebSocket}
@@ -29,8 +29,6 @@ export const Filters: React.FC<FiltersProps> = ({query, setQuery, backgroundColo
 };
 
 interface QueryFormProps {
-    query: string
-    setQuery: any
     backgroundColor: string
     ws: any
     openWebSocket: (query: string, resetEntries: boolean) => void;
@@ -50,9 +48,10 @@ export const modalStyle = {
     color: '#000',
 };
 
-export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, backgroundColor, ws, openWebSocket}) => {
+export const QueryForm: React.FC<QueryFormProps> = ({backgroundColor, ws, openWebSocket}) => {
 
     const formRef = useRef<HTMLFormElement>(null);
+    const [query, setQuery] = useRecoilState(queryAtom);
 
     const [openModal, setOpenModal] = useState(false);
 
@@ -63,6 +62,8 @@ export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, background
         setQuery(e.target.value);
     }
 
+    
+
     const handleSubmit = (e) => {
         ws.close();
         if (query) {
@@ -73,6 +74,8 @@ export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, background
         e.preventDefault();
     }
 
+    useKeyPress(shortcutsKeyboard.ctrlEnter, handleSubmit, formRef.current);
+
     return <>
         <form
             ref={formRef}

ui/src/components/Header/EntHeader.tsx

@@ -1,4 +1,4 @@
-import React, {useContext, useEffect, useState} from "react";
+import React, {useEffect, useState} from "react";
 import logo from '../assets/MizuEntLogo.svg';
 import './Header.sass';
 import userImg from '../assets/user-circle.svg';
@@ -9,7 +9,8 @@ import logoutIcon from '../assets/logout.png';
 import {SettingsModal} from "../SettingsModal/SettingModal";
 import Api from "../../helpers/api";
 import {toast} from "react-toastify";
-import {MizuContext, Page} from "../../EntApp";
+import {useSetRecoilState} from "recoil";
+import entPageAtom, {Page} from "../../recoil/entPage";
 
 const api = Api.getInstance();
 
@@ -49,12 +50,12 @@ export const EntHeader: React.FC<EntHeaderProps> = ({isFirstLogin, setIsFirstLog
 
 const ProfileButton = () => {
 
-    const {setPage} = useContext(MizuContext);
+    const setEntPage = useSetRecoilState(entPageAtom);
 
     const logout = async (popupState) => {
         try {
             await api.logout();
-            setPage(Page.Login);
+            setEntPage(Page.Login);
         } catch (e) {
             toast.error("Something went wrong, please check the console");
             console.error(e);

ui/src/components/InstallPage.tsx

@@ -1,11 +1,15 @@
 import { Button } from "@material-ui/core";
-import React, { useContext, useState } from "react";
-import { MizuContext, Page } from "../EntApp";
+import React, { useState,useRef } from "react";
 import { adminUsername } from "../consts";
 import Api, { FormValidationErrorType } from "../helpers/api";
 import { toast } from 'react-toastify';
 import LoadingOverlay from "./LoadingOverlay";
 import { useCommonStyles } from "../helpers/commonStyle";
+import {useSetRecoilState} from "recoil";
+import entPageAtom, {Page} from "../recoil/entPage";
+import useKeyPress from "../hooks/useKeyPress"
+import shortcutsKeyboard from "../configs/shortcutsKeyboard"
+
 
 const api = Api.getInstance();
 
@@ -15,12 +19,13 @@ interface InstallPageProps {
 
 export const InstallPage: React.FC<InstallPageProps> = ({onFirstLogin}) => {
 
+    const formRef = useRef(null);
     const classes = useCommonStyles();
     const [isLoading, setIsLoading] = useState(false);
     const [password, setPassword] = useState("");
     const [passwordConfirm, setPasswordConfirm] = useState("");
 
-    const {setPage} = useContext(MizuContext);
+    const setEntPage = useSetRecoilState(entPageAtom);
 
     const onFormSubmit = async () => {
         if (password.length < 4) {
@@ -35,7 +40,7 @@ export const InstallPage: React.FC<InstallPageProps> = ({onFirstLogin}) => {
             setIsLoading(true);
             await api.register(adminUsername, password);
             if (!await api.isAuthenticationNeeded()) {
-                setPage(Page.Traffic);
+                setEntPage(Page.Traffic);
                 onFirstLogin();
             }
         } catch (e) {
@@ -53,7 +58,9 @@ export const InstallPage: React.FC<InstallPageProps> = ({onFirstLogin}) => {
 
     }
 
-    return <div className="centeredForm">
+    useKeyPress(shortcutsKeyboard.enter, onFormSubmit, formRef.current);
+
+    return <div className="centeredForm" ref={formRef}>
             {isLoading && <LoadingOverlay/>}
             <div className="form-title left-text">Setup</div>
             <span className="form-subtitle">Welcome to Mizu, please set up the admin user to continue</span>

ui/src/components/LoginPage.tsx

@@ -1,10 +1,14 @@
 import { Button } from "@material-ui/core";
-import React, { useContext, useState } from "react";
+import React, { useState,useRef } from "react";
 import { toast } from "react-toastify";
-import { MizuContext, Page } from "../EntApp";
 import Api from "../helpers/api";
 import { useCommonStyles } from "../helpers/commonStyle";
 import LoadingOverlay from "./LoadingOverlay";
+import entPageAtom, {Page} from "../recoil/entPage";
+import {useSetRecoilState} from "recoil";
+import useKeyPress from "../hooks/useKeyPress"
+import shortcutsKeyboard from "../configs/shortcutsKeyboard"
+
 
 const api = Api.getInstance();
 
@@ -14,8 +18,9 @@ const LoginPage: React.FC = () => {
     const [isLoading, setIsLoading] = useState(false);
     const [username, setUsername] = useState("");
     const [password, setPassword] = useState("");
+    const formRef = useRef(null);
 
-    const {setPage} = useContext(MizuContext);
+    const setEntPage = useSetRecoilState(entPageAtom);
 
     const onFormSubmit = async () => {
         setIsLoading(true);
@@ -23,7 +28,7 @@ const LoginPage: React.FC = () => {
         try {
             await api.login(username, password);
             if (!await api.isAuthenticationNeeded()) {
-                setPage(Page.Traffic);
+                setEntPage(Page.Traffic);
             } else {
                 toast.error("Invalid credentials");
             }
@@ -35,14 +40,9 @@ const LoginPage: React.FC = () => {
         }
     }
 
-    const handleFormOnKeyPress = (e: React.KeyboardEvent<HTMLInputElement>) => {
-        if (e.key === "Enter") {
-            onFormSubmit();
-        }
-    };
+    useKeyPress(shortcutsKeyboard.enter, onFormSubmit, formRef.current);
 
-
-    return <div className="centeredForm" onKeyPress={handleFormOnKeyPress}>
+    return <div className="centeredForm" ref={formRef}> 
             {isLoading && <LoadingOverlay/>}
             <div className="form-title left-text">Login</div>
             <div className="form-input">

ui/src/components/TrafficPage.tsx

@@ -12,6 +12,12 @@ import {StatusBar} from "./UI/StatusBar";
 import Api, {MizuWebsocketURL} from "../helpers/api";
 import { toast } from 'react-toastify';
 import debounce from 'lodash/debounce';
+import {useRecoilState, useRecoilValue} from "recoil";
+import tappingStatusAtom from "../recoil/tappingStatus";
+import entriesAtom from "../recoil/entries";
+import focusedEntryIdAtom from "../recoil/focusedEntryId";
+import websocketConnectionAtom, {WsConnectionStatus} from "../recoil/wsConnection";
+import queryAtom from "../recoil/query";
 
 const useLayoutStyles = makeStyles(() => ({
     details: {
@@ -33,11 +39,6 @@ const useLayoutStyles = makeStyles(() => ({
     }
 }));
 
-enum ConnectionStatus {
-    Closed,
-    Connected,
-}
-
 interface TrafficPageProps {
     onTLSDetected: (destAddress: string) => void;
     setAnalyzeStatus?: (status: any) => void;
@@ -48,21 +49,16 @@ const api = Api.getInstance();
 export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnalyzeStatus}) => {
 
     const classes = useLayoutStyles();
-
-    const [entries, setEntries] = useState([] as any);
-    const [focusedEntryId, setFocusedEntryId] = useState(null);
-    const [selectedEntryData, setSelectedEntryData] = useState(null);
-    const [connection, setConnection] = useState(ConnectionStatus.Closed);
+    const [tappingStatus, setTappingStatus] = useRecoilState(tappingStatusAtom);
+    const [entries, setEntries] = useRecoilState(entriesAtom);
+    const [focusedEntryId, setFocusedEntryId] = useRecoilState(focusedEntryIdAtom);
+    const [wsConnection, setWsConnection] = useRecoilState(websocketConnectionAtom);
+    const query = useRecoilValue(queryAtom);
 
     const [noMoreDataTop, setNoMoreDataTop] = useState(false);
-
-    const [tappingStatus, setTappingStatus] = useState(null);
-
     const [isSnappedToBottom, setIsSnappedToBottom] = useState(true);
 
-    const [query, setQuery] = useState("");
     const [queryBackgroundColor, setQueryBackgroundColor] = useState("#f5f5f5");
-    const [addition, updateQuery] = useState("");
 
     const [queriedCurrent, setQueriedCurrent] = useState(0);
     const [queriedTotal, setQueriedTotal] = useState(0);
@@ -94,15 +90,6 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
         handleQueryChange(query);
     }, [query, handleQueryChange]);
 
-    useEffect(() => {
-        if (query) {
-            setQuery(`${query} and ${addition}`);
-        } else {
-            setQuery(addition);
-        }
-        // eslint-disable-next-line
-    }, [addition]);
-
     const ws = useRef(null);
 
     const listEntry = useRef(null);
@@ -117,11 +104,11 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
         }
         ws.current = new WebSocket(MizuWebsocketURL);
         ws.current.onopen = () => {
-            setConnection(ConnectionStatus.Connected);
+            setWsConnection(WsConnectionStatus.Connected);
             ws.current.send(query);
         }
         ws.current.onclose = () => {
-            setConnection(ConnectionStatus.Closed);
+            setWsConnection(WsConnectionStatus.Closed);
         }
         ws.current.onerror = (event) => {
             console.error("WebSocket error:", event);
@@ -206,36 +193,9 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
         // eslint-disable-next-line
     }, []);
 
-
-    useEffect(() => {
-        if (!focusedEntryId) return;
-        setSelectedEntryData(null);
-        (async () => {
-            try {
-                const entryData = await api.getEntry(focusedEntryId, query);
-                setSelectedEntryData(entryData);
-            } catch (error) {
-                if (error.response?.data?.type) {
-                    toast[error.response.data.type](`Entry[${focusedEntryId}]: ${error.response.data.msg}`, {
-                        position: "bottom-right",
-                        theme: "colored",
-                        autoClose: error.response.data.autoClose,
-                        hideProgressBar: false,
-                        closeOnClick: true,
-                        pauseOnHover: true,
-                        draggable: true,
-                        progress: undefined,
-                    });
-                }
-                console.error(error);
-            }
-        })();
-        // eslint-disable-next-line
-    }, [focusedEntryId]);
-
     const toggleConnection = () => {
         ws.current.close();
-        if (connection !== ConnectionStatus.Connected) {
+        if (wsConnection !== WsConnectionStatus.Connected) {
             if (query) {
                 openWebSocket(`(${query}) and leftOff(-1)`, true);
             } else {
@@ -248,8 +208,8 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
 
     const getConnectionStatusClass = (isContainer) => {
         const container = isContainer ? "Container" : "";
-        switch (connection) {
-            case ConnectionStatus.Connected:
+        switch (wsConnection) {
+            case WsConnectionStatus.Connected:
                 return "greenIndicator" + container;
             default:
                 return "redIndicator" + container;
@@ -257,8 +217,8 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
     }
 
     const getConnectionTitle = () => {
-        switch (connection) {
-            case ConnectionStatus.Connected:
+        switch (wsConnection) {
+            case WsConnectionStatus.Connected:
                 return "streaming live traffic"
             default:
                 return "streaming paused";
@@ -267,7 +227,7 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
 
     const onSnapBrokenEvent = () => {
         setIsSnappedToBottom(false);
-        if (connection === ConnectionStatus.Connected) {
+        if (wsConnection === WsConnectionStatus.Connected) {
             ws.current.close();
         }
     }
@@ -275,9 +235,9 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
     return (
         <div className="TrafficPage">
             <div className="TrafficPageHeader">
-                <img className="playPauseIcon" style={{visibility: connection === ConnectionStatus.Connected ? "visible" : "hidden"}} alt="pause"
+                <img className="playPauseIcon" style={{visibility: wsConnection === WsConnectionStatus.Connected ? "visible" : "hidden"}} alt="pause"
                     src={pauseIcon} onClick={toggleConnection}/>
-                <img className="playPauseIcon" style={{position: "absolute", visibility: connection === ConnectionStatus.Connected ? "hidden" : "visible"}} alt="play"
+                <img className="playPauseIcon" style={{position: "absolute", visibility: wsConnection === WsConnectionStatus.Connected ? "hidden" : "visible"}} alt="play"
                     src={playIcon} onClick={toggleConnection}/>
                 <div className="connectionText">
                     {getConnectionTitle()}
@@ -289,17 +249,12 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
             {<div className="TrafficPage-Container">
                 <div className="TrafficPage-ListContainer">
                     <Filters
-                        query={query}
-                        setQuery={setQuery}
                         backgroundColor={queryBackgroundColor}
                         ws={ws.current}
                         openWebSocket={openWebSocket}
                     />
                     <div className={styles.container}>
                         <EntriesList
-                            entries={entries}
-                            setEntries={setEntries}
-                            query={query}
                             listEntryREF={listEntry}
                             onSnapBrokenEvent={onSnapBrokenEvent}
                             isSnappedToBottom={isSnappedToBottom}
@@ -311,12 +266,8 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
                             startTime={startTime}
                             noMoreDataTop={noMoreDataTop}
                             setNoMoreDataTop={setNoMoreDataTop}
-                            focusedEntryId={focusedEntryId}
-                            setFocusedEntryId={setFocusedEntryId}
-                            updateQuery={updateQuery}
                             leftOffTop={leftOffTop}
                             setLeftOffTop={setLeftOffTop}
-                            isWebSocketConnectionClosed={connection === ConnectionStatus.Closed}
                             ws={ws.current}
                             openWebSocket={openWebSocket}
                             leftOffBottom={leftOffBottom}
@@ -327,10 +278,10 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({onTLSDetected, setAnaly
                     </div>
                 </div>
                 <div className={classes.details}>
-                    {selectedEntryData && <EntryDetailed entryData={selectedEntryData} updateQuery={updateQuery}/>}
+                    {focusedEntryId && <EntryDetailed/>}
                 </div>
             </div>}
-            {tappingStatus && <StatusBar tappingStatus={tappingStatus}/>}
+            {tappingStatus && <StatusBar/>}
         </div>
     )
 };

ui/src/components/UI/Protocol.tsx

@@ -18,14 +18,12 @@ export interface ProtocolInterface {
 interface ProtocolProps {
     protocol: ProtocolInterface
     horizontal: boolean
-    updateQuery: any
 }
 
-const Protocol: React.FC<ProtocolProps> = ({protocol, horizontal, updateQuery}) => {
+const Protocol: React.FC<ProtocolProps> = ({protocol, horizontal}) => {
     if (horizontal) {
         return <Queryable
             query={protocol.macro}
-            updateQuery={updateQuery}
             displayIconOnMouseOver={true}
         >
             <a target="_blank" rel="noopener noreferrer" href={protocol.referenceLink}>
@@ -45,7 +43,6 @@ const Protocol: React.FC<ProtocolProps> = ({protocol, horizontal, updateQuery})
     } else {
         return <Queryable
             query={protocol.macro}
-            updateQuery={updateQuery}
             displayIconOnMouseOver={true}
             flipped={false}
             iconStyle={{marginTop: "52px", marginRight: "10px", zIndex: 1000}}

ui/src/components/UI/Queryable.tsx

@@ -2,10 +2,11 @@ import React, { useEffect, useState } from 'react';
 import { CopyToClipboard } from 'react-copy-to-clipboard';
 import AddCircleIcon from '@material-ui/icons/AddCircle';
 import './style/Queryable.sass';
+import {useRecoilState} from "recoil";
+import queryAtom from "../../recoil/query";
 
 interface Props {
     query: string,
-    updateQuery: any,
     style?: object,
     iconStyle?: object,
     className?: string,
@@ -14,9 +15,10 @@ interface Props {
     flipped?: boolean,
 }
 
-const Queryable: React.FC<Props> = ({query, updateQuery, style, iconStyle, className, useTooltip= true, displayIconOnMouseOver = false, flipped = false, children}) => {
+const Queryable: React.FC<Props> = ({query, style, iconStyle, className, useTooltip= true, displayIconOnMouseOver = false, flipped = false, children}) => {
     const [showAddedNotification, setAdded] = useState(false);
     const [showTooltip, setShowTooltip] = useState(false);
+    const [queryState, setQuery] = useRecoilState(queryAtom);
 
     const onCopy = () => {
         setAdded(true)
@@ -25,13 +27,15 @@ const Queryable: React.FC<Props> = ({query, updateQuery, style, iconStyle, class
     useEffect(() => {
         let timer;
         if (showAddedNotification) {
-            updateQuery(query);
+            setQuery(queryState ? `${queryState} and ${query}` : query);
             timer = setTimeout(() => {
                 setAdded(false);
             }, 1000);
         }
         return () => clearTimeout(timer);
-    }, [showAddedNotification, query, updateQuery]);
+
+    // eslint-disable-next-line
+    }, [showAddedNotification, query, setQuery]);
 
     const addButton = query ? <CopyToClipboard text={query} onCopy={onCopy}>
                     <span
@@ -39,7 +43,7 @@ const Queryable: React.FC<Props> = ({query, updateQuery, style, iconStyle, class
                         title={`Add "${query}" to the filter`}
                         style={iconStyle}
                     >
-                        <AddCircleIcon fontSize="small" color="inherit"></AddCircleIcon>
+                        <AddCircleIcon fontSize="small" color="inherit"/>
                         {showAddedNotification && <span className={'Queryable-AddNotifier'}>Added</span>}
                     </span>
 				</CopyToClipboard> : null;

ui/src/components/UI/StatusBar.tsx

@@ -3,33 +3,18 @@ import React, {useState} from "react";
 import warningIcon from '../assets/warning_icon.svg';
 import failIcon from '../assets/failed.svg';
 import successIcon from '../assets/success.svg';
-
-export interface TappingStatusPod {
-    name: string;
-    namespace: string;
-    isTapped: boolean;
-}
-
-export interface TappingStatus {
-    pods: TappingStatusPod[];
-}
-
-export interface Props {
-    tappingStatus: TappingStatusPod[]
-}
+import {useRecoilValue} from "recoil";
+import tappingStatusAtom, {tappingStatusDetails} from "../../recoil/tappingStatus";
 
 const pluralize = (noun: string, amount: number) => {
     return `${noun}${amount !== 1 ? 's' : ''}`
 }
 
-export const StatusBar: React.FC<Props> = ({tappingStatus}) => {
+export const StatusBar = () => {
 
+    const tappingStatus = useRecoilValue(tappingStatusAtom);
     const [expandedBar, setExpandedBar] = useState(false);
-
-    const uniqueNamespaces = Array.from(new Set(tappingStatus.map(pod => pod.namespace)));
-    const amountOfPods = tappingStatus.length;
-    const amountOfTappedPods = tappingStatus.filter(pod => pod.isTapped).length;
-    const amountOfUntappedPods = amountOfPods - amountOfTappedPods;
+    const {uniqueNamespaces, amountOfPods, amountOfTappedPods, amountOfUntappedPods} = useRecoilValue(tappingStatusDetails);
 
     return <div className={'statusBar' + (expandedBar ? ' expandedStatusBar' : "")} onMouseOver={() => setExpandedBar(true)} onMouseLeave={() => setExpandedBar(false)}>
         <div className="podsCount">

ui/src/components/UI/StatusCode.tsx

@@ -10,16 +10,14 @@ export enum StatusCodeClassification {
 
 interface EntryProps {
     statusCode: number
-    updateQuery: any
 }
 
-const StatusCode: React.FC<EntryProps> = ({statusCode, updateQuery}) => {
+const StatusCode: React.FC<EntryProps> = ({statusCode}) => {
 
     const classification = getClassification(statusCode)
 
     return <Queryable
         query={`response.status == ${statusCode}`}
-        updateQuery={updateQuery}
         displayIconOnMouseOver={true}
         flipped={true}
         iconStyle={{marginTop: "40px", paddingLeft: "10px"}}

ui/src/components/UI/Summary.tsx

@@ -6,16 +6,14 @@ import Queryable from "./Queryable";
 interface SummaryProps {
     method: string
     summary: string
-    updateQuery: any
 }
 
-export const Summary: React.FC<SummaryProps> = ({method, summary, updateQuery}) => {
+export const Summary: React.FC<SummaryProps> = ({method, summary}) => {
 
     return <div className={styles.container}>
         {method && <Queryable
             query={`method == "${method}"`}
             className={`${miscStyles.protocol} ${miscStyles.method}`}
-            updateQuery={updateQuery}
             displayIconOnMouseOver={true}
             style={{whiteSpace: "nowrap"}}
         >
@@ -25,7 +23,6 @@ export const Summary: React.FC<SummaryProps> = ({method, summary, updateQuery})
         </Queryable>}
         {summary && <Queryable
             query={`summary == "${summary}"`}
-            updateQuery={updateQuery}
             displayIconOnMouseOver={true}
         >
             <div

ui/src/configs/shortcutsKeyboard.ts

@@ -0,0 +1,7 @@
+const dictionary = {
+    ctrlEnter : [{metaKey : true, code:"Enter"}, {ctrlKey:true, code:"Enter"}], // support Ctrl/command
+    enter : [{code:"Enter"}]
+};
+
+
+export default dictionary;

ui/src/hooks/useKeyPress.ts

@@ -0,0 +1,35 @@
+import { useCallback, useEffect, useLayoutEffect, useRef } from 'react';
+
+const useKeyPress = (eventConfigs, callback, node = null) => {
+  // implement the callback ref pattern
+  const callbackRef = useRef(callback);
+  useLayoutEffect(() => {
+    callbackRef.current = callback;
+  });
+
+  // handle what happens on key press
+  const handleKeyPress = useCallback(
+    (event) => {
+      // check if one of the key is part of the ones we want
+      if (eventConfigs.some((eventConfig) => Object.keys(eventConfig).every(nameKey => eventConfig[nameKey] === event[nameKey]))) {
+        callbackRef.current(event);
+      }
+    },
+    [eventConfigs]
+  );
+
+  useEffect(() => {
+    // target is either the provided node or the document
+    const targetNode = node ?? document;
+    // attach the event listener
+    targetNode &&
+      targetNode.addEventListener("keydown", handleKeyPress);
+
+    // remove the event listener
+    return () =>
+      targetNode &&
+        targetNode.removeEventListener("keydown", handleKeyPress);
+  }, [handleKeyPress, node]);
+};
+
+export default useKeyPress;

ui/src/index.tsx

@@ -5,23 +5,26 @@ import App from './App';
 import EntApp from "./EntApp";
 import {ToastContainer} from "react-toastify";
 import 'react-toastify/dist/ReactToastify.css';
+import {RecoilRoot} from "recoil";
 
 ReactDOM.render(
   <React.StrictMode>
-    <>
-      {window["isEnt"] ? <EntApp/> : <App/>}
-      <ToastContainer
-          position="bottom-right"
-          autoClose={5000}
-          hideProgressBar={false}
-          newestOnTop={false}
-          closeOnClick
-          rtl={false}
-          pauseOnFocusLoss
-          draggable
-          pauseOnHover
-      />
-    </>
+      <RecoilRoot>
+        <>
+          {window["isEnt"] ? <EntApp/> : <App/>}
+          <ToastContainer
+              position="bottom-right"
+              autoClose={5000}
+              hideProgressBar={false}
+              newestOnTop={false}
+              closeOnClick
+              rtl={false}
+              pauseOnFocusLoss
+              draggable
+              pauseOnHover
+          />
+        </>
+      </RecoilRoot>
   </React.StrictMode>,
   document.getElementById('root')
 );

ui/src/recoil/entPage/atom.ts

@@ -0,0 +1,8 @@
+import { atom } from "recoil"
+
+const entPageAtom = atom({
+    key: "entPageAtom",
+    default: 0
+})
+
+export default entPageAtom

ui/src/recoil/entPage/index.ts

@@ -0,0 +1,11 @@
+import atom from "./atom";
+
+enum Page {
+    Traffic,
+    Setup,
+    Login
+}
+
+export { Page };
+
+export default atom;

ui/src/recoil/entries/atom.ts

@@ -0,0 +1,8 @@
+import { atom } from "recoil";
+
+const entriesAtom = atom({
+    key: "entriesAtom",
+    default: []
+});
+
+export default entriesAtom;

ui/src/recoil/entries/index.ts

@@ -0,0 +1,3 @@
+import atom from "./atom";
+
+export default atom

ui/src/recoil/focusedEntryId/atom.ts

@@ -0,0 +1,8 @@
+import { atom } from "recoil";
+
+const focusedEntryIdAtom = atom({
+    key: "focusedEntryIdAtom",
+    default: null
+});
+
+export default focusedEntryIdAtom;

ui/src/recoil/focusedEntryId/index.ts

@@ -0,0 +1,3 @@
+import atom from "./atom";
+
+export default atom

ui/src/recoil/query/atom.ts

@@ -0,0 +1,8 @@
+import { atom } from "recoil";
+
+const queryAtom = atom({
+    key: "queryAtom",
+    default: ""
+});
+
+export default queryAtom;

ui/src/recoil/query/index.ts

@@ -0,0 +1,3 @@
+import atom from "./atom";
+
+export default atom;

ui/src/recoil/tappingStatus/atom.ts

@@ -0,0 +1,9 @@
+import { atom } from "recoil";
+import {TappingStatusPod} from "./index";
+
+const tappingStatusAtom = atom({
+    key: "tappingStatusAtom",
+    default: null as TappingStatusPod[]
+});
+
+export default tappingStatusAtom;

ui/src/recoil/tappingStatus/details.ts

@@ -0,0 +1,22 @@
+import {selector} from "recoil";
+import tappingStatusAtom from "./atom";
+
+const tappingStatusDetails = selector({
+    key: 'tappingStatusDetails',
+    get: ({get}) => {
+        const tappingStatus = get(tappingStatusAtom);
+        const uniqueNamespaces = Array.from(new Set(tappingStatus.map(pod => pod.namespace)));
+        const amountOfPods = tappingStatus.length;
+        const amountOfTappedPods = tappingStatus.filter(pod => pod.isTapped).length;
+        const amountOfUntappedPods = amountOfPods - amountOfTappedPods;
+
+        return {
+            uniqueNamespaces,
+            amountOfPods,
+            amountOfTappedPods,
+            amountOfUntappedPods,
+        };
+    },
+});
+
+export default tappingStatusDetails;

ui/src/recoil/tappingStatus/index.ts

@@ -0,0 +1,17 @@
+import atom from "./atom";
+import tappingStatusDetails from './details';
+
+interface TappingStatusPod {
+    name: string;
+    namespace: string;
+    isTapped: boolean;
+}
+
+interface TappingStatus {
+    pods: TappingStatusPod[];
+}
+
+export type {TappingStatus, TappingStatusPod};
+export {tappingStatusDetails};
+
+export default atom;

ui/src/recoil/wsConnection/atom.ts

@@ -0,0 +1,8 @@
+import { atom } from "recoil";
+
+const wsConnectionAtom = atom({
+    key: "wsConnectionAtom",
+    default: 0
+});
+
+export default wsConnectionAtom;

ui/src/recoil/wsConnection/index.ts

@@ -0,0 +1,10 @@
+import atom from "./atom";
+
+enum WsConnectionStatus {
+    Closed,
+    Connected,
+}
+
+export {WsConnectionStatus};
+
+export default atom