Add docker buildx acceptance tests image build (#1039)

Co-authored-by: Leon <>

.github/workflows/acceptance_tests.yml

@@ -8,6 +8,9 @@ on:
     branches:
       - 'develop'
 
+env:
+  MIZU_CI_IMAGE: mizu/ci:0.0
+
 jobs:
   run-acceptance-tests:
     name: Run acceptance tests
@@ -21,6 +24,19 @@ jobs:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          load: true
+          tags: ${{ env.MIZU_CI_IMAGE }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Setup acceptance test
         run: ./acceptanceTests/setup.sh
 

.github/workflows/acceptance_tests_on_pr.yml

@@ -0,0 +1,54 @@
+name: Acceptance tests on PR
+
+on: push
+
+env:
+  MIZU_CI_IMAGE: mizu/ci:0.0
+
+concurrency:
+  group: acceptance-tests-on-pr-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-tests:
+    name: Run tests
+    runs-on: ubuntu-latest
+    if: ${{ contains(github.event.head_commit.message, '#run_acceptance_tests') }}
+
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          load: true
+          tags: ${{ env.MIZU_CI_IMAGE }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Setup acceptance test
+        run: ./acceptanceTests/setup.sh
+
+      - name: Create k8s users and change context
+        env:
+          USERNAME_UNRESTRICTED: user-with-clusterwide-access
+          USERNAME_RESTRICTED: user-with-restricted-access
+        run: |
+          ./acceptanceTests/create_user.sh "${USERNAME_UNRESTRICTED}"
+          ./acceptanceTests/create_user.sh "${USERNAME_RESTRICTED}"
+          kubectl apply -f cli/cmd/permissionFiles/permissions-all-namespaces-tap.yaml
+          kubectl config use-context ${USERNAME_UNRESTRICTED}
+
+      - name: Test
+        run: make acceptance-test

.github/workflows/inactive-issues-close.yaml

@@ -13,10 +13,11 @@ jobs:
       - uses: actions/stale@v3
         with:
           days-before-issue-stale: 30
-          days-before-issue-close: 14
+          days-before-issue-close: -1
+          exempt-issue-labels: "enhancement"
           stale-issue-label: "stale"
           stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
-          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+          close-issue-message: ""
           days-before-pr-stale: -1
           days-before-pr-close: -1
           repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -56,7 +56,7 @@ jobs:
       
       - name: Check extensions modified files
         id: ext_modified_files
-        run: devops/check_modified_files.sh tap/extensions/
+        run: devops/check_modified_files.sh tap/extensions/ tap/api/
 
       - name: Extensions Test
         if: github.event_name == 'push' || steps.ext_modified_files.outputs.matched == 'true'
@@ -64,4 +64,3 @@ jobs:
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
-

Dockerfile

@@ -87,8 +87,8 @@ RUN go build -ldflags="-extldflags=-static -s -w \
     -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum
-ADD https://github.com/up9inc/basenine/releases/download/v0.7.1/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
-ADD https://github.com/up9inc/basenine/releases/download/v0.7.1/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
+ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
 
 RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
     chmod +x ./basenine_linux_"${GOARCH}" && \

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -39,13 +39,13 @@ export function verifyMinimumEntries() {
     });
 }
 
-export function leftTextCheck(entryNum, path, expectedText) {
-    cy.get(`#list #entry-${entryNum} ${path}`).invoke('text').should('eq', expectedText);
+export function leftTextCheck(entryId, path, expectedText) {
+    cy.get(`#list #entry-${entryId} ${path}`).invoke('text').should('eq', expectedText);
 }
 
-export function leftOnHoverCheck(entryNum, path, filterName) {
-    cy.get(`#list #entry-${entryNum} ${path}`).trigger('mouseover');
-    cy.get(`#list #entry-${entryNum} [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(filterName));
+export function leftOnHoverCheck(entryId, path, filterName) {
+    cy.get(`#list #entry-${entryId} ${path}`).trigger('mouseover');
+    cy.get(`#list #entry-${entryId} [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(filterName));
 }
 
 export function rightTextCheck(path, expectedText) {
@@ -89,13 +89,13 @@ export function checkFilterByMethod(funcDict) {
                     cy.get('#entries-length').invoke('text').then(len => {
                         resizeIfNeeded(len);
                         listElmWithIdAttr.forEach(entry => {
-                            if (entry?.id && entry.id.match(RegExp(/entry-(\d{2}|\d{1})$/gm))) {
-                                const entryNum = getEntryNumById(entry.id);
+                            if (entry?.id && entry.id.match(RegExp(/entry-(\d{24})$/gm))) {
+                                const entryId = getEntryId(entry.id);
 
-                                leftTextCheck(entryNum, methodDict.pathLeft, methodDict.expectedText);
-                                leftTextCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedTextLeft);
+                                leftTextCheck(entryId, methodDict.pathLeft, methodDict.expectedText);
+                                leftTextCheck(entryId, protocolDict.pathLeft, protocolDict.expectedTextLeft);
                                 if (summaryDict)
-                                    leftTextCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedText);
+                                    leftTextCheck(entryId, summaryDict.pathLeft, summaryDict.expectedText);
 
                                 if (!doneCheckOnFirst) {
                                     deepCheck(funcDict, protocolDict, methodDict, entry);
@@ -111,6 +111,11 @@ export function checkFilterByMethod(funcDict) {
     });
 }
 
+export function getEntryId(id) {
+    // take the second part from the string (entry-<ID>)
+    return id.split('-')[1];
+}
+
 function resizeIfNeeded(entriesLen) {
     if (entriesLen > maxEntriesInDom){
         Cypress.config().viewportHeight === Cypress.env('normalMizuHeight') ?
@@ -119,14 +124,14 @@ function resizeIfNeeded(entriesLen) {
 }
 
 function deepCheck(generalDict, protocolDict, methodDict, entry) {
-    const entryNum = getEntryNumById(entry.id);
+    const entryId = getEntryId(entry.id);
     const {summary, value} = generalDict;
     const summaryDict = getSummaryDict(summary);
 
-    leftOnHoverCheck(entryNum, methodDict.pathLeft, methodDict.expectedOnHover);
-    leftOnHoverCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedOnHover);
+    leftOnHoverCheck(entryId, methodDict.pathLeft, methodDict.expectedOnHover);
+    leftOnHoverCheck(entryId, protocolDict.pathLeft, protocolDict.expectedOnHover);
     if (summaryDict)
-        leftOnHoverCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedOnHover);
+        leftOnHoverCheck(entryId, summaryDict.pathLeft, summaryDict.expectedOnHover);
 
     cy.get(`#${entry.id}`).click();
 
@@ -183,7 +188,3 @@ function getProtocolDict(protocol, protocolText) {
         expectedOnHover: protocol.toLowerCase()
     };
 }
-
-function getEntryNumById (id) {
-    return parseInt(id.split('-')[1]);
-}

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -15,15 +15,11 @@ function checkEntries() {
         checkThatAllEntriesShown();
         resizeToHugeMizu();
 
-        cy.get('#total-entries').then(number => {
-            const numOfEntries = parseInt(number.text());
-            [...Array(numOfEntries).keys()].map(checkEntry);
+        cy.get('#list [id^=entry]').each(entryElement => {
+            entryElement.click();
+            cy.get('#tbody-Headers').should('be.visible');
+            isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
         });
     });
 }
 
-function checkEntry(entryIndex) {
-    cy.get(`#entry-${entryIndex}`).click();
-    cy.get('#tbody-Headers').should('be.visible');
-    isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
-}

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -1,5 +1,6 @@
 import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
 import {
+    getEntryId,
     leftOnHoverCheck,
     leftTextCheck,
     resizeToHugeMizu,
@@ -148,9 +149,6 @@ function checkFilterNoResults(filterName) {
             // the DOM should show 0 entries
             cy.get('#entries-length').should('have.text', '0');
 
-            // going through every potential entry and verifies that it doesn't exist
-            [...Array(parseInt(totalEntries)).keys()].map(shouldNotExist);
-
             cy.get('[title="Fetch old records"]').click();
             cy.get('#noMoreDataTop', {timeout: refreshWaitTimeout}).should('be.visible');
             cy.get('#entries-length').should('have.text', '0'); // after loading all entries there should still be 0 entries
@@ -162,10 +160,6 @@ function checkFilterNoResults(filterName) {
     });
 }
 
-function shouldNotExist(entryNum) {
-    cy.get(`entry-${entryNum}`).should('not.exist');
-}
-
 function checkIllegalFilter(illegalFilterName) {
     it(`should show red search bar with the input: ${illegalFilterName}`, function () {
         cy.reload();
@@ -185,30 +179,44 @@ function checkIllegalFilter(illegalFilterName) {
         });
     });
 }
-function checkFilter(filterDetails){
-    const {name, leftSidePath, rightSidePath, rightSideExpectedText, leftSideExpectedText, applyByEnter} = filterDetails;
+
+function checkFilter(filterDetails) {
+    const {
+        name,
+        leftSidePath,
+        rightSidePath,
+        rightSideExpectedText,
+        leftSideExpectedText,
+        applyByEnter
+    } = filterDetails;
+
     const entriesForDeeperCheck = 5;
 
     it(`checking the filter: ${name}`, function () {
         cy.get('#total-entries').should('not.have.text', '0').then(number => {
             const totalEntries = number.text();
 
-            // checks the hover on the last entry (the only one in DOM at the beginning)
-            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
+            cy.get(`#list [id^=entry]`).last().then(elem => {
+                const element = elem[0];
+                const entryId = getEntryId(element.id);
+                // checks the hover on the last entry (the only one in DOM at the beginning)
+                leftOnHoverCheck(entryId, leftSidePath, name);
 
-            cy.get('.w-tc-editor-text').clear();
-            // applying the filter with alt+enter or with the button
-            cy.get('.w-tc-editor-text').type(`${name}${applyByEnter ? '{alt+enter}' : ''}`);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
-            if (!applyByEnter)
-                cy.get('[type="submit"]').click();
+                cy.get('.w-tc-editor-text').clear();
+                // applying the filter with alt+enter or with the button
+                cy.get('.w-tc-editor-text').type(`${name}${applyByEnter ? '{alt+enter}' : ''}`);
+                cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
+                if (!applyByEnter)
+                    cy.get('[type="submit"]').click();
 
-            // only one entry in DOM after filtering, checking all checks on it
-            leftTextCheck(totalEntries - 1, leftSidePath, leftSideExpectedText);
-            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
-            rightTextCheck(rightSidePath, rightSideExpectedText);
-            rightOnHoverCheck(rightSidePath, name);
-            checkRightSideResponseBody();
+                // only one entry in DOM after filtering, checking all checks on it
+                leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
+                leftOnHoverCheck(entryId, leftSidePath, name);
+
+                rightTextCheck(rightSidePath, rightSideExpectedText);
+                rightOnHoverCheck(rightSidePath, name);
+                checkRightSideResponseBody();
+            });
 
             cy.get('[title="Fetch old records"]').click();
             resizeToHugeMizu();
@@ -217,28 +225,33 @@ function checkFilter(filterDetails){
             cy.get('#entries-length', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
 
             // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
-            [...Array(parseInt(totalEntries)).keys()].forEach(entryNum => {
-                leftTextCheck(entryNum, leftSidePath, leftSideExpectedText);
+            cy.get(`#list [id^=entry]`).each(elem => {
+                const element = elem[0];
+                let entryId = getEntryId(element.id);
+                leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
             });
 
             // making the other 3 checks on the first X entries (longer time for each check)
-            deeperChcek(leftSidePath, rightSidePath, name, leftSideExpectedText, rightSideExpectedText, entriesForDeeperCheck);
+            deeperCheck(leftSidePath, rightSidePath, name, leftSideExpectedText, rightSideExpectedText, entriesForDeeperCheck);
 
             // reloading then waiting for the entries number to load
             resizeToNormalMizu();
             cy.reload();
             cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-        });
+        })
     });
 }
 
-function deeperChcek(leftSidePath, rightSidePath, filterName, leftSideExpectedText, rightSideExpectedText, entriesNumToCheck) {
-    [...Array(entriesNumToCheck).keys()].forEach(entryNum => {
-        leftOnHoverCheck(entryNum, leftSidePath, filterName);
+function deeperCheck(leftSidePath, rightSidePath, filterName, leftSideExpectedText, rightSideExpectedText, entriesNumToCheck) {
+    cy.get(`#list [id^=entry]`).each((element, index) => {
+        if (index < entriesNumToCheck) {
+            const entryId = getEntryId(element[0].id);
+            leftOnHoverCheck(entryId, leftSidePath, filterName);
 
-        cy.get(`#list #entry-${entryNum}`).click();
-        rightTextCheck(rightSidePath, rightSideExpectedText);
-        rightOnHoverCheck(rightSidePath, filterName);
+            cy.get(`#list #entry-${entryId}`).click();
+            rightTextCheck(rightSidePath, rightSideExpectedText);
+            rightOnHoverCheck(rightSidePath, filterName);
+        }
     });
 }
 

acceptanceTests/setup.sh

@@ -57,11 +57,16 @@ kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests
 echo "Starting proxy"
 kubectl proxy --port=8080 &
 
-echo "Setting minikube docker env"
-eval $(minikube docker-env)
+if [[ -z "${CI}" ]]; then
+  echo "Setting env var of mizu ci image"
+  export MIZU_CI_IMAGE="mizu/ci:0.0"
+  echo "Build agent image"
+  docker build -t "${MIZU_CI_IMAGE}" .
+else
+  echo "not building docker image in CI because it is created as separate step"
+fi
 
-echo "Build agent image"
-docker build -t mizu/ci:0.0 .
+minikube image load "${MIZU_CI_IMAGE}"
 
 echo "Build cli"
 cd cli && make build GIT_BRANCH=ci SUFFIX=ci

acceptanceTests/testsUtils.go

@@ -212,9 +212,10 @@ func DeleteKubeFile(kubeContext string, namespace string, filename string) error
 }
 
 func getDefaultCommandArgs() []string {
+	agentImageValue := os.Getenv("MIZU_CI_IMAGE")
 	setFlag := "--set"
 	telemetry := "telemetry=false"
-	agentImage := "agent-image=mizu/ci:0.0"
+	agentImage := fmt.Sprintf("agent-image=%s", agentImageValue)
 	imagePullPolicy := "image-pull-policy=IfNotPresent"
 	headless := "headless=true"
 

agent/go.mod

@@ -19,9 +19,9 @@ require (
 	github.com/nav-inc/datetime v0.1.3
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v1.0.0
-	github.com/patrickmn/go-cache v2.1.0+incompatible
+    github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/stretchr/testify v1.7.0
-	github.com/up9inc/basenine/client/go v0.0.0-20220413023528-c741e4aa1cf2
+	github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
@@ -49,7 +49,6 @@ require (
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beevik/etree v1.1.0 // indirect
-	github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 // indirect
 	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
 	github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect
 	github.com/cilium/ebpf v0.8.0 // indirect

agent/go.sum

@@ -108,8 +108,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M=
-github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -683,8 +681,8 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
 github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
-github.com/up9inc/basenine/client/go v0.0.0-20220413023528-c741e4aa1cf2 h1:2Ol+X82EOLac/GGP/oB0ACVP/g2vIJrAuzCjZcn+RRI=
-github.com/up9inc/basenine/client/go v0.0.0-20220413023528-c741e4aa1cf2/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607 h1:UqxUSkOYOmsLZWQtMSk02ttnhdRwBRLOLt2aDiS9tEk=
+github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=

agent/main.go

@@ -320,6 +320,7 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time.
 	for i := 1; i < retryAmount; i++ {
 		socketConnection, _, err := dialer.Dial(socketAddress, nil)
 		if err != nil {
+			lastErr = err
 			if i < retryAmount {
 				logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay/time.Second)
 				time.Sleep(retryDelay)

agent/pkg/api/main.go

@@ -103,12 +103,18 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		panic("Channel of captured messages is nil")
 	}
 
+BasenineReconnect:
 	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
 	if err != nil {
-		logger.Log.Panicf("Can't establish a new connection to Basenine server: %v", err)
+		logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
+		time.Sleep(shared.BasenineReconnectInterval * time.Second)
+		goto BasenineReconnect
 	}
 	if err = connection.InsertMode(); err != nil {
-		logger.Log.Panicf("Insert mode call failed: %v", err)
+		logger.Log.Errorf("Insert mode call failed: %v", err)
+		connection.Close()
+		time.Sleep(shared.BasenineReconnectInterval * time.Second)
+		goto BasenineReconnect
 	}
 
 	disableOASValidation := false
@@ -122,19 +128,24 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 	for item := range outputItems {
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation, namespace := resolveIP(item.ConnectionInfo)
+		
+		if namespace == "" && item.Namespace != tapApi.UNKNOWN_NAMESPACE {
+			namespace = item.Namespace
+		}
+		
 		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation, namespace)
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
 				var httpPair tapApi.HTTPRequestResponsePair
 				if err := json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair); err != nil {
 					logger.Log.Error(err)
+				} else {
+					contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
+					mizuEntry.ContractStatus = contract.Status
+					mizuEntry.ContractRequestReason = contract.RequestReason
+					mizuEntry.ContractResponseReason = contract.ResponseReason
+					mizuEntry.ContractContent = contract.Content
 				}
-
-				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
-				mizuEntry.ContractStatus = contract.Status
-				mizuEntry.ContractRequestReason = contract.RequestReason
-				mizuEntry.ContractResponseReason = contract.ResponseReason
-				mizuEntry.ContractContent = contract.Content
 			}
 
 			harEntry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
@@ -146,13 +157,17 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 
 		data, err := json.Marshal(mizuEntry)
 		if err != nil {
-			panic(err)
+			logger.Log.Errorf("Error while marshaling entry: %v", err)
+			continue
 		}
 
 		providers.EntryAdded(len(data))
 
 		if err = connection.SendText(string(data)); err != nil {
-			logger.Log.Panicf("An error occured while inserting a new record to database: %v", err)
+			logger.Log.Errorf("An error occured while inserting a new record to database: %v", err)
+			connection.Close()
+			time.Sleep(shared.BasenineReconnectInterval * time.Second)
+			goto BasenineReconnect
 		}
 
 		serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMapSink)

agent/pkg/api/socket_data_streamer.go

@@ -24,7 +24,7 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 
 	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
 	if err != nil {
-		logger.Log.Errorf("failed to establish a connection to Basenine: %v", err)
+		logger.Log.Errorf("Failed to establish a connection to Basenine: %v", err)
 		entryStreamerSocketConnector.CleanupSocket(socketId)
 		return err
 	}
@@ -80,7 +80,9 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 	go handleMetaChannel(connection, meta)
 
 	if err = connection.Query(query, data, meta); err != nil {
-		logger.Log.Panicf("Query mode call failed: %v", err)
+		logger.Log.Errorf("Query mode call failed: %v", err)
+		entryStreamerSocketConnector.CleanupSocket(socketId)
+		return err
 	}
 
 	go func() {

agent/pkg/api/socket_routes.go

@@ -123,22 +123,23 @@ func SendToSocket(socketId int, message []byte) error {
 		return fmt.Errorf("socket %v is disconnected", socketId)
 	}
 
-	var sent = false
-	time.AfterFunc(time.Second*5, func() {
-		if !sent {
-			logger.Log.Error("socket timed out")
-			socketCleanup(socketId, socketObj)
-		}
-	})
-
 	socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
-	err := socketObj.connection.WriteMessage(1, message)
-	socketObj.lock.Unlock()
-	sent = true
+	defer socketObj.lock.Unlock()
 
-	if err != nil {
-		return fmt.Errorf("failed to write message to socket %v, err: %w", socketId, err)
+	if connectedWebsockets[socketId] == nil {
+		return fmt.Errorf("socket %v is disconnected", socketId)
 	}
+
+	if err := socketObj.connection.SetWriteDeadline(time.Now().Add(time.Second * 10)); err != nil {
+		socketCleanup(socketId, socketObj)
+		return fmt.Errorf("error setting timeout to socket %v, err: %v", socketId, err)
+	}
+
+	if err := socketObj.connection.WriteMessage(websocket.TextMessage, message); err != nil {
+		socketCleanup(socketId, socketObj)
+		return fmt.Errorf("failed to write message to socket %v, err: %v", socketId, err)
+	}
+
 	return nil
 }
 

agent/pkg/controllers/status_controller.go

@@ -2,6 +2,7 @@ package controllers
 
 import (
 	"net/http"
+
 	core "k8s.io/api/core/v1"
 
 	"github.com/gin-gonic/gin"
@@ -93,10 +94,6 @@ func GetGeneralStats(c *gin.Context) {
 	c.JSON(http.StatusOK, providers.GetGeneralStats())
 }
 
-func GetRecentTLSLinks(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
-}
-
 func GetCurrentResolvingInformation(c *gin.Context) {
 	c.JSON(http.StatusOK, holder.GetResolver().GetMap())
 }

agent/pkg/entries/entries_provider.go

@@ -2,6 +2,7 @@ package entries
 
 import (
 	"encoding/json"
+	"errors"
 	"time"
 
 	basenine "github.com/up9inc/basenine/client/go"
@@ -64,7 +65,7 @@ func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntr
 	}
 	err = json.Unmarshal(bytes, &entry)
 	if err != nil {
-		return nil, err
+		return nil, errors.New(string(bytes))
 	}
 
 	extension := app.ExtensionsMap[entry.Protocol.Name]

agent/pkg/models/models.go

@@ -9,7 +9,6 @@ import (
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap"
 )
 
 type EntriesRequest struct {
@@ -44,11 +43,6 @@ type WebSocketTappedEntryMessage struct {
 	Data *tapApi.OutputChannelItem
 }
 
-type WebsocketOutboundLinkMessage struct {
-	*shared.WebSocketMessageMetadata
-	Data *tap.OutboundLink
-}
-
 type AuthStatus struct {
 	Email string `json:"email"`
 	Model string `json:"model"`

agent/pkg/oas/oas_generator.go

@@ -142,17 +142,18 @@ func (g *defaultOasGenerator) runGenerator() {
 
 func (g *defaultOasGenerator) handleEntry(mizuEntry *api.Entry) {
 	if mizuEntry.Protocol.Name == "http" {
+		dest := mizuEntry.Destination.Name
+		if dest == "" {
+			logger.Log.Debugf("OAS: Unresolved entry %d", mizuEntry.Id)
+			return
+		}
+
 		entry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
 		if err != nil {
 			logger.Log.Warningf("Failed to turn MizuEntry %d into HAR Entry: %s", mizuEntry.Id, err)
 			return
 		}
 
-		dest := mizuEntry.Destination.Name
-		if dest == "" {
-			dest = mizuEntry.Destination.IP + ":" + mizuEntry.Destination.Port
-		}
-
 		entryWSource := &EntryWithSource{
 			Entry:       *entry,
 			Source:      mizuEntry.Source.Name,

agent/pkg/providers/status_provider.go

@@ -4,19 +4,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"time"
 
-	"github.com/patrickmn/go-cache"
 	"github.com/up9inc/mizu/agent/pkg/models"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap"
 )
 
-const tlsLinkRetainmentTime = time.Minute * 15
-
 var (
-	authStatus     *models.AuthStatus
-	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
+	authStatus *models.AuthStatus
 )
 
 func GetAuthStatus() (*models.AuthStatus, error) {
@@ -51,16 +45,3 @@ func GetAuthStatus() (*models.AuthStatus, error) {
 
 	return authStatus, nil
 }
-
-func GetAllRecentTLSAddresses() []string {
-	recentTLSLinks := make([]string, 0)
-
-	for _, outboundLinkItem := range RecentTLSLinks.Items() {
-		outboundLink, castOk := outboundLinkItem.Object.(*tap.OutboundLink)
-		if castOk {
-			recentTLSLinks = append(recentTLSLinks, outboundLink.DstIP)
-		}
-	}
-
-	return recentTLSLinks
-}

agent/pkg/routes/status_routes.go

@@ -21,7 +21,5 @@ func StatusRoutes(ginApp *gin.Engine) {
 
 	routeGroup.GET("/general", controllers.GetGeneralStats) // get general stats about entries in DB
 
-	routeGroup.GET("/recentTLSLinks", controllers.GetRecentTLSLinks)
-
 	routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
 }

agent/pkg/up9/main.go

@@ -211,11 +211,15 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 
 	logger.Log.Infof("Getting entries from the database")
 
+BasenineReconnect:
 	var connection *basenine.Connection
 	var err error
 	connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
 	if err != nil {
-		panic(err)
+		logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
+		connection.Close()
+		time.Sleep(shared.BasenineReconnectInterval * time.Second)
+		goto BasenineReconnect
 	}
 
 	data := make(chan []byte)
@@ -324,7 +328,10 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 	wg.Add(2)
 
 	if err = connection.Query(query, data, meta); err != nil {
-		logger.Log.Panicf("Query mode call failed: %v", err)
+		logger.Log.Errorf("Query mode call failed: %v", err)
+		connection.Close()
+		time.Sleep(shared.BasenineReconnectInterval * time.Second)
+		goto BasenineReconnect
 	}
 
 	wg.Wait()

cli/Makefile

@@ -36,16 +36,16 @@ build-base: ## Build mizu CLI binary (select platform via GOOS / GOARCH env vari
 	(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
 
 build-all: ## Build for all supported platforms.
-	@echo "Compiling for every OS and Platform"
-	@mkdir -p bin && sed s/_VER_/$(VER)/g README.md.TEMPLATE >  bin/README.md
-	@$(MAKE) build GOOS=linux GOARCH=amd64
-	@$(MAKE) build GOOS=linux GOARCH=arm64
-	@$(MAKE) build GOOS=darwin GOARCH=amd64
-	@$(MAKE) build GOOS=darwin GOARCH=arm64
-	@$(MAKE) build GOOS=windows GOARCH=amd64
-	@mv ./bin/mizu_windows_amd64 ./bin/mizu.exe
-	@echo "---------"
-	@find ./bin -ls
+	echo "Compiling for every OS and Platform" && \
+	mkdir -p bin && sed s/_VER_/$(VER)/g README.md.TEMPLATE >  bin/README.md && \
+	$(MAKE) build GOOS=linux GOARCH=amd64 && \
+	$(MAKE) build GOOS=linux GOARCH=arm64 && \
+	$(MAKE) build GOOS=darwin GOARCH=amd64 && \
+	$(MAKE) build GOOS=darwin GOARCH=arm64 && \
+	$(MAKE) build GOOS=windows GOARCH=amd64 && \
+	mv ./bin/mizu_windows_amd64 ./bin/mizu.exe && \
+	echo "---------" && \
+	find ./bin -ls
 
 clean: ## Clean all build artifacts.
 	go clean

cli/config/configStructs/tapConfig.go

@@ -11,7 +11,6 @@ import (
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 
-	basenine "github.com/up9inc/basenine/server/lib"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/units"
 )
@@ -79,10 +78,6 @@ func (config *TapConfig) GetInsertionFilter() string {
 			}
 		}
 	}
-	_, err := basenine.Parse(insertionFilter)
-	if err != nil {
-		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Insertion filter syntax error: %v", err))
-	}
 	return insertionFilter
 }
 

cli/go.mod

@@ -11,7 +11,6 @@ require (
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.3.0
 	github.com/spf13/pflag v1.0.5
-	github.com/up9inc/basenine/server/lib v0.0.0-20220413023528-c741e4aa1cf2
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
@@ -33,11 +32,8 @@ require (
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
-	github.com/alecthomas/participle/v2 v2.0.0-alpha7 // indirect
 	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
-	github.com/clbanning/mxj/v2 v2.5.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dlclark/regexp2 v1.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
@@ -72,7 +68,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
-	github.com/ohler55/ojg v1.14.0 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

cli/go.sum

@@ -83,10 +83,6 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/alecthomas/participle/v2 v2.0.0-alpha7 h1:cK4vjj0VSgb3lN1nuKA5F7dw+1s1pWBe5bx7nNCnN+c=
-github.com/alecthomas/participle/v2 v2.0.0-alpha7/go.mod h1:NumScqsC42o9x+dGj8/YqsIfhrIQjFEOFovxotbBirA=
-github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E=
-github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -120,8 +116,6 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
-github.com/clbanning/mxj/v2 v2.5.5 h1:oT81vUeEiQQ/DcHbzSytRngP6Ky9O+L+0Bw0zSJag9E=
-github.com/clbanning/mxj/v2 v2.5.5/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -155,8 +149,6 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS
 github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
-github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -487,8 +479,6 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/ohler55/ojg v1.14.0 h1:DyHomsCwofNswmKj7BLMdx51xnKbXxgIo1rVWCaBcNk=
-github.com/ohler55/ojg v1.14.0/go.mod h1:3+GH+0PggMKocQtbZCrFifal3yRpHiBT4QUkxFJI6e8=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -600,8 +590,6 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/up9inc/basenine/server/lib v0.0.0-20220413023528-c741e4aa1cf2 h1:rgm5a2ALbYKbItaSXx25K8vavZpeF0HN1Pk0qmOqy50=
-github.com/up9inc/basenine/server/lib v0.0.0-20220413023528-c741e4aa1cf2/go.mod h1:v0hIh31iwDGbkkdeSSppdMNm1oIigfCA2mG2XajKnf8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=

shared/consts.go

@@ -1,19 +1,20 @@
 package shared
 
 const (
-	MizuFilteringOptionsEnvVar       = "SENSITIVE_DATA_FILTERING_OPTIONS"
-	SyncEntriesConfigEnvVar          = "SYNC_ENTRIES_CONFIG"
-	HostModeEnvVar                   = "HOST_MODE"
-	NodeNameEnvVar                   = "NODE_NAME"
-	ConfigDirPath                    = "/app/config/"
-	DataDirPath                      = "/app/data/"
-	ValidationRulesFileName          = "validation-rules.yaml"
-	ContractFileName                 = "contract-oas.yaml"
-	ConfigFileName                   = "mizu-config.json"
-	GoGCEnvVar                       = "GOGC"
-	DefaultApiServerPort             = 8899
-	LogLevelEnvVar                   = "LOG_LEVEL"
-	MizuAgentImageRepo               = "docker.io/up9inc/mizu"
-	BasenineHost                     = "127.0.0.1"
-	BaseninePort                     = "9099"
+	MizuFilteringOptionsEnvVar = "SENSITIVE_DATA_FILTERING_OPTIONS"
+	SyncEntriesConfigEnvVar    = "SYNC_ENTRIES_CONFIG"
+	HostModeEnvVar             = "HOST_MODE"
+	NodeNameEnvVar             = "NODE_NAME"
+	ConfigDirPath              = "/app/config/"
+	DataDirPath                = "/app/data/"
+	ValidationRulesFileName    = "validation-rules.yaml"
+	ContractFileName           = "contract-oas.yaml"
+	ConfigFileName             = "mizu-config.json"
+	GoGCEnvVar                 = "GOGC"
+	DefaultApiServerPort       = 8899
+	LogLevelEnvVar             = "LOG_LEVEL"
+	MizuAgentImageRepo         = "docker.io/up9inc/mizu"
+	BasenineHost               = "127.0.0.1"
+	BaseninePort               = "9099"
+	BasenineReconnectInterval  = 3
 )

shared/kubernetes/mizuTapperSyncer.go

@@ -329,7 +329,7 @@ func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error {
 			TapperDaemonSetName,
 			tapperSyncer.config.AgentImage,
 			TapperPodName,
-			fmt.Sprintf("%s.%s.svc.cluster.local", ApiServerPodName, tapperSyncer.config.MizuResourcesNamespace),
+			fmt.Sprintf("%s.%s.svc", ApiServerPodName, tapperSyncer.config.MizuResourcesNamespace),
 			nodeNames,
 			serviceAccountName,
 			tapperSyncer.config.TapperResources,

shared/kubernetes/provider.go

@@ -805,14 +805,20 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	agentResources := applyconfcore.ResourceRequirements().WithRequests(agentResourceRequests).WithLimits(agentResourceLimits)
 	agentContainer.WithResources(agentResources)
 
-	nodeSelectorRequirement := applyconfcore.NodeSelectorRequirement()
-	nodeSelectorRequirement.WithKey("kubernetes.io/hostname")
-	nodeSelectorRequirement.WithOperator(core.NodeSelectorOpIn)
-	nodeSelectorRequirement.WithValues(nodeNames...)
-	nodeSelectorTerm := applyconfcore.NodeSelectorTerm()
-	nodeSelectorTerm.WithMatchExpressions(nodeSelectorRequirement)
+	matchFields := make([]*applyconfcore.NodeSelectorTermApplyConfiguration, 0)
+	for _, nodeName := range nodeNames {
+		nodeSelectorRequirement := applyconfcore.NodeSelectorRequirement()
+		nodeSelectorRequirement.WithKey("metadata.name")
+		nodeSelectorRequirement.WithOperator(core.NodeSelectorOpIn)
+		nodeSelectorRequirement.WithValues(nodeName)
+
+		nodeSelectorTerm := applyconfcore.NodeSelectorTerm()
+		nodeSelectorTerm.WithMatchFields(nodeSelectorRequirement)
+		matchFields = append(matchFields, nodeSelectorTerm)
+	}
+
 	nodeSelector := applyconfcore.NodeSelector()
-	nodeSelector.WithNodeSelectorTerms(nodeSelectorTerm)
+	nodeSelector.WithNodeSelectorTerms(matchFields...)
 	nodeAffinity := applyconfcore.NodeAffinity()
 	nodeAffinity.WithRequiredDuringSchedulingIgnoredDuringExecution(nodeSelector)
 	affinity := applyconfcore.Affinity()

shared/kubernetes/utils.go

@@ -26,7 +26,8 @@ func GetNodeHostToTappedPodsMap(tappedPods []core.Pod) shared.NodeToPodsMap {
 func getMinimizedPod(fullPod core.Pod) core.Pod {
 	return core.Pod{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: fullPod.Name,
+			Name:      fullPod.Name,
+			Namespace: fullPod.Namespace,
 		},
 		Status: core.PodStatus{
 			PodIP:             fullPod.Status.PodIP,

shared/models.go

@@ -20,7 +20,6 @@ const (
 	WebSocketMessageTypeUpdateStatus     WebSocketMessageType = "status"
 	WebSocketMessageTypeUpdateTappedPods WebSocketMessageType = "tappedPods"
 	WebSocketMessageTypeAnalyzeStatus    WebSocketMessageType = "analyzeStatus"
-	WebsocketMessageTypeOutboundLink     WebSocketMessageType = "outboundLink"
 	WebSocketMessageTypeToast            WebSocketMessageType = "toast"
 	WebSocketMessageTypeQueryMetadata    WebSocketMessageType = "queryMetadata"
 	WebSocketMessageTypeStartTime        WebSocketMessageType = "startTime"
@@ -92,7 +91,7 @@ func (np NodeToPodsMap) Summary() map[string][]string {
 	summary := make(map[string][]string)
 	for node, pods := range np {
 		for _, pod := range pods {
-			summary[node] = append(summary[node], pod.Namespace + "/" +  pod.Name)
+			summary[node] = append(summary[node], pod.Namespace+"/"+pod.Name)
 		}
 	}
 

tap/api/api.go

@@ -18,6 +18,7 @@ import (
 )
 
 const mizuTestEnvVar = "MIZU_TEST"
+const UNKNOWN_NAMESPACE = ""
 
 var UnknownIp net.IP = net.IP{0, 0, 0, 0}
 var UnknownPort uint16 = 0
@@ -92,14 +93,15 @@ type RequestResponsePair struct {
 	Response GenericMessage `json:"response"`
 }
 
-// `Protocol` is modified in the later stages of data propagation. Therefore it's not a pointer.
 type OutputChannelItem struct {
+	// `Protocol` is modified in later stages of data propagation. Therefore, it's not a pointer.
 	Protocol       Protocol
 	Capture        Capture
 	Timestamp      int64
 	ConnectionInfo *ConnectionInfo
 	Pair           *RequestResponsePair
 	Summary        *BaseEntry
+	Namespace      string
 }
 
 type SuperTimer struct {

tap/extensions/amqp/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect7/amqp/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/amqp/\* expect

tap/extensions/http/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect7/http/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/http/\* expect

tap/extensions/kafka/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect7/kafka/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/kafka/\* expect

tap/extensions/redis/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect7/redis/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/redis/\* expect

tap/go.mod

@@ -3,7 +3,6 @@ module github.com/up9inc/mizu/tap
 go 1.17
 
 require (
-	github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4
 	github.com/cilium/ebpf v0.8.0
 	github.com/go-errors/errors v1.4.2
 	github.com/google/gopacket v1.1.19

tap/go.sum

@@ -91,8 +91,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M=
-github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=

tap/net_utils.go

@@ -29,21 +29,6 @@ func getLocalhostIPs() ([]string, error) {
 	return myIPs, nil
 }
 
-//lint:ignore U1000 will be used in the future
-func isPrivateIP(ipStr string) bool {
-	ip := net.ParseIP(ipStr)
-	if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
-		return true
-	}
-
-	for _, block := range privateIPBlocks {
-		if block.Contains(ip) {
-			return true
-		}
-	}
-	return false
-}
-
 func initPrivateIPBlocks() {
 	for _, cidr := range []string{
 		"127.0.0.0/8",    // IPv4 loopback

tap/outboundlinks.go

@@ -1,39 +0,0 @@
-package tap
-
-type OutboundLinkProtocol string
-
-const (
-	TLSProtocol OutboundLinkProtocol = "tls"
-)
-
-type OutboundLink struct {
-	Src                   string
-	DstIP                 string
-	DstPort               int
-	SuggestedResolvedName string
-	SuggestedProtocol     OutboundLinkProtocol
-}
-
-func NewOutboundLinkWriter() *OutboundLinkWriter {
-	return &OutboundLinkWriter{
-		OutChan: make(chan *OutboundLink),
-	}
-}
-
-type OutboundLinkWriter struct {
-	OutChan chan *OutboundLink
-}
-
-func (olw *OutboundLinkWriter) WriteOutboundLink(src string, DstIP string, DstPort int, SuggestedResolvedName string, SuggestedProtocol OutboundLinkProtocol) {
-	olw.OutChan <- &OutboundLink{
-		Src:                   src,
-		DstIP:                 DstIP,
-		DstPort:               DstPort,
-		SuggestedResolvedName: SuggestedResolvedName,
-		SuggestedProtocol:     SuggestedProtocol,
-	}
-}
-
-func (olw *OutboundLinkWriter) Stop() {
-	close(olw.OutChan)
-}

tap/passive_tapper.go

@@ -27,9 +27,6 @@ import (
 
 const cleanPeriod = time.Second * 10
 
-//lint:ignore U1000 will be used in the future
-var remoteOnlyOutboundPorts = []int{80, 443}
-
 var maxcount = flag.Int64("c", -1, "Only grab this many packets, then exit")
 var decoder = flag.String("decoder", "", "Name of the decoder to use (default: guess from capture)")
 var statsevery = flag.Int("stats", 60, "Output statistics every N seconds")
@@ -58,7 +55,7 @@ var tls = flag.Bool("tls", false, "Enable TLS tapper")
 var memprofile = flag.String("memprofile", "", "Write memory profile")
 
 type TapOpts struct {
-	HostMode          bool
+	HostMode bool
 }
 
 var extensions []*api.Extension                     // global
@@ -68,24 +65,6 @@ var packetSourceManager *source.PacketSourceManager // global
 var mainPacketInputChan chan source.TcpPacketInfo   // global
 var tlsTapperInstance *tlstapper.TlsTapper          // global
 
-func inArrayInt(arr []int, valueToCheck int) bool {
-	for _, value := range arr {
-		if value == valueToCheck {
-			return true
-		}
-	}
-	return false
-}
-
-func inArrayString(arr []string, valueToCheck string) bool {
-	for _, value := range arr {
-		if value == valueToCheck {
-			return true
-		}
-	}
-	return false
-}
-
 func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem, extensionsRef []*api.Extension, options *api.TrafficFilteringOptions) {
 	extensions = extensionsRef
 	filteringOptions = options

tap/settings.go

@@ -13,6 +13,7 @@ const (
 	MaxBufferedPagesTotalEnvVarName           = "MAX_BUFFERED_PAGES_TOTAL"
 	MaxBufferedPagesPerConnectionEnvVarName   = "MAX_BUFFERED_PAGES_PER_CONNECTION"
 	TcpStreamChannelTimeoutMsEnvVarName       = "TCP_STREAM_CHANNEL_TIMEOUT_MS"
+	CloseTimedoutTcpChannelsIntervalMsEnvVar  = "CLOSE_TIMEDOUT_TCP_STREAM_CHANNELS_INTERVAL_MS"
 	MaxBufferedPagesTotalDefaultValue         = 5000
 	MaxBufferedPagesPerConnectionDefaultValue = 5000
 	TcpStreamChannelTimeoutMsDefaultValue     = 10000

tap/tcp_reader.go

@@ -7,13 +7,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/bradleyfalzon/tlsx"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap/api"
 )
 
-const checkTLSPacketAmount = 100
-
 type tcpReaderDataMsg struct {
 	bytes     []byte
 	timestamp time.Time
@@ -33,22 +30,21 @@ type ConnectionInfo struct {
  * Implements io.Reader interface (Read)
  */
 type tcpReader struct {
-	ident              string
-	tcpID              *api.TcpID
-	isClosed           bool
-	isClient           bool
-	isOutgoing         bool
-	msgQueue           chan tcpReaderDataMsg // Channel of captured reassembled tcp payload
-	data               []byte
-	progress           *api.ReadProgress
-	superTimer         *api.SuperTimer
-	parent             *tcpStream
-	packetsSeen        uint
-	outboundLinkWriter *OutboundLinkWriter
-	extension          *api.Extension
-	emitter            api.Emitter
-	counterPair        *api.CounterPair
-	reqResMatcher      api.RequestResponseMatcher
+	ident         string
+	tcpID         *api.TcpID
+	isClosed      bool
+	isClient      bool
+	isOutgoing    bool
+	msgQueue      chan tcpReaderDataMsg // Channel of captured reassembled tcp payload
+	data          []byte
+	progress      *api.ReadProgress
+	superTimer    *api.SuperTimer
+	parent        *tcpStream
+	packetsSeen   uint
+	extension     *api.Extension
+	emitter       api.Emitter
+	counterPair   *api.CounterPair
+	reqResMatcher api.RequestResponseMatcher
 	sync.Mutex
 }
 
@@ -64,16 +60,6 @@ func (h *tcpReader) Read(p []byte) (int, error) {
 		if len(h.data) > 0 {
 			h.packetsSeen += 1
 		}
-		if h.packetsSeen < checkTLSPacketAmount && len(msg.bytes) > 5 { // packets with less than 5 bytes cause tlsx to panic
-			clientHello := tlsx.ClientHello{}
-			err := clientHello.Unmarshall(msg.bytes)
-			if err == nil {
-				logger.Log.Debugf("Detected TLS client hello with SNI %s", clientHello.SNI)
-				// TODO: Throws `panic: runtime error: invalid memory address or nil pointer dereference` error.
-				// numericPort, _ := strconv.Atoi(h.tcpID.DstPort)
-				// h.outboundLinkWriter.WriteOutboundLink(h.tcpID.SrcIP, h.tcpID.DstIP, numericPort, clientHello.SNI, TLSProtocol)
-			}
-		}
 	}
 	if !ok || len(h.data) == 0 {
 		return 0, io.EOF

tap/tcp_stream_factory.go

@@ -20,12 +20,11 @@ import (
  * Generates a new tcp stream for each new tcp connection. Closes the stream when the connection closes.
  */
 type tcpStreamFactory struct {
-	wg                 sync.WaitGroup
-	outboundLinkWriter *OutboundLinkWriter
-	Emitter            api.Emitter
-	streamsMap         *tcpStreamMap
-	ownIps             []string
-	opts               *TapOpts
+	wg         sync.WaitGroup
+	Emitter    api.Emitter
+	streamsMap *tcpStreamMap
+	ownIps     []string
+	opts       *TapOpts
 }
 
 type tcpStreamWrapper struct {
@@ -63,9 +62,6 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
 	srcPort := transport.Src().String()
 	dstPort := transport.Dst().String()
 
-	// if factory.shouldNotifyOnOutboundLink(dstIp, dstPort) {
-	// 	factory.outboundLinkWriter.WriteOutboundLink(net.Src().String(), dstIp, dstPort, "", "")
-	// }
 	props := factory.getStreamProps(srcIp, srcPort, dstIp, dstPort)
 	isTapTarget := props.isTapTarget
 	stream := &tcpStream{
@@ -99,14 +95,13 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
 					SrcPort: srcPort,
 					DstPort: dstPort,
 				},
-				parent:             stream,
-				isClient:           true,
-				isOutgoing:         props.isOutgoing,
-				outboundLinkWriter: factory.outboundLinkWriter,
-				extension:          extension,
-				emitter:            factory.Emitter,
-				counterPair:        counterPair,
-				reqResMatcher:      reqResMatcher,
+				parent:        stream,
+				isClient:      true,
+				isOutgoing:    props.isOutgoing,
+				extension:     extension,
+				emitter:       factory.Emitter,
+				counterPair:   counterPair,
+				reqResMatcher: reqResMatcher,
 			})
 			stream.servers = append(stream.servers, tcpReader{
 				msgQueue:   make(chan tcpReaderDataMsg),
@@ -119,14 +114,13 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
 					SrcPort: transport.Dst().String(),
 					DstPort: transport.Src().String(),
 				},
-				parent:             stream,
-				isClient:           false,
-				isOutgoing:         props.isOutgoing,
-				outboundLinkWriter: factory.outboundLinkWriter,
-				extension:          extension,
-				emitter:            factory.Emitter,
-				counterPair:        counterPair,
-				reqResMatcher:      reqResMatcher,
+				parent:        stream,
+				isClient:      false,
+				isOutgoing:    props.isOutgoing,
+				extension:     extension,
+				emitter:       factory.Emitter,
+				counterPair:   counterPair,
+				reqResMatcher: reqResMatcher,
 			})
 
 			factory.streamsMap.Store(stream.id, &tcpStreamWrapper{
@@ -174,15 +168,6 @@ func (factory *tcpStreamFactory) getStreamProps(srcIP string, srcPort string, ds
 	}
 }
 
-//lint:ignore U1000 will be used in the future
-func (factory *tcpStreamFactory) shouldNotifyOnOutboundLink(dstIP string, dstPort int) bool {
-	if inArrayInt(remoteOnlyOutboundPorts, dstPort) {
-		isDirectedHere := inArrayString(factory.ownIps, dstIP)
-		return !isDirectedHere && !isPrivateIP(dstIP)
-	}
-	return true
-}
-
 func getPacketOrigin(ac reassembly.AssemblerContext) api.Capture {
 	c, ok := ac.(*context)
 

tap/tcp_streams_map.go

@@ -1,8 +1,10 @@
 package tap
 
 import (
+	"os"
 	"runtime"
 	_debug "runtime/debug"
+	"strconv"
 	"sync"
 	"time"
 
@@ -34,10 +36,35 @@ func (streamMap *tcpStreamMap) nextId() int64 {
 	return streamMap.streamId
 }
 
+func (streamMap *tcpStreamMap) getCloseTimedoutTcpChannelsInterval() time.Duration {
+	defaultDuration := 1000 * time.Millisecond
+	rangeMin := 10
+	rangeMax := 10000
+	closeTimedoutTcpChannelsIntervalMsStr := os.Getenv(CloseTimedoutTcpChannelsIntervalMsEnvVar)
+	if closeTimedoutTcpChannelsIntervalMsStr == "" {
+		return defaultDuration
+	} else {
+		closeTimedoutTcpChannelsIntervalMs, err := strconv.Atoi(closeTimedoutTcpChannelsIntervalMsStr)
+		if err != nil {
+			logger.Log.Warningf("Error parsing environment variable %s: %v\n", CloseTimedoutTcpChannelsIntervalMsEnvVar, err)
+			return defaultDuration
+		} else {
+			if closeTimedoutTcpChannelsIntervalMs < rangeMin || closeTimedoutTcpChannelsIntervalMs > rangeMax {
+				logger.Log.Warningf("The value of environment variable %s is not in acceptable range: %d - %d\n", CloseTimedoutTcpChannelsIntervalMsEnvVar, rangeMin, rangeMax)
+				return defaultDuration
+			} else {
+				return time.Duration(closeTimedoutTcpChannelsIntervalMs) * time.Millisecond
+			}
+		}
+	}
+}
+
 func (streamMap *tcpStreamMap) closeTimedoutTcpStreamChannels() {
 	tcpStreamChannelTimeout := GetTcpChannelTimeoutMs()
+	closeTimedoutTcpChannelsIntervalMs := streamMap.getCloseTimedoutTcpChannelsInterval()
+	logger.Log.Infof("Using %d ms as the close timedout TCP stream channels interval", closeTimedoutTcpChannelsIntervalMs/time.Millisecond)
 	for {
-		time.Sleep(10 * time.Millisecond)
+		time.Sleep(closeTimedoutTcpChannelsIntervalMs)
 		_debug.FreeOSMemory()
 		streamMap.streams.Range(func(key interface{}, value interface{}) bool {
 			streamWrapper := value.(*tcpStreamWrapper)
@@ -47,7 +74,7 @@ func (streamMap *tcpStreamMap) closeTimedoutTcpStreamChannels() {
 					stream.Close()
 					diagnose.AppStats.IncDroppedTcpStreams()
 					logger.Log.Debugf("Dropped an unidentified TCP stream because of timeout. Total dropped: %d Total Goroutines: %d Timeout (ms): %d",
-						diagnose.AppStats.DroppedTcpStreams, runtime.NumGoroutine(), tcpStreamChannelTimeout/1000000)
+						diagnose.AppStats.DroppedTcpStreams, runtime.NumGoroutine(), tcpStreamChannelTimeout/time.Millisecond)
 				}
 			} else {
 				if !stream.superIdentifier.IsClosedOthers {

tap/tlstapper/tls_emitter.go

@@ -0,0 +1,13 @@
+package tlstapper
+
+import "github.com/up9inc/mizu/tap/api"
+
+type tlsEmitter struct {
+	delegate  api.Emitter
+	namespace string
+}
+
+func (e *tlsEmitter) Emit(item *api.OutputChannelItem) {
+	item.Namespace = e.namespace
+	e.delegate.Emit(item)
+}

tap/tlstapper/tls_poller.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"fmt"
 	"net"
+	"sync"
 
 	"encoding/binary"
 	"encoding/hex"
@@ -19,13 +20,14 @@ import (
 )
 
 type tlsPoller struct {
-	tls           *TlsTapper
-	readers       map[string]*tlsReader
-	closedReaders chan string
-	reqResMatcher api.RequestResponseMatcher
-	chunksReader  *perf.Reader
-	extension     *api.Extension
-	procfs        string
+	tls            *TlsTapper
+	readers        map[string]*tlsReader
+	closedReaders  chan string
+	reqResMatcher  api.RequestResponseMatcher
+	chunksReader   *perf.Reader
+	extension      *api.Extension
+	procfs         string
+	pidToNamespace sync.Map
 }
 
 func newTlsPoller(tls *TlsTapper, extension *api.Extension, procfs string) *tlsPoller {
@@ -151,16 +153,21 @@ func (p *tlsPoller) startNewTlsReader(chunk *tlsChunk, ip net.IP, port uint16, k
 
 	tcpid := p.buildTcpId(chunk, ip, port)
 
-	go dissect(extension, reader, chunk.isRequest(), &tcpid, emitter, options, p.reqResMatcher)
+	tlsEmitter := &tlsEmitter{
+		delegate:  emitter,
+		namespace: p.getNamespace(chunk.Pid),
+	}
+
+	go dissect(extension, reader, chunk.isRequest(), &tcpid, tlsEmitter, options, p.reqResMatcher)
 	return reader
 }
 
 func dissect(extension *api.Extension, reader *tlsReader, isRequest bool, tcpid *api.TcpID,
-	emitter api.Emitter, options *api.TrafficFilteringOptions, reqResMatcher api.RequestResponseMatcher) {
+	tlsEmitter *tlsEmitter, options *api.TrafficFilteringOptions, reqResMatcher api.RequestResponseMatcher) {
 	b := bufio.NewReader(reader)
 
 	err := extension.Dissector.Dissect(b, reader.progress, api.Ebpf, isRequest, tcpid, &api.CounterPair{},
-		&api.SuperTimer{}, &api.SuperIdentifier{}, emitter, options, reqResMatcher)
+		&api.SuperTimer{}, &api.SuperIdentifier{}, tlsEmitter, options, reqResMatcher)
 
 	if err != nil {
 		logger.Log.Warningf("Error dissecting TLS %v - %v", tcpid, err)
@@ -205,6 +212,33 @@ func (p *tlsPoller) buildTcpId(chunk *tlsChunk, ip net.IP, port uint16) api.TcpI
 	}
 }
 
+func (p *tlsPoller) addPid(pid uint32, namespace string) {
+	p.pidToNamespace.Store(pid, namespace)
+}
+
+func (p *tlsPoller) getNamespace(pid uint32) string {
+	namespaceIfc, ok := p.pidToNamespace.Load(pid)
+
+	if !ok {
+		return api.UNKNOWN_NAMESPACE
+	}
+
+	namespace, ok := namespaceIfc.(string)
+
+	if !ok {
+		return api.UNKNOWN_NAMESPACE
+	}
+
+	return namespace
+}
+
+func (p *tlsPoller) clearPids() {
+	p.pidToNamespace.Range(func(key, v interface{}) bool {
+		p.pidToNamespace.Delete(key)
+		return true
+	})
+}
+
 func (p *tlsPoller) logTls(chunk *tlsChunk, ip net.IP, port uint16) {
 	var flagsStr string
 

tap/tlstapper/tls_process_discoverer.go

@@ -24,11 +24,11 @@ func UpdateTapTargets(tls *TlsTapper, pods *[]v1.Pod, procfs string) error {
 	if err != nil {
 		return err
 	}
-	
+
 	tls.ClearPids()
 
-	for _, pid := range containerPids {
-		if err := tls.AddPid(procfs, pid); err != nil {
+	for pid, pod := range containerPids {
+		if err := tls.AddPid(procfs, pid, pod.Namespace); err != nil {
 			LogError(err)
 		}
 	}
@@ -36,8 +36,8 @@ func UpdateTapTargets(tls *TlsTapper, pods *[]v1.Pod, procfs string) error {
 	return nil
 }
 
-func findContainerPids(procfs string, containerIds map[string]bool) ([]uint32, error) {
-	result := make([]uint32, 0)
+func findContainerPids(procfs string, containerIds map[string]v1.Pod) (map[uint32]v1.Pod, error) {
+	result := make(map[uint32]v1.Pod)
 
 	pids, err := ioutil.ReadDir(procfs)
 
@@ -63,7 +63,9 @@ func findContainerPids(procfs string, containerIds map[string]bool) ([]uint32, e
 			continue
 		}
 
-		if _, ok := containerIds[cgroup]; !ok {
+		pod, ok := containerIds[cgroup]
+
+		if !ok {
 			continue
 		}
 
@@ -73,14 +75,14 @@ func findContainerPids(procfs string, containerIds map[string]bool) ([]uint32, e
 			continue
 		}
 
-		result = append(result, uint32(pidNumber))
+		result[uint32(pidNumber)] = pod
 	}
 
 	return result, nil
 }
 
-func buildContainerIdsMap(pods *[]v1.Pod) map[string]bool {
-	result := make(map[string]bool)
+func buildContainerIdsMap(pods *[]v1.Pod) map[string]v1.Pod {
+	result := make(map[string]v1.Pod)
 
 	for _, pod := range *pods {
 		for _, container := range pod.Status.ContainerStatuses {
@@ -91,7 +93,7 @@ func buildContainerIdsMap(pods *[]v1.Pod) map[string]bool {
 				continue
 			}
 
-			result[url.Host] = true
+			result[url.Host] = pod
 		}
 	}
 
@@ -141,14 +143,14 @@ func extractCgroup(lines []string) string {
 //  /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod3beae8e0_164d_4689_a087_efd902d8c2ab.slice/docker-<ID>.scope
 //  /kubepods/besteffort/pod7709c1d5-447c-428f-bed9-8ddec35c93f4/<ID>
 //
-// This function extract the <ID> out of the cgroup path, the <ID> should match 
+// This function extract the <ID> out of the cgroup path, the <ID> should match
 //	the "Container ID:" field when running kubectl describe pod <POD>
 //
 func normalizeCgroup(cgrouppath string) string {
 	basename := strings.TrimSpace(path.Base(cgrouppath))
-	
+
 	if strings.Contains(basename, "-") {
-		basename = basename[strings.Index(basename, "-") + 1:]
+		basename = basename[strings.Index(basename, "-")+1:]
 	}
 
 	if strings.Contains(basename, ".") {

tap/tlstapper/tls_tapper.go

@@ -1,11 +1,12 @@
 package tlstapper
 
 import (
+	"sync"
+
 	"github.com/cilium/ebpf/rlimit"
 	"github.com/go-errors/errors"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap/api"
-	"sync"
 )
 
 const GLOABL_TAP_PID = 0
@@ -58,10 +59,10 @@ func (t *TlsTapper) PollForLogging() {
 }
 
 func (t *TlsTapper) GlobalTap(sslLibrary string) error {
-	return t.tapPid(GLOABL_TAP_PID, sslLibrary)
+	return t.tapPid(GLOABL_TAP_PID, sslLibrary, api.UNKNOWN_NAMESPACE)
 }
 
-func (t *TlsTapper) AddPid(procfs string, pid uint32) error {
+func (t *TlsTapper) AddPid(procfs string, pid uint32, namespace string) error {
 	sslLibrary, err := findSsllib(procfs, pid)
 
 	if err != nil {
@@ -69,7 +70,7 @@ func (t *TlsTapper) AddPid(procfs string, pid uint32) error {
 		return nil // hide the error on purpose, its OK for a process to not use libssl.so
 	}
 
-	return t.tapPid(pid, sslLibrary)
+	return t.tapPid(pid, sslLibrary, namespace)
 }
 
 func (t *TlsTapper) RemovePid(pid uint32) error {
@@ -85,12 +86,13 @@ func (t *TlsTapper) RemovePid(pid uint32) error {
 }
 
 func (t *TlsTapper) ClearPids() {
+	t.poller.clearPids()
 	t.registeredPids.Range(func(key, v interface{}) bool {
 		pid := key.(uint32)
 		if pid == GLOABL_TAP_PID {
 			return true
 		}
-		
+
 		if err := t.RemovePid(pid); err != nil {
 			LogError(err)
 		}
@@ -133,7 +135,7 @@ func setupRLimit() error {
 	return nil
 }
 
-func (t *TlsTapper) tapPid(pid uint32, sslLibrary string) error {
+func (t *TlsTapper) tapPid(pid uint32, sslLibrary string, namespace string) error {
 	logger.Log.Infof("Tapping TLS (pid: %v) (sslLibrary: %v)", pid, sslLibrary)
 
 	newSsl := sslHooks{}
@@ -144,12 +146,14 @@ func (t *TlsTapper) tapPid(pid uint32, sslLibrary string) error {
 
 	t.sslHooksStructs = append(t.sslHooksStructs, newSsl)
 
+	t.poller.addPid(pid, namespace)
+
 	pids := t.bpfObjects.tlsTapperMaps.PidsMap
 
 	if err := pids.Put(pid, uint32(1)); err != nil {
 		return errors.Wrap(err, 0)
 	}
-	
+
 	t.registeredPids.Store(pid, true)
 
 	return nil

ui-common/example/src/api.js

@@ -54,11 +54,6 @@ export default class Api {
         return response.data;
     }
 
-    getRecentTLSLinks = async () => {
-        const response = await client.get("/status/recentTLSLinks");
-        return response.data;
-    }
-
     getOasServices = async () => {
         const response = await client.get("/oas");
         return response.data;
@@ -121,4 +116,4 @@ export function getWebsocketUrl(){
     }
 
     return websocketUrl;
-}
+}

ui-common/package.json

@@ -49,7 +49,7 @@
     "moment": "^2.29.1",
     "node-fetch": "^3.1.1",
     "numeral": "^2.0.6",
-    "protobuf-decoder": "^0.1.0",
+    "protobuf-decoder": "^0.1.2",
     "react-graph-vis": "^1.0.7",
     "react-lowlight": "^3.0.0",
     "react-router-dom": "^6.2.1",

ui-common/src/components/ServiceMapModal/ServiceMapModal.tsx

@@ -78,15 +78,8 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
             const serviceMapData: ServiceMapGraph = await getServiceMapDataApi()
             setServiceMapApiData(serviceMapData)
             const newGraphData: GraphData = { nodes: [], edges: [] }
-
-            if (serviceMapData.nodes) {
-                newGraphData.nodes = serviceMapData.nodes.map(mapNodesDatatoGraph)
-            }
-
-            if (serviceMapData.edges) {
-                newGraphData.edges = serviceMapData.edges.map(mapEdgesDatatoGraph)
-            }
-
+            newGraphData.nodes = serviceMapData.nodes.map(mapNodesDatatoGraph)
+            newGraphData.edges = serviceMapData.edges.map(mapEdgesDatatoGraph)
             setGraphData(newGraphData)
         } catch (ex) {
             toast.error("An error occurred while loading Mizu Service Map, see console for mode details", { containerId: TOAST_CONTAINER_ID });
@@ -124,27 +117,32 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
         .sort((a, b) => { return a.key.localeCompare(b.key) });
 
     const getServicesForFilter = useMemo(() => {
-
         const resolved = mapToKeyValForFilter(serviceMapApiData.nodes?.filter(x => x.resolved))
         const unResolved = mapToKeyValForFilter(serviceMapApiData.nodes?.filter(x => !x.resolved))
         return [...resolved, ...unResolved]
     }, [serviceMapApiData])
 
-    const filterServiceMap = (newProtocolsFilters?: any[], newServiceFilters?: string[]) => {
-        const filterProt = newProtocolsFilters || checkedProtocols
-        const filterService = newServiceFilters || checkedServices
-        setCheckedProtocols(filterProt)
-        setCheckedServices(filterService)
+    useEffect(() => {
         const newGraphData: GraphData = {
-            nodes: serviceMapApiData.nodes?.map(mapNodesDatatoGraph).filter(node => filterService.includes(node.label)),
-            edges: serviceMapApiData.edges?.filter(edge => filterProt.includes(edge.protocol.abbr)).map(mapEdgesDatatoGraph)
+            nodes: serviceMapApiData.nodes?.map(mapNodesDatatoGraph).filter(node => checkedServices.includes(node.label)),
+            edges: serviceMapApiData.edges?.filter(edge => checkedProtocols.includes(edge.protocol.abbr)).map(mapEdgesDatatoGraph)
         }
         setGraphData(newGraphData);
+    }, [checkedServices, checkedProtocols, serviceMapApiData])
+
+    const onProtocolsChange = (newProtocolsFiltersnewProt) => {
+        const filterProt = newProtocolsFiltersnewProt || checkedProtocols
+        setCheckedProtocols(filterProt)
+    }
+
+    const onServiceChanges = (newServiceFilters) => {
+        const filterService = newServiceFilters || checkedServices
+        setCheckedServices([...filterService])
     }
 
     useEffect(() => {
-        if (checkedServices.length > 0) return // only after refresh
-        filterServiceMap(checkedProtocols, getServicesForFilter.map(x => x.key).filter(serviceName => !Utils.isIpAddress(serviceName)))
+        if (checkedServices.length == 0)
+            setCheckedServices(getServicesForFilter.map(x => x.key).filter(serviceName => !Utils.isIpAddress(serviceName)))
     }, [getServicesForFilter])
 
     useEffect(() => {
@@ -176,18 +174,18 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
                 <Box sx={modalStyle}>
                     <div className={styles.modalContainer}>
                         <div className={styles.filterSection}>
-                            <Resizeable minWidth={170}>
+                            <Resizeable minWidth={170} maxWidth={320}>
                                 <div className={styles.filterWrapper}>
                                     <div className={styles.protocolsFilterList}>
                                         <SelectList items={protocols} checkBoxWidth="5%" tableName={"Protocols"} multiSelect={true}
-                                            checkedValues={checkedProtocols} setCheckedValues={filterServiceMap} tableClassName={styles.filters} />
+                                            checkedValues={checkedProtocols} setCheckedValues={onProtocolsChange} tableClassName={styles.filters} />
                                     </div>
                                     <div className={styles.separtorLine}></div>
                                     <div className={styles.servicesFilter}>
                                         <input className={commonClasses.textField + ` ${styles.servicesFilterSearch}`} placeholder="search service" value={servicesSearchVal} onChange={(event) => setServicesSearchVal(event.target.value)} />
                                         <div className={styles.servicesFilterList}>
                                             <SelectList items={getServicesForFilter} tableName={"Services"} tableClassName={styles.filters} multiSelect={true} searchValue={servicesSearchVal}
-                                                checkBoxWidth="5%" checkedValues={checkedServices} setCheckedValues={(newServicesForFilter) => filterServiceMap(null, newServicesForFilter)} />
+                                                checkBoxWidth="5%" checkedValues={checkedServices} setCheckedValues={onServiceChanges} />
                                         </div>
                                     </div>
                                 </div>

ui-common/src/components/ServiceMapModal/ServiceMapOptions.ts

@@ -67,11 +67,11 @@ const ServiceMapOptions = {
             color: '#343434',
             size: 12, // px
             face: 'arial',
-            background: 'none',
+            background: 'rgba(255,255,255,0.7)',
             strokeWidth: 2, // px
             strokeColor: '#ffffff',
             align: 'horizontal',
-            multi: false,
+            multi: false
         },
         labelHighlightBold: true,
         selectionWidth: 1,

ui-common/src/components/TLSWarning/TLSWarning.sass

@@ -1,12 +0,0 @@
-.httpsDomains
-  display: none
-  margin: 0
-  padding: 0
-  list-style: none
-
-.customWarningStyle
-  &:hover
-    overflow-y: scroll
-    height: 85px
-    .httpsDomains
-      display: block

ui-common/src/components/TLSWarning/TLSWarning.tsx

@@ -1,44 +0,0 @@
-import {Snackbar} from "@material-ui/core";
-import MuiAlert from "@material-ui/lab/Alert";
-import React, {useEffect} from "react";
-import { RecoilState, useRecoilValue } from "recoil";
-import TrafficViewerApiAtom from "../../recoil/TrafficViewerApi/atom";
-import TrafficViewerApi from "../TrafficViewer/TrafficViewerApi";
-import './TLSWarning.sass';
-
-interface TLSWarningProps {
-    showTLSWarning: boolean
-    setShowTLSWarning: (show: boolean) => void
-    addressesWithTLS: Set<string>
-    setAddressesWithTLS: (addresses: Set<string>) => void
-    userDismissedTLSWarning: boolean
-    setUserDismissedTLSWarning: (flag: boolean) => void
-}
-
-export const TLSWarning: React.FC<TLSWarningProps>  = ({showTLSWarning, setShowTLSWarning, addressesWithTLS, setAddressesWithTLS, userDismissedTLSWarning, setUserDismissedTLSWarning}) => {
-
-    const trafficViewerApi = useRecoilValue(TrafficViewerApiAtom as RecoilState<TrafficViewerApi>)
-    useEffect(() => {
-        (async () => {
-            try {
-                const getRecentTLSLinksFunc = trafficViewerApi?.getRecentTLSLinks ? trafficViewerApi?.getRecentTLSLinks : function(){}
-                const recentTLSLinks = await getRecentTLSLinksFunc();
-                if (recentTLSLinks?.length > 0) {
-                    setAddressesWithTLS(new Set(recentTLSLinks));
-                    setShowTLSWarning(true);
-                }
-            } catch (e) {
-                console.error(e);
-            }
-        })();
-    }, [setShowTLSWarning, setAddressesWithTLS,trafficViewerApi]);
-
-    return (<Snackbar open={showTLSWarning && !userDismissedTLSWarning}>
-        <MuiAlert classes={{filledWarning: 'customWarningStyle'}} elevation={6} variant="filled"
-                  onClose={() => setUserDismissedTLSWarning(true)} severity="warning">
-            Mizu is detecting TLS traffic, this type of traffic will not be displayed.
-            {addressesWithTLS.size > 0 &&
-            <ul className="httpsDomains"> {Array.from(addressesWithTLS, address => <li>{address}</li>)} </ul>}
-        </MuiAlert>
-    </Snackbar>);
-}

ui-common/src/components/TrafficViewer/EntriesList.tsx

@@ -12,7 +12,7 @@ import TrafficViewerApiAtom from "../../recoil/TrafficViewerApi";
 import TrafficViewerApi from "./TrafficViewerApi";
 import focusedEntryIdAtom from "../../recoil/focusedEntryId";
 import {toast} from "react-toastify";
-import {TOAST_CONTAINER_ID} from "../../configs/Consts";
+import {MAX_ENTRIES, TOAST_CONTAINER_ID} from "../../configs/Consts";
 import tappingStatusAtom from "../../recoil/tappingStatus";
 import leftOffTopAtom from "../../recoil/leftOffTop";
 
@@ -89,7 +89,7 @@ export const EntriesList: React.FC<EntriesListProps> = ({
     setLeftOffTop(data.meta.leftOff);
 
     let scrollTo: boolean;
-    if (data.meta.leftOff === 0) {
+    if (data.meta.noMoreData) {
       setNoMoreDataTop(true);
       scrollTo = false;
     } else {
@@ -98,8 +98,8 @@ export const EntriesList: React.FC<EntriesListProps> = ({
     setIsLoadingTop(false);
 
     const newEntries = [...data.data.reverse(), ...entries];
-    if(newEntries.length > 10000) {
-      newEntries.splice(10000, newEntries.length - 10000)
+    if(newEntries.length > MAX_ENTRIES) {
+      newEntries.splice(MAX_ENTRIES, newEntries.length - MAX_ENTRIES)
     }
     setEntries(newEntries);
 
@@ -118,23 +118,28 @@ export const EntriesList: React.FC<EntriesListProps> = ({
 
   const scrollbarVisible = scrollableRef.current?.childWrapperRef.current.clientHeight > scrollableRef.current?.wrapperRef.current.clientHeight;
 
+  useEffect(() => {
+    if (!focusedEntryId && entries.length > 0)
+      setFocusedEntryId(entries[0].id);
+  }, [focusedEntryId, entries])
 
+  useEffect(() => {
+    const newEntries = [...entries];
+    if (newEntries.length > MAX_ENTRIES) {
+      setLeftOffTop(newEntries[0].id);
+      newEntries.splice(0, newEntries.length - MAX_ENTRIES)
+      setNoMoreDataTop(false);
+      setEntries(newEntries);
+    }
+  }, [entries])
 
-  if (ws.current) {
+  if(ws.current && !ws.current.onmessage) {
     ws.current.onmessage = (e) => {
       if (!e?.data) return;
       const message = JSON.parse(e.data);
       switch (message.messageType) {
         case "entry":
-          const entry = message.data;
-          if (!focusedEntryId) setFocusedEntryId(entry.id);
-          const newEntries = [...entries, entry];
-          if (newEntries.length > 10000) {
-            setLeftOffTop(newEntries[0].id);
-            newEntries.splice(0, newEntries.length - 10000)
-            setNoMoreDataTop(false);
-          }
-          setEntries(newEntries);
+          setEntries(entriesState => [...entriesState,  message.data]);
           break;
         case "status":
           setTappingStatus(message.tappingStatus);
@@ -151,9 +156,7 @@ export const EntriesList: React.FC<EntriesListProps> = ({
         case "queryMetadata":
           setTruncatedTimestamp(message.data.truncatedTimestamp);
           setQueriedTotal(message.data.total);
-          if (leftOffTop === "") {
-            setLeftOffTop(message.data.leftOff);
-          }
+          setLeftOffTop(leftOffState => leftOffState === "" ? message.data.leftOff : leftOffState);
           break;
         case "startTime":
           setStartTime(message.data);
@@ -206,10 +209,10 @@ export const EntriesList: React.FC<EntriesListProps> = ({
       </div>
 
       <div className={styles.footer}>
-        <div>Displaying <b id="entries-length">{entries?.length}</b> results out of <b
+        <div>Displaying <b id="entries-length">{entries?.length > MAX_ENTRIES ? MAX_ENTRIES : entries?.length}</b> results out of <b
           id="total-entries">{queriedTotal}</b> total
         </div>
-        {startTime !== 0 && <div>Started listening at <span style={{
+        {startTime !== 0 && <div>First traffic entry time <span style={{
           marginRight: 5,
           fontWeight: 600,
           fontSize: 13

ui-common/src/components/TrafficViewer/EntryDetailed/EntrySections.tsx

@@ -1,13 +1,13 @@
 import styles from "./EntrySections.module.sass";
-import React, {useState} from "react";
-import {SyntaxHighlighter} from "../../UI/SyntaxHighlighter/index";
+import React, { useState } from "react";
+import { SyntaxHighlighter } from "../../UI/SyntaxHighlighter/index";
 import CollapsibleContainer from "../../UI/CollapsibleContainer";
 import FancyTextDisplay from "../../UI/FancyTextDisplay";
 import Queryable from "../../UI/Queryable";
 import Checkbox from "../../UI/Checkbox";
 import ProtobufDecoder from "protobuf-decoder";
-import {default as jsonBeautify} from "json-beautify";
-import {default as xmlBeautify} from "xml-formatter";
+import { default as jsonBeautify } from "json-beautify";
+import { default as xmlBeautify } from "xml-formatter";
 
 interface EntryViewLineProps {
     label: string;
@@ -18,40 +18,40 @@ interface EntryViewLineProps {
     useTooltip?: boolean;
 }
 
-const EntryViewLine: React.FC<EntryViewLineProps> = ({label, value, selector = "", overrideQueryValue = "", displayIconOnMouseOver = true, useTooltip = true}) => {
+const EntryViewLine: React.FC<EntryViewLineProps> = ({ label, value, selector = "", overrideQueryValue = "", displayIconOnMouseOver = true, useTooltip = true }) => {
     let query: string;
     if (!selector) {
         query = "";
     } else if (overrideQueryValue) {
         query = `${selector} == ${overrideQueryValue}`;
-    } else if (typeof(value) == "string") {
+    } else if (typeof (value) == "string") {
         query = `${selector} == "${JSON.stringify(value).slice(1, -1)}"`;
     } else {
         query = `${selector} == ${value}`;
     }
     return (label && <tr className={styles.dataLine}>
-                    <td className={`${styles.dataKey}`}>
-                        <Queryable
-                            query={query}
-                            style={{float: "right", height: "18px"}}
-                            iconStyle={{marginRight: "20px"}}
-                            flipped={true}
-                            useTooltip={useTooltip}
-                            displayIconOnMouseOver={displayIconOnMouseOver}
-                        >
-                            {label}
-                        </Queryable>
-                    </td>
-                <td>
-                    <FancyTextDisplay
-                        className={styles.dataValue}
-                        text={value}
-                        applyTextEllipsis={false}
-                        flipped={true}
-                        displayIconOnMouseOver={true}
-                    />
-                </td>
-            </tr>) || null;
+        <td className={`${styles.dataKey}`}>
+            <Queryable
+                query={query}
+                style={{ float: "right", height: "18px" }}
+                iconStyle={{ marginRight: "20px" }}
+                flipped={true}
+                useTooltip={useTooltip}
+                displayIconOnMouseOver={displayIconOnMouseOver}
+            >
+                {label}
+            </Queryable>
+        </td>
+        <td>
+            <FancyTextDisplay
+                className={styles.dataValue}
+                text={value}
+                applyTextEllipsis={false}
+                flipped={true}
+                displayIconOnMouseOver={true}
+            />
+        </td>
+    </tr>) || null;
 }
 
 
@@ -63,11 +63,11 @@ interface EntrySectionCollapsibleTitleProps {
     query?: string,
 }
 
-const EntrySectionCollapsibleTitle: React.FC<EntrySectionCollapsibleTitleProps> = ({title, color, expanded, setExpanded, query = ""}) => {
+const EntrySectionCollapsibleTitle: React.FC<EntrySectionCollapsibleTitleProps> = ({ title, color, expanded, setExpanded, query = "" }) => {
     return <div className={styles.title}>
         <div
             className={`${styles.button} ${expanded ? styles.expanded : ''}`}
-            style={{backgroundColor: color}}
+            style={{ backgroundColor: color }}
             onClick={() => {
                 setExpanded(!expanded)
             }}
@@ -90,12 +90,12 @@ interface EntrySectionContainerProps {
     query?: string,
 }
 
-export const EntrySectionContainer: React.FC<EntrySectionContainerProps> = ({title, color, children, query = ""}) => {
+export const EntrySectionContainer: React.FC<EntrySectionContainerProps> = ({ title, color, children, query = "" }) => {
     const [expanded, setExpanded] = useState(true);
     return <CollapsibleContainer
         className={styles.collapsibleContainer}
         expanded={expanded}
-        title={<EntrySectionCollapsibleTitle title={title} color={color} expanded={expanded} setExpanded={setExpanded} query={query}/>}
+        title={<EntrySectionCollapsibleTitle title={title} color={color} expanded={expanded} setExpanded={setExpanded} query={query} />}
     >
         {children}
     </CollapsibleContainer>
@@ -130,6 +130,7 @@ export const EntryBodySection: React.FC<EntryBodySectionProps> = ({
 
     const isBase64Encoding = encoding === 'base64';
     const supportsPrettying = supportedFormats.some(format => contentType?.indexOf(format) > -1);
+    const [isDecodeGrpc, setIsDecodeGrpc] = useState(true);
 
     const formatTextBody = (body: any): string => {
         if (!decodeBase64) return body;
@@ -141,7 +142,7 @@ export const EntryBodySection: React.FC<EntryBodySectionProps> = ({
             if (jsonLikeFormats.some(format => contentType?.indexOf(format) > -1)) {
                 if (!isPretty) return bodyBuf;
                 return jsonBeautify(JSON.parse(bodyBuf), null, 2, 80);
-            }  else if (xmlLikeFormats.some(format => contentType?.indexOf(format) > -1)) {
+            } else if (xmlLikeFormats.some(format => contentType?.indexOf(format) > -1)) {
                 if (!isPretty) return bodyBuf;
                 return xmlBeautify(bodyBuf, {
                     indentation: '  ',
@@ -157,49 +158,55 @@ export const EntryBodySection: React.FC<EntryBodySectionProps> = ({
                 return jsonBeautify(protobufDecoded, null, 2, 80);
             }
         } catch (error) {
-            console.error(error);
+            if(isDecodeGrpc)
+                setIsDecodeGrpc(false);
+            if (!String(error).includes("More than one message in")){
+                console.error(error);
+            }
         }
         return bodyBuf;
     }
 
     return <React.Fragment>
         {content && content?.length > 0 && <EntrySectionContainer
-                                                title={title}
-                                                color={color}
-                                                query={`${selector} == r".*"`}
-                                            >
-            <div style={{display: 'flex', alignItems: 'center', alignContent: 'center', margin: "5px 0"}}>
-                {supportsPrettying && <div style={{paddingTop: 3}}>
-                    <Checkbox checked={isPretty} onToggle={() => {setIsPretty(!isPretty)}}/>
+            title={title}
+            color={color}
+            query={`${selector} == r".*"`}
+        >
+            <div style={{ display: 'flex', alignItems: 'center', alignContent: 'center', margin: "5px 0" }}>
+                {supportsPrettying && <div style={{ paddingTop: 3 }}>
+                    <Checkbox checked={isPretty} onToggle={() => { setIsPretty(!isPretty) }} />
                 </div>}
-                {supportsPrettying && <span style={{marginLeft: '.2rem'}}>Pretty</span>}
+                {supportsPrettying && <span style={{ marginLeft: '.2rem' }}>Pretty</span>}
 
-                <div style={{paddingTop: 3, paddingLeft: supportsPrettying ? 20 : 0}}>
-                    <Checkbox checked={showLineNumbers} onToggle={() => {setShowLineNumbers(!showLineNumbers)}}/>
+                <div style={{ paddingTop: 3, paddingLeft: supportsPrettying ? 20 : 0 }}>
+                    <Checkbox checked={showLineNumbers} onToggle={() => { setShowLineNumbers(!showLineNumbers) }} />
                 </div>
-                <span style={{marginLeft: '.2rem'}}>Line numbers</span>
-
-                {isBase64Encoding && <div style={{paddingTop: 3, paddingLeft: 20}}>
-                    <Checkbox checked={decodeBase64} onToggle={() => {setDecodeBase64(!decodeBase64)}}/>
+                <span style={{ marginLeft: '.2rem' }}>Line numbers</span>      
+                {isBase64Encoding && <div style={{ paddingTop: 3, paddingLeft: 20 }}>
+                    <Checkbox checked={decodeBase64} onToggle={() => { setDecodeBase64(!decodeBase64) }} />
                 </div>}
-                {isBase64Encoding && <span style={{marginLeft: '.2rem'}}>Decode Base64</span>}
-            </div>
+                {isBase64Encoding && <span style={{ marginLeft: '.2rem' }}>Decode Base64</span>}
+                {!isDecodeGrpc && <span style={{ fontSize: '12px', color: '#DB2156', marginLeft: '.8rem' }}>More than one message in protobuf payload is not supported</span>}
+            </div>  
 
-            <SyntaxHighlighter
+            <SyntaxHighlighter  
                 code={formatTextBody(content)}
                 showLineNumbers={showLineNumbers}
             />
+
         </EntrySectionContainer>}
     </React.Fragment>
 }
 
+
 interface EntrySectionProps {
     title: string,
     color: string,
     arrayToIterate: any[],
 }
 
-export const EntryTableSection: React.FC<EntrySectionProps> = ({title, color, arrayToIterate}) => {
+export const EntryTableSection: React.FC<EntrySectionProps> = ({ title, color, arrayToIterate }) => {
     let arrayToIterateSorted: any[];
     if (arrayToIterate) {
         arrayToIterateSorted = arrayToIterate.sort((a, b) => {
@@ -220,7 +227,7 @@ export const EntryTableSection: React.FC<EntrySectionProps> = ({title, color, ar
                 <EntrySectionContainer title={title} color={color}>
                     <table>
                         <tbody id={`tbody-${title}`}>
-                            {arrayToIterateSorted.map(({name, value, selector}, index) => <EntryViewLine
+                            {arrayToIterateSorted.map(({ name, value, selector }, index) => <EntryViewLine
                                 key={index}
                                 label={name}
                                 value={value}
@@ -228,7 +235,7 @@ export const EntryTableSection: React.FC<EntrySectionProps> = ({title, color, ar
                             />)}
                         </tbody>
                     </table>
-                </EntrySectionContainer> : <span/>
+                </EntrySectionContainer> : <span />
         }
     </React.Fragment>
 }
@@ -247,7 +254,7 @@ interface EntryPolicySectionCollapsibleTitleProps {
     setExpanded: any;
 }
 
-const EntryPolicySectionCollapsibleTitle: React.FC<EntryPolicySectionCollapsibleTitleProps> = ({label, matched, expanded, setExpanded}) => {
+const EntryPolicySectionCollapsibleTitle: React.FC<EntryPolicySectionCollapsibleTitleProps> = ({ label, matched, expanded, setExpanded }) => {
     return <div className={styles.title}>
         <span
             className={`${styles.button}
@@ -260,8 +267,8 @@ const EntryPolicySectionCollapsibleTitle: React.FC<EntryPolicySectionCollapsible
         </span>
         <span>
             <tr className={styles.dataLine}>
-            <td className={`${styles.dataKey} ${styles.rulesTitleSuccess}`}>{label}</td>
-            <td className={`${styles.dataKey} ${matched === 'Success' ? styles.rulesMatchedSuccess : styles.rulesMatchedFailure}`}>{matched}</td>
+                <td className={`${styles.dataKey} ${styles.rulesTitleSuccess}`}>{label}</td>
+                <td className={`${styles.dataKey} ${matched === 'Success' ? styles.rulesMatchedSuccess : styles.rulesMatchedFailure}`}>{matched}</td>
             </tr>
         </span>
     </div>
@@ -273,28 +280,28 @@ interface EntryPolicySectionContainerProps {
     children?: any;
 }
 
-export const EntryPolicySectionContainer: React.FC<EntryPolicySectionContainerProps> = ({label, matched, children}) => {
+export const EntryPolicySectionContainer: React.FC<EntryPolicySectionContainerProps> = ({ label, matched, children }) => {
     const [expanded, setExpanded] = useState(false);
     return <CollapsibleContainer
         className={styles.collapsibleContainer}
         expanded={expanded}
-        title={<EntryPolicySectionCollapsibleTitle label={label} matched={matched} expanded={expanded} setExpanded={setExpanded}/>}
+        title={<EntryPolicySectionCollapsibleTitle label={label} matched={matched} expanded={expanded} setExpanded={setExpanded} />}
     >
         {children}
     </CollapsibleContainer>
 }
 
-export const EntryTablePolicySection: React.FC<EntryPolicySectionProps> = ({title, color, latency, arrayToIterate}) => {
+export const EntryTablePolicySection: React.FC<EntryPolicySectionProps> = ({ title, color, latency, arrayToIterate }) => {
     return <React.Fragment>
         {
             arrayToIterate && arrayToIterate.length > 0 ?
                 <React.Fragment>
-                <EntrySectionContainer title={title} color={color}>
-                    <table>
-                        <tbody>
-                            {arrayToIterate.map(({rule, matched}, index) => {
+                    <EntrySectionContainer title={title} color={color}>
+                        <table>
+                            <tbody>
+                                {arrayToIterate.map(({ rule, matched }, index) => {
                                     return (
-                                        <EntryPolicySectionContainer key={index} label={rule.Name} matched={matched && (rule.Type === 'slo' ? rule.ResponseTime >= latency : true)? "Success" : "Failure"}>
+                                        <EntryPolicySectionContainer key={index} label={rule.Name} matched={matched && (rule.Type === 'slo' ? rule.ResponseTime >= latency : true) ? "Success" : "Failure"}>
                                             {
                                                 <React.Fragment>
                                                     {
@@ -330,11 +337,11 @@ export const EntryTablePolicySection: React.FC<EntryPolicySectionProps> = ({titl
                                         </EntryPolicySectionContainer>
                                     )
                                 }
-                            )
-                            }
-                        </tbody>
-                    </table>
-                </EntrySectionContainer>
+                                )
+                                }
+                            </tbody>
+                        </table>
+                    </EntrySectionContainer>
                 </React.Fragment> : <span className={styles.noRules}>No rules could be applied to this request.</span>
         }
     </React.Fragment>
@@ -347,7 +354,7 @@ interface EntryContractSectionProps {
     contractContent: string,
 }
 
-export const EntryContractSection: React.FC<EntryContractSectionProps> = ({color, requestReason, responseReason, contractContent}) => {
+export const EntryContractSection: React.FC<EntryContractSectionProps> = ({ color, requestReason, responseReason, contractContent }) => {
     return <React.Fragment>
         {requestReason && <EntrySectionContainer title="Request" color={color}>
             {requestReason}

ui-common/src/components/TrafficViewer/EntryListItem/EntryListItem.module.sass

@@ -67,7 +67,6 @@
 
 .capture img
   height: 14px
-  z-index: 1000
   margin-top: 12px
   margin-left: -2px
 

ui-common/src/components/TrafficViewer/TrafficViewer.tsx

@@ -14,7 +14,6 @@ import {RecoilRoot, RecoilState, useRecoilState, useRecoilValue, useSetRecoilSta
 import entriesAtom from "../../recoil/entries";
 import focusedEntryIdAtom from "../../recoil/focusedEntryId";
 import queryAtom from "../../recoil/query";
-import {TLSWarning} from "../TLSWarning/TLSWarning";
 import trafficViewerApiAtom from "../../recoil/TrafficViewerApi"
 import TrafficViewerApi from "./TrafficViewerApi";
 import {StatusBar} from "../UI/StatusBar";
@@ -77,10 +76,6 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
   const setLeftOffTop = useSetRecoilState(leftOffTopAtom);
   const scrollableRef = useRef(null);
 
-  const [showTLSWarning, setShowTLSWarning] = useState(false);
-  const [userDismissedTLSWarning, setUserDismissedTLSWarning] = useState(false);
-  const [addressesWithTLS, setAddressesWithTLS] = useState(new Set<string>());
-
   const handleQueryChange = useMemo(
     () =>
       debounce(async (query: string) => {
@@ -286,12 +281,6 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
           <EntryDetailed/>
         </div>
       </div>}
-      <TLSWarning showTLSWarning={showTLSWarning}
-                  setShowTLSWarning={setShowTLSWarning}
-                  addressesWithTLS={addressesWithTLS}
-                  setAddressesWithTLS={setAddressesWithTLS}
-                  userDismissedTLSWarning={userDismissedTLSWarning}
-                  setUserDismissedTLSWarning={setUserDismissedTLSWarning}/>
     </div>
   );
 };

ui-common/src/components/TrafficViewer/TrafficViewerApi.ts

@@ -4,7 +4,6 @@ type TrafficViewerApi = {
   analyzeStatus: () => any
   fetchEntries: (leftOff: any, direction: number, query: any, limit: number, timeoutMs: number) => any
   getEntry: (entryId: any, query: string) => any
-  getRecentTLSLinks: () => any,
   webSocket: {
     close: () => void
   }

ui-common/src/components/UI/InformationIcon.tsx

@@ -1,19 +1,22 @@
-import React, { CSSProperties } from "react";
+import React from "react";
 import styles from "./style/InformationIcon.module.sass"
 
 const DEFUALT_LINK = "https://getmizu.io/docs"
 
-export interface InformationIconProps {
+interface LinkProps {
     link?: string,
-    style?: CSSProperties
+    className?: string
+    title?: string
 }
 
-export const InformationIcon: React.FC<InformationIconProps> = ({ link, style }) => {
-    return <React.Fragment>
-        <a href={DEFUALT_LINK ? DEFUALT_LINK : link} style={style} className={styles.linkStyle} title="documentation" target="_blank">
-            <span>Docs</span>
-        </a>
-    </React.Fragment>
+export const Link: React.FC<LinkProps> = ({ link, className, title, children }) => {
+    return <a href={link} className={className} title={title} target="_blank">
+        {children}
+    </a>
 }
 
-
+export const InformationIcon: React.FC<LinkProps> = ({ className }) => {
+    return <Link title="documentation" className={`${styles.linkStyle} ${className}`} link={DEFUALT_LINK}>
+        <span>Docs</span>
+    </Link>
+}

ui-common/src/components/UI/Resizeable.tsx

@@ -5,9 +5,10 @@ import styles from './style/Resizeable.module.sass'
 export interface Props {
     children
     minWidth: number
+    maxWidth?: number
 }
 
-const Resizeable: React.FC<Props> = ({ children, minWidth }) => {
+const Resizeable: React.FC<Props> = ({ children, minWidth, maxWidth }) => {
     const resizeble = useRef(null)
     let mousePos = { x: 0, y: 0 }
     let elementDimention = { w: 0, h: 0 }
@@ -47,14 +48,11 @@ const Resizeable: React.FC<Props> = ({ children, minWidth }) => {
     };
 
     return (
-        <React.Fragment>
-            <div className={styles.resizable} ref={resizeble} style={{ width: elemWidth }}>
-                {children}
-                <div className={`${styles.resizer} ${styles.resizerRight}`} onMouseDown={mouseDownHandler}></div>
-                {/* <div className={`${styles.resizer} ${styles.resizerB}`} onMouseDown={mouseDownHandler}></div> -- FutureUse*/}
-            </div>
-
-        </React.Fragment>
+        <div className={styles.resizable} ref={resizeble} style={{ width: elemWidth, maxWidth: maxWidth }}>
+            {children}
+            <div className={`${styles.resizer} ${styles.resizerRight}`} onMouseDown={mouseDownHandler}></div>
+            {/* <div className={`${styles.resizer} ${styles.resizerB}`} onMouseDown={mouseDownHandler}></div> -- FutureUse*/}
+        </div>
     );
 };
 

ui-common/src/components/UI/index.ts

@@ -5,10 +5,10 @@ import Tooltip from "./Tooltip";
 import Checkbox from "./Checkbox"
 import { StatusBar } from "./StatusBar";
 import CustomModal from "./CustomModal";
-import { InformationIcon } from "./InformationIcon";
+import { InformationIcon, Link } from "./InformationIcon";
 import SelectList from "./SelectList";
 import NoDataMessage from "./NoDataMessage";
 
 
-export { LoadingOverlay, Select, Tabs, Tooltip, Checkbox, CustomModal, InformationIcon, SelectList, NoDataMessage }
+export { LoadingOverlay, Select, Tabs, Tooltip, Checkbox, CustomModal, InformationIcon, SelectList, NoDataMessage, Link }
 export { StatusBar }

ui-common/src/components/style/LoadingOverlay.module.sass

@@ -8,6 +8,7 @@
     align-items: center
     justify-content: center
     background-color: rgba(25, 25, 25, 0.5)
+    z-index: 1000
 
 .loadingOverlaySpinner
     width: 60px

ui-common/src/configs/Consts.ts

@@ -1 +1,2 @@
-export const TOAST_CONTAINER_ID = "Common";
+export const TOAST_CONTAINER_ID = "Common";
+export const MAX_ENTRIES = 10000;

ui/package.json

@@ -27,7 +27,6 @@
     "node-fetch": "^3.1.1",
     "node-sass": "^6.0.0",
     "numeral": "^2.0.6",
-    "protobuf-decoder": "^0.1.0",
     "react": "^17.0.2",
     "react-copy-to-clipboard": "^5.0.3",
     "react-dom": "^17.0.2",

ui/src/helpers/api.js

@@ -62,11 +62,6 @@ export default class Api {
         return response.data;
     }
 
-    getRecentTLSLinks = async () => {
-        const response = await client.get("/status/recentTLSLinks");
-        return response.data;
-    }
-
     getAuthStatus = async () => {
         const response = await client.get("/status/auth");
         return response.data;

ui/src/index.sass

@@ -139,6 +139,7 @@ button
 
 ::-webkit-scrollbar
   width: 6px
+  height: 6px
 
 ::-webkit-scrollbar-thumb
   background-color: $light-blue-color