Fix the Redis dissector (#739)

Co-authored-by: Igor Gov <iggvrv@gmail.com>

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: PR validation
+name: Build
 
 on:
   pull_request:
@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   build-cli:
-    name: Build CLI
+    name: CLI executable build
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.16
@@ -27,35 +27,11 @@ jobs:
         run: make cli
 
   build-agent:
-    name: Build Agent
+    name: Agent docker image build
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.16'
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
-      - shell: bash
-        run: |
-          sudo apt-get install libpcap-dev
-
       - name: Build Agent
-        run: make agent
-
-  build-ui:
-    name: Build UI
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Node 16
-        uses: actions/setup-node@v2
-        with:
-          node-version: '16'
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Build UI
-        run: make ui
+        run: make agent-docker

.github/workflows/static_code_analysis.yml

@@ -0,0 +1,87 @@
+name: Static code analysis
+
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+
+permissions:
+  contents: read
+
+jobs:
+  golangci:
+    name: Go lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y libpcap-dev
+
+      - name: Go lint - agent
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: agent
+          args: --timeout=3m
+
+      - name: Go lint - shared
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: shared
+          args: --timeout=3m
+
+      - name: Go lint - tap
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: tap
+          args: --timeout=3m
+
+      - name: Go lint - CLI
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: cli
+          args: --timeout=3m
+
+      - name: Go lint - acceptanceTests
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: acceptanceTests
+          args: --timeout=3m
+
+      - name: Go lint - tap/api
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: tap/api
+
+      - name: Go lint - tap/extensions/amqp
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: tap/extensions/amqp
+
+      - name: Go lint - tap/extensions/http
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: tap/extensions/http
+
+      - name: Go lint - tap/extensions/kafka
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: tap/extensions/kafka
+
+      - name: Go lint - tap/extensions/redis
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: tap/extensions/redis

.github/workflows/test.yml

@@ -1,14 +1,10 @@
-name: tests validation
+name: Test
 
 on:
   pull_request:
     branches:
       - 'develop'
       - 'main'
-  push:
-    branches:
-      - 'develop'
-      - 'main'
 
 concurrency:
   group: mizu-tests-validation-${{ github.ref }}
@@ -16,7 +12,7 @@ concurrency:
 
 jobs:
   run-tests-cli:
-    name: Run CLI tests
+    name: CLI Tests
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.16
@@ -34,7 +30,7 @@ jobs:
         uses: codecov/codecov-action@v2
 
   run-tests-agent:
-    name: Run Agent tests
+    name: Agent Tests
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.16

Dockerfile

@@ -16,7 +16,7 @@ RUN npm run build-ent
 ### Base builder image for native builds architecture
 FROM golang:1.16-alpine AS builder-native-base
 ENV CGO_ENABLED=1 GOOS=linux
-RUN apk add libpcap-dev g++
+RUN apk add libpcap-dev g++ perl-utils
 
 
 ### Intermediate builder image for x86-64 to x86-64 native builds
@@ -77,20 +77,29 @@ RUN go build -ldflags="-extldflags=-static -s -w \
     -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
     -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
+# Download Basenine executable, verify the sha1sum
+ADD https://github.com/up9inc/basenine/releases/download/v0.4.13/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
+ADD https://github.com/up9inc/basenine/releases/download/v0.4.13/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
+RUN shasum -a 256 -c basenine_linux_${GOARCH}.sha256
+RUN chmod +x ./basenine_linux_${GOARCH}
+RUN mv ./basenine_linux_${GOARCH} ./basenine
+
 
 ### The shipped image
 ARG TARGETARCH=amd64
 FROM ${TARGETARCH}/busybox:latest
 
+# gin-gonic runs in debug mode without this
+ENV GIN_MODE=release
+
+WORKDIR /app/data/
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
+COPY --from=builder ["/app/agent-build/basenine", "/usr/local/bin/basenine"]
 COPY --from=front-end ["/app/ui-build/build", "site"]
 COPY --from=front-end ["/app/ui-build/build-ent", "site-standalone"]
 
-# gin-gonic runs in debug mode without this
-ENV GIN_MODE=release
-
 # this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
-ENTRYPOINT "/app/mizuagent"
+ENTRYPOINT ["/app/mizuagent"]

Makefile

@@ -47,6 +47,10 @@ agent-debug: ## Build agent for debug.
 docker: ## Build and publish agent docker image.
 	$(MAKE) push-docker
 
+agent-docker: ## Build agent docker image.
+	@echo "Building agent docker image"
+	@docker build -t up9inc/mizu:devlatest .
+
 push: push-docker push-cli ## Build and publish agent docker image & CLI.
 
 push-docker: ## Build and publish agent docker image.

acceptanceTests/cypress.json

@@ -4,13 +4,16 @@
   "viewportHeight": 1080,
   "video": false,
   "screenshotOnRunFailure": false,
+  "defaultCommandTimeout": 6000,
   "testFiles": [
     "tests/GuiPort.js",
     "tests/MultipleNamespaces.js",
     "tests/Redact.js",
     "tests/NoRedact.js",
     "tests/Regex.js",
-    "tests/RegexMasking.js"
+    "tests/RegexMasking.js",
+    "tests/IgnoredUserAgents.js",
+    "tests/UiTest.js"
   ],
 
   "env": {

acceptanceTests/cypress/integration/configurations/HugeMizu.json

@@ -1,14 +0,0 @@
-{
-  "watchForFileChanges":false,
-  "viewportWidth": 1920,
-  "viewportHeight": 3500,
-  "video": false,
-  "screenshotOnRunFailure": false,
-  "testFiles": [
-    "tests/IgnoredUserAgents.js"
-  ],
-
-  "env": {
-    "testUrl": "http://localhost:8899/"
-  }
-}

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -7,3 +7,12 @@ export function isValueExistsInElement(shouldInclude, content, domPathToContaine
         });
     });
 }
+
+export function resizeToHugeMizu() {
+    cy.viewport(1920, 3500);
+}
+
+export function resizeToNormalMizu() {
+    cy.viewport(1920, 1080);
+}
+

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -1,10 +1,11 @@
-import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
+import {isValueExistsInElement, resizeToHugeMizu} from "../testHelpers/TrafficHelper";
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
 });
 
 it('going through each entry', function () {
+    resizeToHugeMizu();
     cy.get('#total-entries').then(number => {
         const getNum = () => {
             const numOfEntries = number.text();

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -0,0 +1,281 @@
+import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
+import {resizeToHugeMizu, resizeToNormalMizu} from "../testHelpers/TrafficHelper";
+const greenFilterColor = 'rgb(210, 250, 210)';
+const redFilterColor = 'rgb(250, 214, 220)';
+const refreshWaitTimeout = 10000;
+
+it('opening mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+it('top bar check', function () {
+    const podName1 = 'httpbin', namespace1 = 'mizu-tests';
+    const podName2 = 'httpbin2', namespace2 = 'mizu-tests';
+
+    cy.get('.podsCount').trigger('mouseover');
+    findLineAndCheck(getExpectedDetailsDict(podName1, namespace1));
+    findLineAndCheck(getExpectedDetailsDict(podName2, namespace2));
+    cy.reload();
+});
+
+it('filtering guide check', function () {
+    cy.get('[title="Open Filtering Guide (Cheatsheet)"]').click();
+    cy.get('#modal-modal-title').should('be.visible');
+    cy.get('[lang="en"]').click(0, 0);
+    cy.get('#modal-modal-title').should('not.exist');
+});
+
+checkIllegalFilter('invalid filter');
+
+checkFilter({
+    name: 'http',
+    leftSidePath: '> :nth-child(1) > :nth-child(1)',
+    leftSideExpectedText: 'HTTP',
+    rightSidePath: '[title=HTTP]',
+    rightSideExpectedText: 'Hypertext Transfer Protocol -- HTTP/1.1',
+    applyByEnter: true
+});
+
+checkFilter({
+    name: 'response.status == 200',
+    leftSidePath: '[title="Status Code"]',
+    leftSideExpectedText: '200',
+    rightSidePath: '> :nth-child(2) [title="Status Code"]',
+    rightSideExpectedText: '200',
+    applyByEnter: false
+});
+
+checkFilter({
+    name: 'src.name == ""',
+    leftSidePath: '[title="Source Name"]',
+    leftSideExpectedText: '[Unresolved]',
+    rightSidePath: '> :nth-child(2) [title="Source Name"]',
+    rightSideExpectedText: '[Unresolved]',
+    applyByEnter: false
+});
+
+checkFilter({
+    name: 'method == "GET"',
+    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+    leftSideExpectedText: 'GET',
+    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+    rightSideExpectedText: 'GET',
+    applyByEnter: true
+});
+
+checkFilter({
+    name: 'summary == "/get"',
+    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+    leftSideExpectedText: '/get',
+    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+    rightSideExpectedText: '/get',
+    applyByEnter: false
+});
+
+checkFilter({
+    name: 'dst.name == "httpbin.mizu-tests"',
+    leftSidePath: '> :nth-child(3) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
+    leftSideExpectedText: 'httpbin.mizu-tests',
+    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
+    rightSideExpectedText: 'httpbin.mizu-tests',
+    applyByEnter: false
+});
+
+checkFilter({
+    name: 'src.ip == "127.0.0.1"',
+    leftSidePath: '[title="Source IP"]',
+    leftSideExpectedText: '127.0.0.1',
+    rightSidePath: '> :nth-child(2) [title="Source IP"]',
+    rightSideExpectedText: '127.0.0.1',
+    applyByEnter: false
+});
+
+checkFilterNoResults('method == "POST"');
+
+function checkFilterNoResults(filterName) {
+    it(`checking the filter: ${filterName}. Expecting no results`, function () {
+        cy.get('#total-entries').then(number => {
+            const totalEntries = number.text();
+
+            // applying the filter
+            cy.get('.w-tc-editor-text').type(filterName);
+            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', greenFilterColor);
+            cy.get('[type="submit"]').click();
+
+            // waiting for the entries number to load
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+
+            // the DOM should show 0 entries
+            cy.get('#entries-length').should('have.text', '0');
+
+            // going through every potential entry and verifies that it doesn't exist
+            [...Array(parseInt(totalEntries)).keys()].map(shouldNotExist);
+
+            cy.get('[title="Fetch old records"]').click();
+            cy.get('#noMoreDataTop', {timeout: refreshWaitTimeout}).should('be.visible');
+            cy.get('#entries-length').should('have.text', '0'); // after loading all entries there should still be 0 entries
+
+            // reloading then waiting for the entries number to load
+            cy.reload();
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+        });
+    });
+}
+
+function shouldNotExist(entryNum) {
+    cy.get(`entry-${entryNum}`).should('not.exist');
+}
+
+function checkIllegalFilter(illegalFilterName) {
+    it(`should show red search bar with the input: ${illegalFilterName}`, function () {
+        cy.get('#total-entries').then(number => {
+            const totalEntries = number.text();
+
+            cy.get('.w-tc-editor-text').type(illegalFilterName);
+            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', redFilterColor);
+            cy.get('[type="submit"]').click();
+
+            cy.get('[role="alert"]').should('be.visible');
+            cy.get('.w-tc-editor-text').clear();
+
+            // reloading then waiting for the entries number to load
+            cy.reload();
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+        });
+    });
+}
+function checkFilter(filterDetails){
+    const {name, leftSidePath, rightSidePath, rightSideExpectedText, leftSideExpectedText, applyByEnter} = filterDetails;
+    const entriesForDeeperCheck = 5;
+
+    it(`checking the filter: ${name}`, function () {
+        cy.get('#total-entries').then(number => {
+            const totalEntries = number.text();
+
+            // checks the hover on the last entry (the only one in DOM at the beginning)
+            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
+
+            // applying the filter with alt+enter or with the button
+            cy.get('.w-tc-editor-text').type(`${name}${applyByEnter ? '{alt+enter}' : ''}`);
+            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', greenFilterColor);
+            if (!applyByEnter)
+                cy.get('[type="submit"]').click();
+
+            // only one entry in DOM after filtering, checking all four checks on it
+            leftTextCheck(totalEntries - 1, leftSidePath, leftSideExpectedText);
+            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
+            rightTextCheck(rightSidePath, rightSideExpectedText);
+            rightOnHoverCheck(rightSidePath, name);
+
+            cy.get('[title="Fetch old records"]').click();
+            resizeToHugeMizu();
+
+            // waiting for the entries number to load
+            cy.get('#entries-length', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+
+            // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
+            [...Array(parseInt(totalEntries)).keys()].forEach(entryNum => {
+                leftTextCheck(entryNum, leftSidePath, leftSideExpectedText);
+            });
+
+            // making the other 3 checks on the first X entries (longer time for each check)
+            deeperChcek(leftSidePath, rightSidePath, name, leftSideExpectedText, rightSideExpectedText, entriesForDeeperCheck);
+
+            // reloading then waiting for the entries number to load
+            resizeToNormalMizu();
+            cy.reload();
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+        });
+    });
+}
+
+function deeperChcek(leftSidePath, rightSidePath, filterName, leftSideExpectedText, rightSideExpectedText, entriesNumToCheck) {
+    [...Array(entriesNumToCheck).keys()].forEach(entryNum => {
+        leftOnHoverCheck(entryNum, leftSidePath, filterName);
+
+        cy.get(`#list #entry-${entryNum}`).click();
+        rightTextCheck(rightSidePath, rightSideExpectedText);
+        rightOnHoverCheck(rightSidePath, filterName);
+        checkRightSide();
+    });
+}
+
+function leftTextCheck(entryNum, path, expectedText) {
+    cy.get(`#list #entry-${entryNum} ${path}`).invoke('text').should('eq', expectedText);
+}
+
+function leftOnHoverCheck(entryNum, path, filterName) {
+    cy.get(`#list #entry-${entryNum} ${path}`).trigger('mouseover');
+    cy.get(`#list #entry-${entryNum} .Queryable-Tooltip`).should('have.text', filterName);
+}
+
+function rightTextCheck(path, expectedText) {
+    cy.get(`.TrafficPage-Container > :nth-child(2) ${path}`).should('have.text', expectedText);
+}
+
+function rightOnHoverCheck(path, expectedText) {
+    cy.get(`.TrafficPage-Container > :nth-child(2) ${path}`).trigger('mouseover');
+    cy.get(`.TrafficPage-Container > :nth-child(2) .Queryable-Tooltip`).should('have.text', expectedText);
+}
+
+function checkRightSide() {
+    const encodedBody = 'eyJhcmdzIjp7fSwiaGVhZGVycyI6eyJBY2NlcHQtRW5jb2RpbmciOiJnemlwIiwiSG9zdCI6IjEyNy4wLjAuMTo1MDY2OCIsIlVzZXItQWdlbnQiOiJbUkVEQUNURURdIiwiWC1Gb3J3YXJkZWQtVXJpIjoiL2FwaS92MS9uYW1lc3BhY2VzL21penUtdGVzdHMvc2VydmljZXMvaHR0cGJpbi9wcm94eS9nZXQifSwib3JpZ2luIjoiMTI3LjAuMC4xLCAxOTIuMTY4LjQ5LjEiLCJ1cmwiOiJodHRwOi8vMTI3LjAuMC4xOjUwNjY4L2dldCJ9';
+    const decodedBody = atob(encodedBody);
+
+    cy.contains('Response').click();
+    clickCheckbox('Decode Base64');
+    cy.get('.hljs').should('have.text', encodedBody);
+    clickCheckbox('Decode Base64');
+
+    cy.get('.hljs > ').its('length').should('be.gt', 1).then(linesNum => {
+        cy.get('.hljs > >').its('length').should('be.gt', linesNum).then(jsonItemsNum => {
+            checkPrettyAndLineNums(jsonItemsNum, decodedBody);
+
+            clickCheckbox('Line numbers');
+            checkPrettyOrNothing(jsonItemsNum, decodedBody);
+
+            clickCheckbox('Pretty');
+            checkPrettyOrNothing(jsonItemsNum, decodedBody);
+
+            clickCheckbox('Line numbers');
+            checkOnlyLineNumberes(jsonItemsNum, decodedBody);
+        });
+    });
+}
+
+function clickCheckbox(type) {
+    cy.contains(`${type}`).prev().children().click();
+}
+
+function checkPrettyAndLineNums(jsonItemsLen, decodedBody) {
+    decodedBody = decodedBody.replaceAll(' ', '');
+    cy.get('.hljs >').then(elements => {
+        const lines = Object.values(elements);
+        lines.forEach((line, index) => {
+            if (line.getAttribute) {
+                const cleanLine = getCleanLine(line);
+                const currentLineFromDecodedText = decodedBody.substring(0, cleanLine.length);
+
+                expect(cleanLine).to.equal(currentLineFromDecodedText, `expected the text in line number ${index + 1} to match the text that generated by the base64 decoding`)
+
+                decodedBody = decodedBody.substring(cleanLine.length);
+            }
+        });
+    });
+}
+
+function getCleanLine(lineElement) {
+    return (lineElement.innerText.substring(0, lineElement.innerText.length - 1)).replaceAll(' ', '');
+}
+
+function checkPrettyOrNothing(jsonItems, decodedBody) {
+    cy.get('.hljs > ').should('have.length', jsonItems).then(text => {
+        const json = text.text();
+        expect(json).to.equal(decodedBody);
+    });
+}
+
+function checkOnlyLineNumberes(jsonItems, decodedText) {
+    cy.get('.hljs >').should('have.length', 1).and('have.text', decodedText);
+    cy.get('.hljs > >').should('have.length', jsonItems)
+}

acceptanceTests/tap_test.go

@@ -62,35 +62,7 @@ func TestTap(t *testing.T) {
 				}
 			}
 
-			entriesCheckFunc := func() error {
-				timestamp := time.Now().UnixNano() / int64(time.Millisecond)
-
-				entries, err := getDBEntries(timestamp, entriesCount, 1*time.Second)
-				if err != nil {
-					return err
-				}
-				err = checkEntriesAtLeast(entries, 1)
-				if err != nil {
-					return err
-				}
-				entry := entries[0]
-
-				entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"])
-				requestResult, requestErr := executeHttpGetRequest(entryUrl)
-				if requestErr != nil {
-					return fmt.Errorf("failed to get entry, err: %v", requestErr)
-				}
-
-				if requestResult == nil {
-					return fmt.Errorf("unexpected nil entry result")
-				}
-
-				return nil
-			}
-			if err := retriesExecute(shortRetriesCount, entriesCheckFunc); err != nil {
-				t.Errorf("%v", err)
-				return
-			}
+			runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/UiTest.js\"")
 		})
 	}
 }
@@ -138,7 +110,7 @@ func TestTapGuiPort(t *testing.T) {
 				return
 			}
 
-			runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env port=%d --config-file cypress/integration/configurations/Default.json", guiPort))
+			runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env port=%d", guiPort))
 		})
 	}
 }
@@ -184,7 +156,7 @@ func TestTapAllNamespaces(t *testing.T) {
 		return
 	}
 
-	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v --config-file cypress/integration/configurations/Default.json",
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
 		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
 }
 
@@ -233,7 +205,7 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		return
 	}
 
-	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v --config-file cypress/integration/configurations/Default.json",
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
 		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
 }
 
@@ -279,7 +251,7 @@ func TestTapRegex(t *testing.T) {
 		return
 	}
 
-	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Regex.js\" --env name=%v,namespace=%v --config-file cypress/integration/configurations/Default.json",
+	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Regex.js\" --env name=%v,namespace=%v",
 		expectedPods[0].Name, expectedPods[0].Namespace))
 }
 
@@ -377,7 +349,7 @@ func TestTapRedact(t *testing.T) {
 		}
 	}
 
-	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Redact.js\" --config-file cypress/integration/configurations/Default.json"))
+	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/Redact.js\"")
 }
 
 func TestTapNoRedact(t *testing.T) {
@@ -429,7 +401,7 @@ func TestTapNoRedact(t *testing.T) {
 		}
 	}
 
-	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/NoRedact.js\" --config-file cypress/integration/configurations/Default.json")
+	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/NoRedact.js\"")
 }
 
 func TestTapRegexMasking(t *testing.T) {
@@ -480,7 +452,7 @@ func TestTapRegexMasking(t *testing.T) {
 		}
 	}
 
-	runCypressTests(t, "npx cypress run --spec \"cypress/integration/tests/RegexMasking.js\" --config-file cypress/integration/configurations/Default.json")
+	runCypressTests(t, "npx cypress run --spec \"cypress/integration/tests/RegexMasking.js\"")
 
 }
 
@@ -542,7 +514,7 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 		}
 	}
 
-	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/IgnoredUserAgents.js\" --config-file cypress/integration/configurations/HugeMizu.json")
+	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/IgnoredUserAgents.js\"")
 }
 
 func TestTapDumpLogs(t *testing.T) {

acceptanceTests/testsUtils.go

@@ -3,7 +3,6 @@ package acceptanceTests
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -11,12 +10,10 @@ import (
 	"os/exec"
 	"path"
 	"strings"
-	"sync"
 	"syscall"
 	"testing"
 	"time"
 
-	"github.com/gorilla/websocket"
 	"github.com/up9inc/mizu/shared"
 )
 
@@ -28,7 +25,6 @@ const (
 	defaultServiceName    = "httpbin"
 	defaultEntriesCount   = 50
 	waitAfterTapPodsReady = 3 * time.Second
-	cleanCommandTimeout   = 1 * time.Minute
 )
 
 type PodDescriptor struct {
@@ -36,18 +32,6 @@ type PodDescriptor struct {
 	Namespace string
 }
 
-func isPodDescriptorInPodArray(pods []map[string]interface{}, podDescriptor PodDescriptor) bool {
-	for _, pod := range pods {
-		podNamespace := pod["namespace"].(string)
-		podName := pod["name"].(string)
-
-		if podDescriptor.Namespace == podNamespace && strings.Contains(podName, podDescriptor.Name) {
-			return true
-		}
-	}
-	return false
-}
-
 func getCliPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {
@@ -84,10 +68,6 @@ func getApiServerUrl(port uint16) string {
 	return fmt.Sprintf("http://localhost:%v", port)
 }
 
-func getWebSocketUrl(port uint16) string {
-	return fmt.Sprintf("ws://localhost:%v/ws", port)
-}
-
 func getDefaultCommandArgs() []string {
 	setFlag := "--set"
 	telemetry := "telemetry=false"
@@ -130,20 +110,6 @@ func getDefaultConfigCommandArgs() []string {
 	return append([]string{configCommand}, defaultCmdArgs...)
 }
 
-func getDefaultCleanCommandArgs() []string {
-	cleanCommand := "clean"
-	defaultCmdArgs := getDefaultCommandArgs()
-
-	return append([]string{cleanCommand}, defaultCmdArgs...)
-}
-
-func getDefaultViewCommandArgs() []string {
-	viewCommand := "view"
-	defaultCmdArgs := getDefaultCommandArgs()
-
-	return append([]string{viewCommand}, defaultCmdArgs...)
-}
-
 func runCypressTests(t *testing.T, cypressRunCmd string) {
 	cypressCmd := exec.Command("bash", "-c", cypressRunCmd)
 	t.Logf("running command: %v", cypressCmd.String())
@@ -268,36 +234,6 @@ func executeHttpPostRequestWithHeaders(url string, headers map[string]string, bo
 	return executeHttpRequest(response, requestErr)
 }
 
-func runMizuClean() error {
-	cliPath, err := getCliPath()
-	if err != nil {
-		return err
-	}
-
-	cleanCmdArgs := getDefaultCleanCommandArgs()
-
-	cleanCmd := exec.Command(cliPath, cleanCmdArgs...)
-
-	commandDone := make(chan error)
-	go func() {
-		if err := cleanCmd.Run(); err != nil {
-			commandDone <- err
-		}
-		commandDone <- nil
-	}()
-
-	select {
-	case err = <-commandDone:
-		if err != nil {
-			return err
-		}
-	case <-time.After(cleanCommandTimeout):
-		return errors.New("clean command timed out")
-	}
-
-	return nil
-}
-
 func cleanupCommand(cmd *exec.Cmd) error {
 	if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
 		return err
@@ -310,17 +246,6 @@ func cleanupCommand(cmd *exec.Cmd) error {
 	return nil
 }
 
-func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) {
-	tapPodsInterface := tapStatusInterface.([]interface{})
-
-	var pods []map[string]interface{}
-	for _, podInterface := range tapPodsInterface {
-		pods = append(pods, podInterface.(map[string]interface{}))
-	}
-
-	return pods, nil
-}
-
 func getLogsPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {
@@ -331,77 +256,6 @@ func getLogsPath() (string, error) {
 	return logsPath, nil
 }
 
-// waitTimeout waits for the waitgroup for the specified max timeout.
-// Returns true if waiting timed out.
-func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
-	channel := make(chan struct{})
-	go func() {
-		defer close(channel)
-		wg.Wait()
-	}()
-	select {
-	case <-channel:
-		return false // completed normally
-	case <-time.After(timeout):
-		return true // timed out
-	}
-}
-
-// checkEntriesAtLeast checks whether the number of entries greater than or equal to n
-func checkEntriesAtLeast(entries []map[string]interface{}, n int) error {
-	if len(entries) < n {
-		return fmt.Errorf("Unexpected entries result - Expected more than %d entries", n-1)
-	}
-	return nil
-}
-
-// getDBEntries retrieves the entries from the database before the given timestamp.
-// Also limits the results according to the limit parameter.
-// Timeout for the WebSocket connection is defined by the timeout parameter.
-func getDBEntries(timestamp int64, limit int, timeout time.Duration) (entries []map[string]interface{}, err error) {
-	query := fmt.Sprintf("timestamp < %d and limit(%d)", timestamp, limit)
-	webSocketUrl := getWebSocketUrl(defaultApiServerPort)
-
-	var connection *websocket.Conn
-	connection, _, err = websocket.DefaultDialer.Dial(webSocketUrl, nil)
-	if err != nil {
-		return
-	}
-	defer connection.Close()
-
-	handleWSConnection := func(wg *sync.WaitGroup) {
-		defer wg.Done()
-		for {
-			_, message, err := connection.ReadMessage()
-			if err != nil {
-				return
-			}
-
-			var data map[string]interface{}
-			if err = json.Unmarshal([]byte(message), &data); err != nil {
-				return
-			}
-
-			if data["messageType"] == "entry" {
-				entries = append(entries, data)
-			}
-		}
-	}
-
-	err = connection.WriteMessage(websocket.TextMessage, []byte(query))
-	if err != nil {
-		return
-	}
-
-	var wg sync.WaitGroup
-	go handleWSConnection(&wg)
-	wg.Add(1)
-
-	waitTimeout(&wg, timeout)
-
-	return
-}
-
 func Contains(slice []string, containsValue string) bool {
 	for _, sliceValue := range slice {
 		if sliceValue == containsValue {

agent/go.mod

@@ -6,13 +6,13 @@ require (
 	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
 	github.com/chanced/openapi v0.0.6
 	github.com/djherbis/atime v1.0.0
+	github.com/elastic/go-elasticsearch/v7 v7.16.0
 	github.com/getkin/kin-openapi v0.76.0
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.7.7
 	github.com/go-playground/locales v0.13.0
 	github.com/go-playground/universal-translator v0.17.0
 	github.com/go-playground/validator/v10 v10.5.0
-	github.com/google/martian v2.1.0+incompatible
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/websocket v1.4.2
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7

agent/go.sum

@@ -126,6 +126,8 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/elastic/go-elasticsearch/v7 v7.16.0 h1:GHsxDFXIAlhSleXun4kwA89P7kQFADRChqvgOPeYP5A=
+github.com/elastic/go-elasticsearch/v7 v7.16.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=

agent/main.go

@@ -9,6 +9,7 @@ import (
 	"mizuserver/pkg/api"
 	"mizuserver/pkg/config"
 	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/elastic"
 	"mizuserver/pkg/middlewares"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/oas"
@@ -61,7 +62,6 @@ const (
 	socketConnectionRetries    = 30
 	socketConnectionRetryDelay = time.Second * 2
 	socketHandshakeTimeout     = time.Second * 2
-	uiIndexPath                = "./site/index.html"
 )
 
 func main() {
@@ -159,6 +159,7 @@ func enableExpFeatureIfNeeded() {
 	if config.Config.ServiceMap {
 		servicemap.GetInstance().SetConfig(config.Config)
 	}
+	elastic.GetInstance().Configure(config.Config.Elastic)
 }
 
 func configureBasenineServer(host string, port string) {
@@ -167,7 +168,7 @@ func configureBasenineServer(host string, port string) {
 		wait.WithWait(200*time.Millisecond),
 		wait.WithBreak(50*time.Millisecond),
 		wait.WithDeadline(5*time.Second),
-		wait.WithDebug(true),
+		wait.WithDebug(config.Config.LogLevel == logging.DEBUG),
 	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
 		logger.Log.Panicf("Basenine is not available!")
 	}
@@ -246,16 +247,23 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 
 	app.Use(DisableRootStaticCache())
 
-	if err := setUIFlags(); err != nil {
-		logger.Log.Errorf("Error setting ui mode, err: %v", err)
+	var staticFolder string
+	if config.Config.StandaloneMode {
+		staticFolder = "./site-standalone"
+	} else {
+		staticFolder = "./site"
 	}
 
-	if config.Config.StandaloneMode {
-		app.Use(static.ServeRoot("/", "./site-standalone"))
-	} else {
-		app.Use(static.ServeRoot("/", "./site"))
+	indexStaticFile := staticFolder + "/index.html"
+	if err := setUIFlags(indexStaticFile); err != nil {
+		logger.Log.Errorf("Error setting ui flags, err: %v", err)
 	}
 
+	app.Use(static.ServeRoot("/", staticFolder))
+	app.NoRoute(func(c *gin.Context) {
+		c.File(indexStaticFile)
+	})
+
 	app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
 
 	api.WebSocketRoutes(app, &eventHandlers, startTime)
@@ -276,7 +284,6 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 	routes.EntriesRoutes(app)
 	routes.MetadataRoutes(app)
 	routes.StatusRoutes(app)
-	routes.NotFoundRoute(app)
 	utils.StartServer(app)
 }
 
@@ -291,7 +298,7 @@ func DisableRootStaticCache() gin.HandlerFunc {
 	}
 }
 
-func setUIFlags() error {
+func setUIFlags(uiIndexPath string) error {
 	read, err := ioutil.ReadFile(uiIndexPath)
 	if err != nil {
 		return err

agent/pkg/api/main.go

@@ -5,6 +5,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"mizuserver/pkg/elastic"
+	"mizuserver/pkg/har"
 	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
 	"os"
@@ -15,16 +17,15 @@ import (
 
 	"mizuserver/pkg/servicemap"
 
-	"github.com/google/martian/har"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
-
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/oas"
 	"mizuserver/pkg/resolver"
 	"mizuserver/pkg/utils"
 
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
+
 	basenine "github.com/up9inc/basenine/client/go"
 )
 
@@ -41,10 +42,8 @@ func StartResolving(namespace string) {
 	res.Start(ctx)
 	go func() {
 		for {
-			select {
-			case err := <-errOut:
-				logger.Log.Infof("name resolving error %s", err)
-			}
+			err := <-errOut
+			logger.Log.Infof("name resolving error %s", err)
 		}
 	}()
 
@@ -66,7 +65,7 @@ func startReadingFiles(workingDir string) {
 		return
 	}
 
-	for true {
+	for {
 		dir, _ := os.Open(workingDir)
 		dirFiles, _ := dir.Readdir(-1)
 
@@ -117,15 +116,15 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 	}
 
 	for item := range outputItems {
-		providers.EntryAdded()
-
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
 		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
 				var httpPair tapApi.HTTPRequestResponsePair
-				json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair)
+				if err := json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair); err != nil {
+					logger.Log.Error(err)
+				}
 
 				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
 				mizuEntry.ContractStatus = contract.Status
@@ -134,7 +133,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 				mizuEntry.ContractContent = contract.Content
 			}
 
-			harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
+			harEntry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
 			if err == nil {
 				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
 				mizuEntry.Rules = rules
@@ -147,9 +146,13 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		if err != nil {
 			panic(err)
 		}
+
+		providers.EntryAdded(len(data))
+
 		connection.SendText(string(data))
 
 		servicemap.GetInstance().NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
+		elastic.GetInstance().PushEntry(mizuEntry)
 	}
 }
 

agent/pkg/api/socket_routes.go

@@ -2,7 +2,6 @@ package api
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"mizuserver/pkg/models"
 	"net/http"
@@ -13,7 +12,6 @@ import (
 	"github.com/gorilla/websocket"
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
@@ -42,7 +40,7 @@ var connectedWebsocketIdCounter = 0
 
 func init() {
 	websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
-	connectedWebsockets = make(map[int]*SocketConnection, 0)
+	connectedWebsockets = make(map[int]*SocketConnection)
 }
 
 func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
@@ -94,7 +92,10 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 	eventHandlers.WebSocketConnect(socketId, isTapper)
 
 	startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime)
-	SendToSocket(socketId, startTimeBytes)
+
+	if err = SendToSocket(socketId, startTimeBytes); err != nil {
+		logger.Log.Error(err)
+	}
 
 	for {
 		_, msg, err := ws.ReadMessage()
@@ -117,7 +118,9 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 					AutoClose: 5000,
 					Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
 				})
-				SendToSocket(socketId, toastBytes)
+				if err := SendToSocket(socketId, toastBytes); err != nil {
+					logger.Log.Error(err)
+				}
 				break
 			}
 
@@ -137,7 +140,9 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 					base := tapApi.Summarize(entry)
 
 					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
-					SendToSocket(socketId, baseEntryBytes)
+					if err := SendToSocket(socketId, baseEntryBytes); err != nil {
+						logger.Log.Error(err)
+					}
 				}
 			}
 
@@ -156,7 +161,9 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 					}
 
 					metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
-					SendToSocket(socketId, metadataBytes)
+					if err := SendToSocket(socketId, metadataBytes); err != nil {
+						logger.Log.Error(err)
+					}
 				}
 			}
 
@@ -183,14 +190,10 @@ func socketCleanup(socketId int, socketConnection *SocketConnection) {
 	socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
 }
 
-var db = debounce.NewDebouncer(time.Second*5, func() {
-	logger.Log.Error("Successfully sent to socket")
-})
-
 func SendToSocket(socketId int, message []byte) error {
 	socketObj := connectedWebsockets[socketId]
 	if socketObj == nil {
-		return errors.New("Socket is disconnected")
+		return fmt.Errorf("Socket %v is disconnected", socketId)
 	}
 
 	var sent = false
@@ -204,7 +207,10 @@ func SendToSocket(socketId int, message []byte) error {
 	socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
 	err := socketObj.connection.WriteMessage(1, message)
 	socketObj.lock.Unlock()
-
 	sent = true
-	return err
+
+	if err != nil {
+		return fmt.Errorf("Failed to write message to socket %v, err: %w", socketId, err)
+	}
+	return nil
 }

agent/pkg/api/socket_server_handlers.go

@@ -54,9 +54,8 @@ func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 func BroadcastToBrowserClients(message []byte) {
 	for _, socketId := range browserClientSocketUUIDs {
 		go func(socketId int) {
-			err := SendToSocket(socketId, message)
-			if err != nil {
-				logger.Log.Errorf("error sending message to socket ID %d: %v", socketId, err)
+			if err := SendToSocket(socketId, message); err != nil {
+				logger.Log.Error(err)
 			}
 		}(socketId)
 	}

agent/pkg/controllers/entries_controller.go

@@ -2,8 +2,8 @@ package controllers
 
 import (
 	"encoding/json"
+	"mizuserver/pkg/har"
 	"mizuserver/pkg/models"
-	"mizuserver/pkg/utils"
 	"mizuserver/pkg/validation"
 	"net/http"
 	"strconv"
@@ -26,7 +26,7 @@ func InitExtensionsMap(ref map[string]*tapApi.Extension) {
 func Error(c *gin.Context, err error) bool {
 	if err != nil {
 		logger.Log.Errorf("Error getting entry: %v", err)
-		c.Error(err)
+		_ = c.Error(err)
 		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
 			"error":     true,
 			"type":      "error",
@@ -127,11 +127,13 @@ func GetEntry(c *gin.Context) {
 	var rules []map[string]interface{}
 	var isRulesEnabled bool
 	if entry.Protocol.Name == "http" {
-		harEntry, _ := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+		harEntry, _ := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
 		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
 		isRulesEnabled = _isRulesEnabled
 		inrec, _ := json.Marshal(rulesMatched)
-		json.Unmarshal(inrec, &rules)
+		if err := json.Unmarshal(inrec, &rules); err != nil {
+			logger.Log.Error(err)
+		}
 	}
 
 	c.JSON(http.StatusOK, tapApi.EntryWrapper{

agent/pkg/controllers/oas_controller_test.go

@@ -1,10 +1,11 @@
 package controllers
 
 import (
-	"github.com/gin-gonic/gin"
 	"mizuserver/pkg/oas"
 	"net/http/httptest"
 	"testing"
+
+	"github.com/gin-gonic/gin"
 )
 
 func TestGetOASServers(t *testing.T) {
@@ -15,7 +16,6 @@ func TestGetOASServers(t *testing.T) {
 
 	GetOASServers(c)
 	t.Logf("Written body: %s", recorder.Body.String())
-	return
 }
 
 func TestGetOASAllSpecs(t *testing.T) {
@@ -26,7 +26,6 @@ func TestGetOASAllSpecs(t *testing.T) {
 
 	GetOASAllSpecs(c)
 	t.Logf("Written body: %s", recorder.Body.String())
-	return
 }
 
 func TestGetOASSpec(t *testing.T) {
@@ -39,5 +38,4 @@ func TestGetOASSpec(t *testing.T) {
 
 	GetOASSpec(c)
 	t.Logf("Written body: %s", recorder.Body.String())
-	return
 }

agent/pkg/elastic/esClient.go

@@ -0,0 +1,120 @@
+package elastic
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/tap/api"
+	"net/http"
+	"sync"
+	"time"
+)
+
+type client struct {
+	es            *elasticsearch.Client
+	index         string
+	insertedCount int
+}
+
+var instance *client
+var once sync.Once
+
+func GetInstance() *client {
+	once.Do(func() {
+		instance = newClient()
+	})
+	return instance
+}
+
+func (client *client) Configure(config shared.ElasticConfig) {
+	if config.Url == "" || config.User == "" || config.Password == "" {
+		logger.Log.Infof("No elastic configuration was supplied, elastic exporter disabled")
+		return
+	}
+	transport := http.DefaultTransport
+	tlsClientConfig := &tls.Config{InsecureSkipVerify: true}
+	transport.(*http.Transport).TLSClientConfig = tlsClientConfig
+	cfg := elasticsearch.Config{
+		Addresses: []string{config.Url},
+		Username:  config.User,
+		Password:  config.Password,
+		Transport: transport,
+	}
+
+	es, err := elasticsearch.NewClient(cfg)
+	if err != nil {
+		logger.Log.Fatalf("Failed to initialize elastic client %v", err)
+	}
+
+	// Have the client instance return a response
+	res, err := es.Info()
+	if err != nil {
+		logger.Log.Fatalf("Elastic client.Info() ERROR: %v", err)
+	} else {
+		client.es = es
+		client.index = "mizu_traffic_http_" + time.Now().Format("2006_01_02_15_04")
+		client.insertedCount = 0
+		logger.Log.Infof("Elastic client configured, index: %s, cluster info: %v", client.index, res)
+	}
+	defer res.Body.Close()
+}
+
+func newClient() *client {
+	return &client{
+		es:    nil,
+		index: "",
+	}
+}
+
+type httpEntry struct {
+	Source      *api.TCP               `json:"src"`
+	Destination *api.TCP               `json:"dst"`
+	Outgoing    bool                   `json:"outgoing"`
+	CreatedAt   time.Time              `json:"createdAt"`
+	Request     map[string]interface{} `json:"request"`
+	Response    map[string]interface{} `json:"response"`
+	Summary     string                 `json:"summary"`
+	Method      string                 `json:"method"`
+	Status      int                    `json:"status"`
+	ElapsedTime int64                  `json:"elapsedTime"`
+	Path        string                 `json:"path"`
+}
+
+func (client *client) PushEntry(entry *api.Entry) {
+	if client.es == nil {
+		return
+	}
+
+	if entry.Protocol.Name != "http" {
+		return
+	}
+
+	entryToPush := httpEntry{
+		Source:      entry.Source,
+		Destination: entry.Destination,
+		Outgoing:    entry.Outgoing,
+		CreatedAt:   entry.StartTime,
+		Request:     entry.Request,
+		Response:    entry.Response,
+		Summary:     entry.Summary,
+		Method:      entry.Method,
+		Status:      entry.Status,
+		ElapsedTime: entry.ElapsedTime,
+		Path:        entry.Path,
+	}
+
+	entryJson, err := json.Marshal(entryToPush)
+	if err != nil {
+		logger.Log.Errorf("json.Marshal ERROR: %v", err)
+		return
+	}
+	var buffer bytes.Buffer
+	buffer.WriteString(string(entryJson))
+	res, _ := client.es.Index(client.index, &buffer)
+	if res.StatusCode == 201 {
+		client.insertedCount += 1
+	}
+}

agent/pkg/har/types.go

@@ -0,0 +1,375 @@
+package har
+
+import (
+	"encoding/base64"
+	"github.com/up9inc/mizu/shared/logger"
+	"time"
+	"unicode/utf8"
+)
+
+/*
+HTTP Archive (HAR) format
+https://w3c.github.io/web-performance/specs/HAR/Overview.html
+*/
+
+// HAR is a container type for deserialization
+type HAR struct {
+	Log Log `json:"log"`
+}
+
+// Log represents the root of the exported data. This object MUST be present and its name MUST be "log".
+type Log struct {
+	// The object contains the following name/value pairs:
+
+	// Required. Version number of the format.
+	Version string `json:"version"`
+	// Required. An object of type creator that contains the name and version
+	// information of the log creator application.
+	Creator Creator `json:"creator"`
+	// Optional. An object of type browser that contains the name and version
+	// information of the user agent.
+	Browser Browser `json:"browser"`
+	// Optional. An array of objects of type page, each representing one exported
+	// (tracked) page. Leave out this field if the application does not support
+	// grouping by pages.
+	Pages []Page `json:"pages,omitempty"`
+	// Required. An array of objects of type entry, each representing one
+	// exported (tracked) HTTP request.
+	Entries []Entry `json:"entries"`
+	// Optional. A comment provided by the user or the application. Sorting
+	// entries by startedDateTime (starting from the oldest) is preferred way how
+	// to export data since it can make importing faster. However the reader
+	// application should always make sure the array is sorted (if required for
+	// the import).
+	Comment string `json:"comment"`
+}
+
+// Creator contains information about the log creator application
+type Creator struct {
+	// Required. The name of the application that created the log.
+	Name string `json:"name"`
+	// Required. The version number of the application that created the log.
+	Version string `json:"version"`
+	// Optional. A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Browser that created the log
+type Browser struct {
+	// Required. The name of the browser that created the log.
+	Name string `json:"name"`
+	// Required. The version number of the browser that created the log.
+	Version string `json:"version"`
+	// Optional. A comment provided by the user or the browser.
+	Comment string `json:"comment"`
+}
+
+// Page object for every exported web page and one <entry> object for every HTTP request.
+// In case when an HTTP trace tool isn't able to group requests by a page,
+// the <pages> object is empty and individual requests doesn't have a parent page.
+type Page struct {
+	/* There is one <page> object for every exported web page and one <entry>
+	object for every HTTP request. In case when an HTTP trace tool isn't able to
+	group requests by a page, the <pages> object is empty and individual
+	requests doesn't have a parent page.
+	*/
+
+	// Date and time stamp for the beginning of the page load
+	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00).
+	StartedDateTime string `json:"startedDateTime"`
+	// Unique identifier of a page within the . Entries use it to refer the parent page.
+	ID string `json:"id"`
+	// Page title.
+	Title string `json:"title"`
+	// Detailed timing info about page load.
+	PageTiming PageTiming `json:"pageTiming"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// PageTiming describes timings for various events (states) fired during the page load.
+// All times are specified in milliseconds. If a time info is not available appropriate field is set to -1.
+type PageTiming struct {
+	// Content of the page loaded. Number of milliseconds since page load started
+	// (page.startedDateTime). Use -1 if the timing does not apply to the current
+	// request.
+	// Depeding on the browser, onContentLoad property represents DOMContentLoad
+	// event or document.readyState == interactive.
+	OnContentLoad int `json:"onContentLoad"`
+	// Page is loaded (onLoad event fired). Number of milliseconds since page
+	// load started (page.startedDateTime). Use -1 if the timing does not apply
+	// to the current request.
+	OnLoad int `json:"onLoad"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment"`
+}
+
+// Entry is a unique, optional Reference to the parent page.
+// Leave out this field if the application does not support grouping by pages.
+type Entry struct {
+	Pageref string `json:"pageref,omitempty"`
+	// Date and time stamp of the request start
+	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD).
+	StartedDateTime string `json:"startedDateTime"`
+	// Total elapsed time of the request in milliseconds. This is the sum of all
+	// timings available in the timings object (i.e. not including -1 values) .
+	Time int `json:"time"`
+	// Detailed info about the request.
+	Request Request `json:"request"`
+	// Detailed info about the response.
+	Response Response `json:"response"`
+	// Info about cache usage.
+	Cache Cache `json:"cache"`
+	// Detailed timing info about request/response round trip.
+	PageTimings PageTimings `json:"pageTimings"`
+	// optional (new in 1.2) IP address of the server that was connected
+	// (result of DNS resolution).
+	ServerIPAddress string `json:"serverIPAddress,omitempty"`
+	// optional (new in 1.2) Unique ID of the parent TCP/IP connection, can be
+	// the client port number. Note that a port number doesn't have to be unique
+	// identifier in cases where the port is shared for more connections. If the
+	// port isn't available for the application, any other unique connection ID
+	// can be used instead (e.g. connection index). Leave out this field if the
+	// application doesn't support this info.
+	Connection string `json:"connection,omitempty"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Request contains detailed info about performed request.
+type Request struct {
+	// Request method (GET, POST, ...).
+	Method string `json:"method"`
+	// Absolute URL of the request (fragments are not included).
+	URL string `json:"url"`
+	// Request HTTP Version.
+	HTTPVersion string `json:"httpVersion"`
+	// List of cookie objects.
+	Cookies []Cookie `json:"cookies"`
+	// List of header objects.
+	Headers []NVP `json:"headers"`
+	// List of query parameter objects.
+	QueryString []NVP `json:"queryString"`
+	// Posted data.
+	PostData PostData `json:"postData"`
+	// Total number of bytes from the start of the HTTP request message until
+	// (and including) the double CRLF before the body. Set to -1 if the info
+	// is not available.
+	HeaderSize int `json:"headerSize"`
+	// Size of the request body (POST data payload) in bytes. Set to -1 if the
+	// info is not available.
+	BodySize int `json:"bodySize"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment"`
+}
+
+// Response contains detailed info about the response.
+type Response struct {
+	// Response status.
+	Status int `json:"status"`
+	// Response status description.
+	StatusText string `json:"statusText"`
+	// Response HTTP Version.
+	HTTPVersion string `json:"httpVersion"`
+	// List of cookie objects.
+	Cookies []Cookie `json:"cookies"`
+	// List of header objects.
+	Headers []NVP `json:"headers"`
+	// Details about the response body.
+	Content Content `json:"content"`
+	// Redirection target URL from the Location response header.
+	RedirectURL string `json:"redirectURL"`
+	// Total number of bytes from the start of the HTTP response message until
+	// (and including) the double CRLF before the body. Set to -1 if the info is
+	// not available.
+	// The size of received response-headers is computed only from headers that
+	// are really received from the server. Additional headers appended by the
+	// browser are not included in this number, but they appear in the list of
+	// header objects.
+	HeadersSize int `json:"headersSize"`
+	// Size of the received response body in bytes. Set to zero in case of
+	// responses coming from the cache (304). Set to -1 if the info is not
+	// available.
+	BodySize int `json:"bodySize"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Cookie contains list of all cookies (used in <request> and <response> objects).
+type Cookie struct {
+	// The name of the cookie.
+	Name string `json:"name"`
+	// The cookie value.
+	Value string `json:"value"`
+	// optional The path pertaining to the cookie.
+	Path string `json:"path,omitempty"`
+	// optional The host of the cookie.
+	Domain string `json:"domain,omitempty"`
+	// optional Cookie expiration time.
+	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.123+02:00).
+	Expires string `json:"expires,omitempty"`
+	// optional Set to true if the cookie is HTTP only, false otherwise.
+	HTTPOnly bool `json:"httpOnly,omitempty"`
+	// optional (new in 1.2) True if the cookie was transmitted over ssl, false
+	// otherwise.
+	Secure bool `json:"secure,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// NVP is simply a name/value pair with a comment
+type NVP struct {
+	Name    string `json:"name"`
+	Value   string `json:"value"`
+	Comment string `json:"comment,omitempty"`
+}
+
+// PostData describes posted data, if any (embedded in <request> object).
+type PostData struct {
+	//  Mime type of posted data.
+	MimeType string `json:"mimeType"`
+	//  List of posted parameters (in case of URL encoded parameters).
+	Params []PostParam `json:"params"`
+	//  Plain text posted data
+	Text string `json:"text"`
+	// optional (new in 1.2) A comment provided by the user or the
+	// application.
+	Comment string `json:"comment,omitempty"`
+}
+
+func (d PostData) B64Decoded() (bool, []byte, string) {
+	// there is a weird gap in HAR spec 1.2, that does not define encoding for binary POST bodies
+	// we have own convention of putting `base64` into comment field to handle it similar to response `Content`
+	return b64Decoded(d.Comment, d.Text)
+}
+
+// PostParam is a list of posted parameters, if any (embedded in <postData> object).
+type PostParam struct {
+	// name of a posted parameter.
+	Name string `json:"name"`
+	// optional value of a posted parameter or content of a posted file.
+	Value string `json:"value,omitempty"`
+	// optional name of a posted file.
+	FileName string `json:"fileName,omitempty"`
+	// optional content type of a posted file.
+	ContentType string `json:"contentType,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Content describes details about response content (embedded in <response> object).
+type Content struct {
+	// Length of the returned content in bytes. Should be equal to
+	// response.bodySize if there is no compression and bigger when the content
+	// has been compressed.
+	Size int `json:"size"`
+	// optional Number of bytes saved. Leave out this field if the information
+	// is not available.
+	Compression int `json:"compression,omitempty"`
+	// MIME type of the response text (value of the Content-Type response
+	// header). The charset attribute of the MIME type is included (if
+	// available).
+	MimeType string `json:"mimeType"`
+	// optional Response body sent from the server or loaded from the browser
+	// cache. This field is populated with textual content only. The text field
+	// is either HTTP decoded text or a encoded (e.g. "base64") representation of
+	// the response body. Leave out this field if the information is not
+	// available.
+	Text string `json:"text,omitempty"`
+	// optional (new in 1.2) Encoding used for response text field e.g
+	// "base64". Leave out this field if the text field is HTTP decoded
+	// (decompressed & unchunked), than trans-coded from its original character
+	// set into UTF-8.
+	Encoding string `json:"encoding,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+func (c Content) B64Decoded() (bool, []byte, string) {
+	return b64Decoded(c.Encoding, c.Text)
+}
+
+func b64Decoded(enc string, text string) (isBinary bool, asBytes []byte, asString string) {
+	if enc == "base64" {
+		decoded, err := base64.StdEncoding.DecodeString(text)
+		if err != nil {
+			logger.Log.Warningf("Failed to decode content as base64: %s", text)
+			return false, []byte(text), text
+		}
+		valid := utf8.Valid(decoded)
+		return !valid, decoded, string(decoded)
+	} else {
+		return false, nil, text
+	}
+}
+
+// Cache contains info about a request coming from browser cache.
+type Cache struct {
+	// optional State of a cache entry before the request. Leave out this field
+	// if the information is not available.
+	BeforeRequest CacheObject `json:"beforeRequest,omitempty"`
+	// optional State of a cache entry after the request. Leave out this field if
+	// the information is not available.
+	AfterRequest CacheObject `json:"afterRequest,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// CacheObject is used by both beforeRequest and afterRequest
+type CacheObject struct {
+	// optional - Expiration time of the cache entry.
+	Expires string `json:"expires,omitempty"`
+	// The last time the cache entry was opened.
+	LastAccess string `json:"lastAccess"`
+	// Etag
+	ETag string `json:"eTag"`
+	// The number of times the cache entry has been opened.
+	HitCount int `json:"hitCount"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// PageTimings describes various phases within request-response round trip.
+// All times are specified in milliseconds.
+type PageTimings struct {
+	Blocked int `json:"blocked,omitempty"`
+	// optional - Time spent in a queue waiting for a network connection. Use -1
+	// if the timing does not apply to the current request.
+	DNS int `json:"dns,omitempty"`
+	// optional - DNS resolution time. The time required to resolve a host name.
+	// Use -1 if the timing does not apply to the current request.
+	Connect int `json:"connect,omitempty"`
+	// optional - Time required to create TCP connection. Use -1 if the timing
+	// does not apply to the current request.
+	Send int `json:"send"`
+	// Time required to send HTTP request to the server.
+	Wait int `json:"wait"`
+	// Waiting for a response from the server.
+	Receive int `json:"receive"`
+	// Time required to read entire response from the server (or cache).
+	Ssl int `json:"ssl,omitempty"`
+	// optional (new in 1.2) - Time required for SSL/TLS negotiation. If this
+	// field is defined then the time is also included in the connect field (to
+	// ensure backward compatibility with HAR 1.1). Use -1 if the timing does not
+	// apply to the current request.
+	Comment string `json:"comment,omitempty"`
+	// optional (new in 1.2) - A comment provided by the user or the application.
+}
+
+// TestResult contains results for an individual HTTP request
+type TestResult struct {
+	URL       string    `json:"url"`
+	Status    int       `json:"status"` // 200, 500, etc.
+	StartTime time.Time `json:"startTime"`
+	EndTime   time.Time `json:"endTime"`
+	Latency   int       `json:"latency"` // milliseconds
+	Method    string    `json:"method"`
+	HarFile   string    `json:"harfile"`
+}
+
+// aliases for martian lib compatibility
+
+type Header = NVP
+type QueryString = NVP
+type Param = PostParam
+type Timings = PageTimings

agent/pkg/har/types_test.go

@@ -0,0 +1,38 @@
+package har
+
+import "testing"
+
+func TestContentEncoded(t *testing.T) {
+	testCases := []struct {
+		text        string
+		isBinary    bool
+		expectedStr string
+		binaryLen   int
+	}{
+		{"not-base64", false, "not-base64", 10},
+		{"dGVzdA==", false, "test", 4},
+		{"test", true, "\f@A", 3}, // valid UTF-8 with some non-printable chars
+		{"IsDggPCAgPiAgID8gICAgN/vv/e/v/u/v7/9v7+/vyIKIu+3kO+3ke+3ku+3k++3lO+3le+3lu+3l++3mO+3me+3mu+3m++3nO+3ne+3nu+3n++3oO+3oe+3ou+3o++3pO+3pe+3pu+3p++3qO+3qe+3qu+3q++3rO+3re+3ru+3ryIK", true, "test", 132}, // invalid UTF-8 (thus binary), taken from https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
+	}
+
+	for _, tc := range testCases {
+		c := Content{
+			Encoding: "base64",
+			Text:     tc.text,
+		}
+		isBinary, asBytes, asString := c.B64Decoded()
+		_ = asBytes
+
+		if tc.isBinary != isBinary {
+			t.Errorf("Binary flag mismatch: %t != %t", tc.isBinary, isBinary)
+		}
+
+		if !isBinary && tc.expectedStr != asString {
+			t.Errorf("Decode value mismatch: %s != %s", tc.expectedStr, asString)
+		}
+
+		if tc.binaryLen != len(asBytes) {
+			t.Errorf("Binary len mismatch: %d != %d", tc.binaryLen, len(asBytes))
+		}
+	}
+}

agent/pkg/har/utils.go

@@ -1,4 +1,4 @@
-package utils
+package har
 
 import (
 	"bytes"
@@ -8,7 +8,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
 )
 
@@ -55,14 +54,14 @@ import (
 //	return cookies
 //}
 
-func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, string, string, string) {
+func BuildHeaders(rawHeaders []interface{}) ([]Header, string, string, string, string, string) {
 	var host, scheme, authority, path, status string
-	headers := make([]har.Header, 0, len(rawHeaders))
+	headers := make([]Header, 0, len(rawHeaders))
 
 	for _, header := range rawHeaders {
 		h := header.(map[string]interface{})
 
-		headers = append(headers, har.Header{
+		headers = append(headers, Header{
 			Name:  h["name"].(string),
 			Value: h["value"].(string),
 		})
@@ -87,8 +86,8 @@ func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, strin
 	return headers, host, scheme, authority, path, status
 }
 
-func BuildPostParams(rawParams []interface{}) []har.Param {
-	params := make([]har.Param, 0, len(rawParams))
+func BuildPostParams(rawParams []interface{}) []Param {
+	params := make([]Param, 0, len(rawParams))
 	for _, param := range rawParams {
 		p := param.(map[string]interface{})
 		name := ""
@@ -108,10 +107,10 @@ func BuildPostParams(rawParams []interface{}) []har.Param {
 			contentType = p["contentType"].(string)
 		}
 
-		params = append(params, har.Param{
+		params = append(params, Param{
 			Name:        name,
 			Value:       value,
-			Filename:    fileName,
+			FileName:    fileName,
 			ContentType: contentType,
 		})
 	}
@@ -119,25 +118,25 @@ func BuildPostParams(rawParams []interface{}) []har.Param {
 	return params
 }
 
-func NewRequest(request map[string]interface{}) (harRequest *har.Request, err error) {
+func NewRequest(request map[string]interface{}) (harRequest *Request, err error) {
 	headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
+	cookies := make([]Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
 
 	postData, _ := request["postData"].(map[string]interface{})
-	mimeType, _ := postData["mimeType"]
+	mimeType := postData["mimeType"]
 	if mimeType == nil || len(mimeType.(string)) == 0 {
 		mimeType = "text/html"
 	}
-	text, _ := postData["text"]
+	text := postData["text"]
 	postDataText := ""
 	if text != nil {
 		postDataText = text.(string)
 	}
 
-	queryString := make([]har.QueryString, 0)
+	queryString := make([]QueryString, 0)
 	for _, _qs := range request["_queryString"].([]interface{}) {
 		qs := _qs.(map[string]interface{})
-		queryString = append(queryString, har.QueryString{
+		queryString = append(queryString, QueryString{
 			Name:  qs["name"].(string),
 			Value: qs["value"].(string),
 		})
@@ -148,21 +147,21 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er
 		url = fmt.Sprintf("%s://%s%s", scheme, authority, path)
 	}
 
-	harParams := make([]har.Param, 0)
+	harParams := make([]Param, 0)
 	if postData["params"] != nil {
 		harParams = BuildPostParams(postData["params"].([]interface{}))
 	}
 
-	harRequest = &har.Request{
+	harRequest = &Request{
 		Method:      request["method"].(string),
 		URL:         url,
 		HTTPVersion: request["httpVersion"].(string),
-		HeadersSize: -1,
-		BodySize:    int64(bytes.NewBufferString(postDataText).Len()),
+		HeaderSize:  -1,
+		BodySize:    bytes.NewBufferString(postDataText).Len(),
 		QueryString: queryString,
 		Headers:     headers,
 		Cookies:     cookies,
-		PostData: &har.PostData{
+		PostData: PostData{
 			MimeType: mimeType.(string),
 			Params:   harParams,
 			Text:     postDataText,
@@ -172,27 +171,27 @@ func NewRequest(request map[string]interface{}) (harRequest *har.Request, err er
 	return
 }
 
-func NewResponse(response map[string]interface{}) (harResponse *har.Response, err error) {
+func NewResponse(response map[string]interface{}) (harResponse *Response, err error) {
 	headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{}))
-	cookies := make([]har.Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
+	cookies := make([]Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
 
 	content, _ := response["content"].(map[string]interface{})
-	mimeType, _ := content["mimeType"]
+	mimeType := content["mimeType"]
 	if mimeType == nil || len(mimeType.(string)) == 0 {
 		mimeType = "text/html"
 	}
-	encoding, _ := content["encoding"]
-	text, _ := content["text"]
+	encoding := content["encoding"]
+	text := content["text"]
 	bodyText := ""
 	if text != nil {
 		bodyText = text.(string)
 	}
 
-	harContent := &har.Content{
+	harContent := &Content{
 		Encoding: encoding.(string),
 		MimeType: mimeType.(string),
-		Text:     []byte(bodyText),
-		Size:     int64(len(bodyText)),
+		Text:     bodyText,
+		Size:     len(bodyText),
 	}
 
 	status := int(response["status"].(float64))
@@ -206,20 +205,20 @@ func NewResponse(response map[string]interface{}) (harResponse *har.Response, er
 		}
 	}
 
-	harResponse = &har.Response{
+	harResponse = &Response{
 		HTTPVersion: response["httpVersion"].(string),
 		Status:      status,
 		StatusText:  response["statusText"].(string),
 		HeadersSize: -1,
-		BodySize:    int64(bytes.NewBufferString(bodyText).Len()),
+		BodySize:    bytes.NewBufferString(bodyText).Len(),
 		Headers:     headers,
 		Cookies:     cookies,
-		Content:     harContent,
+		Content:     *harContent,
 	}
 	return
 }
 
-func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*har.Entry, error) {
+func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*Entry, error) {
 	harRequest, err := NewRequest(request)
 	if err != nil {
 		logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
@@ -236,16 +235,16 @@ func NewEntry(request map[string]interface{}, response map[string]interface{}, s
 		elapsedTime = 1
 	}
 
-	harEntry := har.Entry{
-		StartedDateTime: startTime,
-		Time:            elapsedTime,
-		Request:         harRequest,
-		Response:        harResponse,
-		Cache:           &har.Cache{},
-		Timings: &har.Timings{
+	harEntry := Entry{
+		StartedDateTime: startTime.Format(time.RFC3339),
+		Time:            int(elapsedTime),
+		Request:         *harRequest,
+		Response:        *harResponse,
+		Cache:           Cache{},
+		PageTimings: PageTimings{
 			Send:    -1,
 			Wait:    -1,
-			Receive: elapsedTime,
+			Receive: int(elapsedTime),
 		},
 	}
 

agent/pkg/models/models.go

@@ -2,11 +2,10 @@ package models
 
 import (
 	"encoding/json"
+	tapApi "github.com/up9inc/mizu/tap/api"
+	"mizuserver/pkg/har"
 	"mizuserver/pkg/rules"
 
-	tapApi "github.com/up9inc/mizu/tap/api"
-
-	"github.com/google/martian/har"
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"

agent/pkg/oas/feeder_test.go

@@ -4,19 +4,20 @@ import (
 	"bufio"
 	"encoding/json"
 	"errors"
-	"github.com/google/martian/har"
-	"github.com/up9inc/mizu/shared/logger"
 	"io"
 	"io/ioutil"
+	"mizuserver/pkg/har"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"testing"
+
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func getFiles(baseDir string) (result []string, err error) {
-	result = make([]string, 0, 0)
+	result = make([]string, 0)
 	logger.Log.Infof("Reading files from tree: %s", baseDir)
 
 	// https://yourbasic.org/golang/list-files-in-directory/
@@ -51,32 +52,42 @@ func fileSize(fname string) int64 {
 	return fi.Size()
 }
 
-func feedEntries(fromFiles []string) (err error) {
+func feedEntries(fromFiles []string, isSync bool) (count int, err error) {
+	badFiles := make([]string, 0)
+	cnt := 0
 	for _, file := range fromFiles {
 		logger.Log.Info("Processing file: " + file)
 		ext := strings.ToLower(filepath.Ext(file))
+		eCnt := 0
 		switch ext {
 		case ".har":
-			err = feedFromHAR(file)
+			eCnt, err = feedFromHAR(file, isSync)
 			if err != nil {
 				logger.Log.Warning("Failed processing file: " + err.Error())
+				badFiles = append(badFiles, file)
 				continue
 			}
 		case ".ldjson":
-			err = feedFromLDJSON(file)
+			eCnt, err = feedFromLDJSON(file, isSync)
 			if err != nil {
 				logger.Log.Warning("Failed processing file: " + err.Error())
+				badFiles = append(badFiles, file)
 				continue
 			}
 		default:
-			return errors.New("Unsupported file extension: " + ext)
+			return 0, errors.New("Unsupported file extension: " + ext)
 		}
+		cnt += eCnt
 	}
 
-	return nil
+	for _, f := range badFiles {
+		logger.Log.Infof("Bad file: %s", f)
+	}
+
+	return cnt, nil
 }
 
-func feedFromHAR(file string) error {
+func feedFromHAR(file string, isSync bool) (int, error) {
 	fd, err := os.Open(file)
 	if err != nil {
 		panic(err)
@@ -86,23 +97,41 @@ func feedFromHAR(file string) error {
 
 	data, err := ioutil.ReadAll(fd)
 	if err != nil {
-		return err
+		return 0, err
 	}
 
 	var harDoc har.HAR
 	err = json.Unmarshal(data, &harDoc)
 	if err != nil {
-		return err
+		return 0, err
 	}
 
+	cnt := 0
 	for _, entry := range harDoc.Log.Entries {
-		GetOasGeneratorInstance().PushEntry(entry)
+		cnt += 1
+		feedEntry(&entry, isSync)
 	}
 
-	return nil
+	return cnt, nil
 }
 
-func feedFromLDJSON(file string) error {
+func feedEntry(entry *har.Entry, isSync bool) {
+	if entry.Response.Status == 302 {
+		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
+	}
+
+	if strings.Contains(entry.Request.URL, "taboola") {
+		logger.Log.Debugf("Interesting: %s", entry.Request.URL)
+	}
+
+	if isSync {
+		GetOasGeneratorInstance().entriesChan <- *entry // blocking variant, right?
+	} else {
+		GetOasGeneratorInstance().PushEntry(entry)
+	}
+}
+
+func feedFromLDJSON(file string, isSync bool) (int, error) {
 	fd, err := os.Open(file)
 	if err != nil {
 		panic(err)
@@ -113,8 +142,8 @@ func feedFromLDJSON(file string) error {
 	reader := bufio.NewReader(fd)
 
 	var meta map[string]interface{}
-
 	buf := strings.Builder{}
+	cnt := 0
 	for {
 		substr, isPrefix, err := reader.ReadLine()
 		if err == io.EOF {
@@ -132,26 +161,28 @@ func feedFromLDJSON(file string) error {
 		if meta == nil {
 			err := json.Unmarshal([]byte(line), &meta)
 			if err != nil {
-				return err
+				return 0, err
 			}
 		} else {
 			var entry har.Entry
 			err := json.Unmarshal([]byte(line), &entry)
 			if err != nil {
 				logger.Log.Warningf("Failed decoding entry: %s", line)
+			} else {
+				cnt += 1
+				feedEntry(&entry, isSync)
 			}
-			GetOasGeneratorInstance().PushEntry(&entry)
 		}
 	}
 
-	return nil
+	return cnt, nil
 }
 
 func TestFilesList(t *testing.T) {
 	res, err := getFiles("./test_artifacts/")
 	t.Log(len(res))
 	t.Log(res)
-	if err != nil || len(res) != 2 {
+	if err != nil || len(res) != 3 {
 		t.Logf("Should return 2 files but returned %d", len(res))
 		t.FailNow()
 	}

agent/pkg/oas/gibberish.go

@@ -12,6 +12,7 @@ var (
 	patEmail    = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
 	patHexLower = regexp.MustCompile(`(0x)?[0-9a-f]{6,}`)
 	patHexUpper = regexp.MustCompile(`(0x)?[0-9A-F]{6,}`)
+	patLongNum  = regexp.MustCompile(`\d{6,}`)
 )
 
 func IsGibberish(str string) bool {
@@ -27,16 +28,12 @@ func IsGibberish(str string) bool {
 		return true
 	}
 
-	if patHexLower.MatchString(str) || patHexUpper.MatchString(str) {
+	if patHexLower.MatchString(str) || patHexUpper.MatchString(str) || patLongNum.MatchString(str) {
 		return true
 	}
 
 	noise := noiseLevel(str)
-	if noise >= 0.2 {
-		return true
-	}
-
-	return false
+	return noise >= 0.2
 }
 
 func noiseLevel(str string) (score float64) {

agent/pkg/oas/ignores.go

@@ -17,17 +17,19 @@ var ignoredHeaders = []string{
 	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data",
 	"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
 	"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
-	"x-wap-profile", "x-scheme",
-	"newrelic", "x-cloud-trace-context", "sentry-trace",
-	"expires", "set-cookie", "p3p", "location", "content-security-policy", "content-security-policy-report-only",
-	"last-modified", "content-language",
+	"x-wap-profile", "x-scheme", "status", "x-cache", "x-application-context", "retry-after",
+	"newrelic", "x-cloud-trace-context", "sentry-trace", "x-cache-hits", "x-served-by", "x-span-name",
+	"expires", "set-cookie", "p3p", "content-security-policy", "content-security-policy-report-only",
+	"last-modified", "content-language", "x-varnish", "true-client-ip", "akamai-origin-hop",
 	"keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by",
-	"age", "allow", "www-authenticate",
+	"age", "allow", "www-authenticate", "expect-ct", "timing-allow-origin", "referrer-policy",
+	"x-aspnet-version", "x-aspnetmvc-version", "x-timer", "x-abuse-info", "x-mod-pagespeed",
+	"duration_ms", // UP9 custom
 }
 
 var ignoredHeaderPrefixes = []string{
 	":", "accept-", "access-control-", "if-", "sec-", "grpc-",
-	"x-forwarded-", "x-original-",
+	"x-forwarded-", "x-original-", "cf-",
 	"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
 	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-",
 }

agent/pkg/oas/oas_generator.go

@@ -3,8 +3,8 @@ package oas
 import (
 	"context"
 	"encoding/json"
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/har"
 	"net/url"
 	"sync"
 )
@@ -17,7 +17,7 @@ var (
 func GetOasGeneratorInstance() *oasGenerator {
 	syncOnce.Do(func() {
 		instance = newOasGenerator()
-		logger.Log.Debug("Oas Generator Initialized")
+		logger.Log.Debug("OAS Generator Initialized")
 	})
 	return instance
 }

agent/pkg/oas/specgen.go

@@ -1,18 +1,18 @@
 package oas
 
 import (
-	"encoding/base64"
 	"encoding/json"
 	"errors"
-	"github.com/chanced/openapi"
-	"github.com/google/martian/har"
-	"github.com/google/uuid"
-	"github.com/up9inc/mizu/shared/logger"
 	"mime"
+	"mizuserver/pkg/har"
 	"net/url"
 	"strconv"
 	"strings"
 	"sync"
+
+	"github.com/chanced/openapi"
+	"github.com/google/uuid"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 type reqResp struct { // hello, generics in Go
@@ -31,7 +31,7 @@ func NewGen(server string) *SpecGen {
 	spec.Version = "3.1.0"
 
 	info := openapi.Info{Title: server}
-	info.Version = "0.0"
+	info.Version = "1.0"
 	spec.Info = &info
 	spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
 
@@ -176,11 +176,18 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
 
 	if isExtIgnored(urlParsed.Path) {
 		logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
+		return "", nil
 	}
 
-	ctype := getRespCtype(entry.Response)
+	if entry.Request.Method == "OPTIONS" {
+		logger.Log.Debugf("Dropped traffic entry due to its method: %s", urlParsed.Path)
+		return "", nil
+	}
+
+	ctype := getRespCtype(&entry.Response)
 	if isCtypeIgnored(ctype) {
 		logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
+		return "", nil
 	}
 
 	if entry.Response.Status < 100 {
@@ -193,9 +200,19 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
 		return "", nil
 	}
 
-	split := strings.Split(urlParsed.Path, "/")
+	if entry.Response.Status == 502 || entry.Response.Status == 503 || entry.Response.Status == 504 {
+		logger.Log.Debugf("Dropped traffic entry due to temporary server error: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	var split []string
+	if urlParsed.RawPath != "" {
+		split = strings.Split(urlParsed.RawPath, "/")
+	} else {
+		split = strings.Split(urlParsed.Path, "/")
+	}
 	node := g.tree.getOrSet(split, new(openapi.PathObj))
-	opObj, err := handleOpObj(entry, node.ops)
+	opObj, err := handleOpObj(entry, node.pathObj)
 
 	if opObj != nil {
 		return opObj.OperationID, err
@@ -216,12 +233,12 @@ func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation
 		return nil, nil
 	}
 
-	err = handleRequest(entry.Request, opObj, isSuccess)
+	err = handleRequest(&entry.Request, opObj, isSuccess)
 	if err != nil {
 		return nil, err
 	}
 
-	err = handleResponse(entry.Response, opObj, isSuccess)
+	err = handleResponse(&entry.Response, opObj, isSuccess)
 	if err != nil {
 		return nil, err
 	}
@@ -233,26 +250,22 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e
 	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
 
 	qstrGW := nvParams{
-		In: openapi.InQuery,
-		Pairs: func() []NVPair {
-			return qstrToNVP(req.QueryString)
-		},
+		In:             openapi.InQuery,
+		Pairs:          req.QueryString,
 		IsIgnored:      func(name string) bool { return false },
 		GeneralizeName: func(name string) string { return name },
 	}
 	handleNameVals(qstrGW, &opObj.Parameters)
 
 	hdrGW := nvParams{
-		In: openapi.InHeader,
-		Pairs: func() []NVPair {
-			return hdrToNVP(req.Headers)
-		},
+		In:             openapi.InHeader,
+		Pairs:          req.Headers,
 		IsIgnored:      isHeaderIgnored,
 		GeneralizeName: strings.ToLower,
 	}
 	handleNameVals(hdrGW, &opObj.Parameters)
 
-	if req.PostData != nil && req.PostData.Text != "" && isSuccess {
+	if req.PostData.Text != "" && isSuccess {
 		reqBody, err := getRequestBody(req, opObj, isSuccess)
 		if err != nil {
 			return err
@@ -260,7 +273,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e
 
 		if reqBody != nil {
 			reqCtype := getReqCtype(req)
-			reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, err)
+			reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype)
 			if err != nil {
 				return err
 			}
@@ -282,7 +295,7 @@ func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool
 
 	respCtype := getRespCtype(resp)
 	respContent := respObj.Content
-	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, err)
+	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype)
 	if err != nil {
 		return err
 	}
@@ -330,11 +343,9 @@ func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj) {
 			}
 		}
 	}
-
-	return
 }
 
-func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err error) (*openapi.MediaType, error) {
+func fillContent(reqResp reqResp, respContent openapi.Content, ctype string) (*openapi.MediaType, error) {
 	content, found := respContent[ctype]
 	if !found {
 		respContent[ctype] = &openapi.MediaType{}
@@ -342,43 +353,36 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err
 	}
 
 	var text string
+	var isBinary bool
 	if reqResp.Req != nil {
-		text = reqResp.Req.PostData.Text
+		isBinary, _, text = reqResp.Req.PostData.B64Decoded()
 	} else {
-		text = decRespText(reqResp.Resp.Content)
+		isBinary, _, text = reqResp.Resp.Content.B64Decoded()
 	}
 
-	var exampleMsg []byte
-	// try treating it as json
-	any, isJSON := anyJSON(text)
-	if isJSON {
-		// re-marshal with forced indent
-		exampleMsg, err = json.MarshalIndent(any, "", "\t")
-		if err != nil {
-			panic("Failed to re-marshal value, super-strange")
-		}
-	} else {
-		exampleMsg, err = json.Marshal(text)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	content.Example = exampleMsg
-	return respContent[ctype], nil
-}
-
-func decRespText(content *har.Content) (res string) {
-	res = string(content.Text)
-	if content.Encoding == "base64" {
-		data, err := base64.StdEncoding.DecodeString(res)
-		if err != nil {
-			logger.Log.Warningf("error decoding response text as base64: %s", err)
+	if !isBinary && text != "" {
+		var exampleMsg []byte
+		// try treating it as json
+		any, isJSON := anyJSON(text)
+		if isJSON {
+			// re-marshal with forced indent
+			if msg, err := json.MarshalIndent(any, "", "\t"); err != nil {
+				panic("Failed to re-marshal value, super-strange")
+			} else {
+				exampleMsg = msg
+			}
 		} else {
-			res = string(data)
+			if msg, err := json.Marshal(text); err != nil {
+				return nil, err
+			} else {
+				exampleMsg = msg
+			}
 		}
+
+		content.Example = exampleMsg
 	}
-	return
+
+	return respContent[ctype], nil
 }
 
 func getRespCtype(resp *har.Response) string {

agent/pkg/oas/specgen_test.go

@@ -3,48 +3,56 @@ package oas
 import (
 	"encoding/json"
 	"github.com/chanced/openapi"
-	"github.com/google/martian/har"
+	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/shared/logger"
 	"io/ioutil"
+	"mizuserver/pkg/har"
 	"os"
 	"strings"
 	"testing"
+	"time"
 )
 
 // if started via env, write file into subdir
-func writeFiles(label string, spec *openapi.OpenAPI) {
+func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) {
+	content, err := json.MarshalIndent(spec, "", "\t")
+	if err != nil {
+		panic(err)
+	}
+
 	if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
 		path := "./oas-samples"
 		err := os.MkdirAll(path, 0o755)
 		if err != nil {
 			panic(err)
 		}
-		content, err := json.MarshalIndent(spec, "", "\t")
-		if err != nil {
-			panic(err)
-		}
 		err = ioutil.WriteFile(path+"/"+label+".json", content, 0644)
 		if err != nil {
 			panic(err)
 		}
+		t.Logf("Written: %s", label)
+	} else {
+		t.Logf("%s", string(content))
 	}
 }
 
 func TestEntries(t *testing.T) {
+	logger.InitLoggerStderrOnly(logging.INFO)
 	files, err := getFiles("./test_artifacts/")
-	// files, err = getFiles("/media/bigdisk/UP9")
 	if err != nil {
 		t.Log(err)
 		t.FailNow()
 	}
 	GetOasGeneratorInstance().Start()
+	loadStartingOAS()
 
-	if err := feedEntries(files); err != nil {
+	cnt, err := feedEntries(files, true)
+	if err != nil {
 		t.Log(err)
 		t.Fail()
 	}
 
-	loadStartingOAS()
+	waitQueueProcessed()
 
 	svcs := strings.Builder{}
 	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
@@ -78,33 +86,35 @@ func TestEntries(t *testing.T) {
 			t.FailNow()
 		}
 
-		specText, _ := json.MarshalIndent(spec, "", "\t")
-		t.Logf("%s", string(specText))
+		outputSpec(svc, spec, t)
 
 		err = spec.Validate()
 		if err != nil {
 			t.Log(err)
 			t.FailNow()
 		}
-		writeFiles(svc, spec)
 
 		return true
 	})
 
+	logger.Log.Infof("Total entries: %d", cnt)
 }
 
-func TestFileLDJSON(t *testing.T) {
+func TestFileSingle(t *testing.T) {
 	GetOasGeneratorInstance().Start()
-	file := "test_artifacts/output_rdwtyeoyrj.har.ldjson"
-	err := feedFromLDJSON(file)
+	// loadStartingOAS()
+	file := "test_artifacts/params.har"
+	files := []string{file}
+	cnt, err := feedEntries(files, true)
 	if err != nil {
 		logger.Log.Warning("Failed processing file: " + err.Error())
 		t.Fail()
 	}
 
-	loadStartingOAS()
+	waitQueueProcessed()
 
-	GetOasGeneratorInstance().ServiceSpecs.Range(func(_, val interface{}) bool {
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+		svc := key.(string)
 		gen := val.(*SpecGen)
 		spec, err := gen.GetSpec()
 		if err != nil {
@@ -112,8 +122,7 @@ func TestFileLDJSON(t *testing.T) {
 			t.FailNow()
 		}
 
-		specText, _ := json.MarshalIndent(spec, "", "\t")
-		t.Logf("%s", string(specText))
+		outputSpec(svc, spec, t)
 
 		err = spec.Validate()
 		if err != nil {
@@ -123,6 +132,19 @@ func TestFileLDJSON(t *testing.T) {
 
 		return true
 	})
+
+	logger.Log.Infof("Processed entries: %d", cnt)
+}
+
+func waitQueueProcessed() {
+	for {
+		time.Sleep(100 * time.Millisecond)
+		queue := len(GetOasGeneratorInstance().entriesChan)
+		logger.Log.Infof("Queue: %d", queue)
+		if queue < 1 {
+			break
+		}
+	}
 }
 
 func loadStartingOAS() {
@@ -149,26 +171,33 @@ func loadStartingOAS() {
 	gen.StartFromSpec(doc)
 
 	GetOasGeneratorInstance().ServiceSpecs.Store("catalogue", gen)
-
-	return
 }
 
 func TestEntriesNegative(t *testing.T) {
 	files := []string{"invalid"}
-	err := feedEntries(files)
+	_, err := feedEntries(files, false)
 	if err == nil {
 		t.Logf("Should have failed")
 		t.Fail()
 	}
 }
 
+func TestEntriesPositive(t *testing.T) {
+	files := []string{"test_artifacts/params.har"}
+	_, err := feedEntries(files, false)
+	if err != nil {
+		t.Logf("Failed")
+		t.Fail()
+	}
+}
+
 func TestLoadValidHAR(t *testing.T) {
 	inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}`
 	var entry *har.Entry
 	var err = json.Unmarshal([]byte(inp), &entry)
 	if err != nil {
 		t.Logf("Failed to decode entry: %s", err)
-		// t.FailNow() demonstrates the problem of library
+		t.FailNow() // demonstrates the problem of `martian` HAR library
 	}
 }
 
@@ -193,5 +222,4 @@ func TestLoadValid3_1(t *testing.T) {
 		t.Log(err)
 		t.FailNow()
 	}
-	return
 }

agent/pkg/oas/test_artifacts/example.har

@@ -13,10 +13,14 @@
         "time": 1022,
         "request": {
           "method": "GET",
-          "url": "http://trcc-api-service/proxies/",
+          "url": "http://trcc-api-service/proxies;matrixparam=example/",
           "httpVersion": "",
           "cookies": [],
           "headers": [
+            {
+              "name": "X-Custom-Demo-Header",
+              "value": "demo"
+            },
             {
               "name": "Sec-Fetch-Dest",
               "value": "empty"
@@ -124,6 +128,10 @@
           "httpVersion": "",
           "cookies": [],
           "headers": [
+            {
+              "name": "X-Custom-Demo-Header2",
+              "value": "demo2"
+            },
             {
               "name": "Vary",
               "value": "Origin"
@@ -24568,7 +24576,7 @@
           "bodySize": -1
         },
         "response": {
-          "status": 200,
+          "status": 308,
           "statusText": "OK",
           "httpVersion": "",
           "cookies": [],
@@ -24635,7 +24643,7 @@
         "time": 1,
         "request": {
           "method": "GET",
-          "url": "http://trcc-api-service/models/aws_kong5/suites/all/runs",
+          "url": "http://trcc-api-service/models/aws_kong5/suites/all/runs.png",
           "httpVersion": "",
           "cookies": [],
           "headers": [
@@ -24894,7 +24902,7 @@
           "bodySize": -1
         },
         "response": {
-          "status": 200,
+          "status": 0,
           "statusText": "OK",
           "httpVersion": "",
           "cookies": [],

agent/pkg/oas/test_artifacts/params.har

@@ -0,0 +1,127 @@
+{
+  "log": {
+    "version": "1.2",
+    "creator": {
+      "name": "mitmproxy har_dump",
+      "version": "0.1",
+      "comment": "mitmproxy version mitmproxy 4.0.4"
+    },
+    "entries": [
+      {
+        "startedDateTime": "2019-09-06T06:14:43.864529+00:00",
+        "time": 111,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/e21f7112-3d3b-4632-9da3-a4af2e0e9166/sub1",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "headersSize": 1542,
+          "bodySize": 0,
+          "queryString": []
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "mimeType": "text/html",
+            "text": "",
+            "size": 0
+          },
+          "redirectURL": "",
+          "headersSize": 245,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 22,
+          "receive": 2,
+          "wait": 87,
+          "connect": -1,
+          "ssl": -1
+        },
+        "serverIPAddress": "54.210.29.33"
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:18.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a/sub2",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:19.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a;mparam=matrixparam",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      }
+    ]
+  }
+}

agent/pkg/oas/tree.go

@@ -1,34 +1,48 @@
 package oas
 
 import (
-	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/shared/logger"
+	"net/url"
 	"strconv"
 	"strings"
+
+	"github.com/chanced/openapi"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 type NodePath = []string
 
 type Node struct {
-	constant *string
-	param    *openapi.ParameterObj
-	ops      *openapi.PathObj
-	parent   *Node
-	children []*Node
+	constant  *string
+	pathParam *openapi.ParameterObj
+	pathObj   *openapi.PathObj
+	parent    *Node
+	children  []*Node
 }
 
-func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Node) {
-	if pathObjToSet == nil {
+func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node *Node) {
+	if existingPathObj == nil {
 		panic("Invalid function call")
 	}
 
 	pathChunk := path[0]
+	potentialMatrix := strings.SplitN(pathChunk, ";", 2)
+	if len(potentialMatrix) > 1 {
+		pathChunk = potentialMatrix[0]
+		logger.Log.Warningf("URI matrix params are not supported: %s", potentialMatrix[1])
+	}
+
 	chunkIsParam := strings.HasPrefix(pathChunk, "{") && strings.HasSuffix(pathChunk, "}")
+	pathChunk, err := url.PathUnescape(pathChunk)
+	if err != nil {
+		logger.Log.Warningf("URI segment is not correctly encoded: %s", pathChunk)
+		// any side effects on continuing?
+	}
+
 	chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
 
 	var paramObj *openapi.ParameterObj
-	if chunkIsParam && pathObjToSet != nil {
-		paramObj = findParamByName(pathObjToSet.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
+	if chunkIsParam && existingPathObj != nil && existingPathObj.Parameters != nil {
+		_, paramObj = findParamByName(existingPathObj.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
 	}
 
 	if paramObj == nil {
@@ -46,34 +60,30 @@ func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Nod
 		n.children = append(n.children, node)
 
 		if paramObj != nil {
-			node.param = paramObj
+			node.pathParam = paramObj
 		} else if chunkIsGibberish {
-			initParams(&pathObjToSet.Parameters)
 
 			newParam := n.createParam()
-			node.param = newParam
-
-			appended := append(*pathObjToSet.Parameters, newParam)
-			pathObjToSet.Parameters = &appended
+			node.pathParam = newParam
 		} else {
 			node.constant = &pathChunk
 		}
 	}
 
-	// add example if it's a param
-	if node.param != nil && !chunkIsParam {
-		exmp := &node.param.Examples
+	// add example if it's a gibberish chunk
+	if node.pathParam != nil && !chunkIsParam {
+		exmp := &node.pathParam.Examples
 		err := fillParamExample(&exmp, pathChunk)
 		if err != nil {
 			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
 		}
 	}
 
-	// TODO: eat up trailing slash, in a smart way: node.ops!=nil && path[1]==""
+	// TODO: eat up trailing slash, in a smart way: node.pathObj!=nil && path[1]==""
 	if len(path) > 1 {
-		return node.getOrSet(path[1:], pathObjToSet)
-	} else if node.ops == nil {
-		node.ops = pathObjToSet
+		return node.getOrSet(path[1:], existingPathObj)
+	} else if node.pathObj == nil {
+		node.pathObj = existingPathObj
 	}
 
 	return node
@@ -82,18 +92,24 @@ func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Nod
 func (n *Node) createParam() *openapi.ParameterObj {
 	name := "param"
 
-	// REST assumption, not always correct
-	if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
-		name = *n.constant
-		name = name[:len(name)-2] + "Id"
-	} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
-		name = *n.constant
-		name = name[:len(name)-1] + "Id"
+	if n.constant != nil { // the node is already a param
+		// REST assumption, not always correct
+		if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
+			name = *n.constant
+			name = name[:len(name)-2] + "Id"
+		} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
+			name = *n.constant
+			name = name[:len(name)-1] + "Id"
+		} else if isAlpha(*n.constant) {
+			name = *n.constant + "Id"
+		}
+
+		name = cleanNonAlnum([]byte(name))
 	}
 
 	newParam := createSimpleParam(name, "path", "string")
 	x := n.countParentParams()
-	if x > 1 {
+	if x > 0 {
 		newParam.Name = newParam.Name + strconv.Itoa(x)
 	}
 
@@ -111,7 +127,7 @@ func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunkIsGibberish b
 			// TODO: check the regex pattern of param? for exceptions etc
 
 			if paramObj != nil {
-				// TODO: mergeParam(subnode.param, paramObj)
+				// TODO: mergeParam(subnode.pathParam, paramObj)
 				return subnode
 			} else {
 				return subnode
@@ -145,15 +161,14 @@ func (n *Node) listPaths() *openapi.Paths {
 	var strChunk string
 	if n.constant != nil {
 		strChunk = *n.constant
-	} else if n.param != nil {
-		strChunk = "{" + n.param.Name + "}"
-	} else {
-		// this is the root node
-	}
+	} else if n.pathParam != nil {
+		strChunk = "{" + n.pathParam.Name + "}"
+	} // else -> this is the root node
 
 	// add self
-	if n.ops != nil {
-		paths.Items[openapi.PathValue(strChunk)] = n.ops
+	if n.pathObj != nil {
+		fillPathParams(n, n.pathObj)
+		paths.Items[openapi.PathValue(strChunk)] = n.pathObj
 	}
 
 	// recurse into children
@@ -173,6 +188,29 @@ func (n *Node) listPaths() *openapi.Paths {
 	return paths
 }
 
+func fillPathParams(n *Node, pathObj *openapi.PathObj) {
+	// collect all path parameters from parent hierarchy
+	node := n
+	for {
+		if node.pathParam != nil {
+			initParams(&pathObj.Parameters)
+
+			idx, paramObj := findParamByName(pathObj.Parameters, openapi.InPath, node.pathParam.Name)
+			if paramObj == nil {
+				appended := append(*pathObj.Parameters, node.pathParam)
+				pathObj.Parameters = &appended
+			} else {
+				(*pathObj.Parameters)[idx] = paramObj
+			}
+		}
+
+		node = node.parent
+		if node == nil {
+			break
+		}
+	}
+}
+
 type PathAndOp struct {
 	path string
 	op   *openapi.Operation
@@ -192,7 +230,7 @@ func (n *Node) countParentParams() int {
 	res := 0
 	node := n
 	for {
-		if node.param != nil {
+		if node.pathParam != nil {
 			res++
 		}
 

agent/pkg/oas/tree_test.go

@@ -0,0 +1,37 @@
+package oas
+
+import (
+	"github.com/chanced/openapi"
+	"strings"
+	"testing"
+)
+
+func TestTree(t *testing.T) {
+	testCases := []struct {
+		inp       string
+		numParams int
+		label     string
+	}{
+		{"/", 0, ""},
+		{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation", 1, "vendor,init,conversation"},
+		{"/v1.0.0/config/launcher/sp_nKNHCzsN/{f34efcae-6583-11eb-908a-00b0fcb9d4f6}/vendor,init,conversation", 0, "vendor,init,conversation"},
+		{"/getSvgs/size/small/brand/SFLY/layoutId/170943/layoutVersion/1/sizeId/742/surface/0/isLandscape/true/childSkus/%7B%7D", 1, ""},
+	}
+
+	tree := new(Node)
+	for _, tc := range testCases {
+		split := strings.Split(tc.inp, "/")
+		pathObj := new(openapi.PathObj)
+		node := tree.getOrSet(split, pathObj)
+		
+		fillPathParams(node, pathObj)
+
+		if node.constant != nil && *node.constant != tc.label {
+			t.Errorf("Constant does not match: %s != %s", *node.constant, tc.label)
+		}
+
+		if tc.numParams > 0 && (pathObj.Parameters == nil || len(*pathObj.Parameters) < tc.numParams) {
+			t.Errorf("Wrong num of params, expected: %d", tc.numParams)
+		}
+	}
+}

agent/pkg/oas/utils.go

@@ -3,11 +3,12 @@ package oas
 import (
 	"encoding/json"
 	"errors"
-	"github.com/chanced/openapi"
-	"github.com/google/martian/har"
-	"github.com/up9inc/mizu/shared/logger"
+	"mizuserver/pkg/har"
 	"strconv"
 	"strings"
+
+	"github.com/chanced/openapi"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func exampleResolver(ref string) (*openapi.ExampleObj, error) {
@@ -32,16 +33,14 @@ func headerResolver(ref string) (*openapi.HeaderObj, error) {
 
 func initParams(obj **openapi.ParameterList) {
 	if *obj == nil {
-		var params openapi.ParameterList
-		params = make([]openapi.Parameter, 0)
+		var params openapi.ParameterList = make([]openapi.Parameter, 0)
 		*obj = &params
 	}
 }
 
 func initHeaders(respObj *openapi.ResponseObj) {
 	if respObj.Headers == nil {
-		var created openapi.Headers
-		created = map[string]openapi.Header{}
+		var created openapi.Headers = map[string]openapi.Header{}
 		respObj.Headers = created
 	}
 }
@@ -71,9 +70,10 @@ func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *op
 	return &newParam
 }
 
-func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (pathParam *openapi.ParameterObj) {
+func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (idx int, pathParam *openapi.ParameterObj) {
 	caseInsensitive := in == openapi.InHeader
-	for _, param := range *params {
+	for i, param := range *params {
+		idx = i
 		paramObj, err := param.ResolveParameter(paramResolver)
 		if err != nil {
 			logger.Log.Warningf("Failed to resolve reference: %s", err)
@@ -84,12 +84,13 @@ func findParamByName(params *openapi.ParameterList, in openapi.In, name string)
 			continue
 		}
 
-		if paramObj.Name == name || (caseInsensitive && strings.ToLower(paramObj.Name) == strings.ToLower(name)) {
+		if paramObj.Name == name || (caseInsensitive && strings.EqualFold(paramObj.Name, name)) {
 			pathParam = paramObj
 			break
 		}
 	}
-	return pathParam
+
+	return idx, pathParam
 }
 
 func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj {
@@ -100,44 +101,23 @@ func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj
 			continue
 		}
 
-		if strings.ToLower(hname) == strings.ToLower(name) {
+		if strings.EqualFold(hname, name) {
 			return hdrObj
 		}
 	}
 	return nil
 }
 
-type NVPair struct {
-	Name  string
-	Value string
-}
-
 type nvParams struct {
 	In             openapi.In
-	Pairs          func() []NVPair
+	Pairs          []har.NVP
 	IsIgnored      func(name string) bool
 	GeneralizeName func(name string) string
 }
 
-func qstrToNVP(list []har.QueryString) []NVPair {
-	res := make([]NVPair, len(list))
-	for idx, val := range list {
-		res[idx] = NVPair{Name: val.Name, Value: val.Value}
-	}
-	return res
-}
-
-func hdrToNVP(list []har.Header) []NVPair {
-	res := make([]NVPair, len(list))
-	for idx, val := range list {
-		res[idx] = NVPair{Name: val.Name, Value: val.Value}
-	}
-	return res
-}
-
 func handleNameVals(gw nvParams, params **openapi.ParameterList) {
 	visited := map[string]*openapi.ParameterObj{}
-	for _, pair := range gw.Pairs() {
+	for _, pair := range gw.Pairs {
 		if gw.IsIgnored(pair.Name) {
 			continue
 		}
@@ -145,7 +125,7 @@ func handleNameVals(gw nvParams, params **openapi.ParameterList) {
 		nameGeneral := gw.GeneralizeName(pair.Name)
 
 		initParams(params)
-		param := findParamByName(*params, gw.In, pair.Name)
+		_, param := findParamByName(*params, gw.In, pair.Name)
 		if param == nil {
 			param = createSimpleParam(nameGeneral, gw.In, openapi.TypeString)
 			appended := append(**params, param)
@@ -342,3 +322,26 @@ func anyJSON(text string) (anyVal interface{}, isJSON bool) {
 
 	return nil, false
 }
+
+func cleanNonAlnum(s []byte) string {
+	j := 0
+	for _, b := range s {
+		if ('a' <= b && b <= 'z') ||
+			('A' <= b && b <= 'Z') ||
+			('0' <= b && b <= '9') ||
+			b == ' ' {
+			s[j] = b
+			j++
+		}
+	}
+	return string(s[:j])
+}
+
+func isAlpha(s string) bool {
+	for _, r := range s {
+		if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') {
+			return false
+		}
+	}
+	return true
+}

agent/pkg/providers/install_provider.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"mizuserver/pkg/config"
 
+	"github.com/up9inc/mizu/shared/logger"
+
 	ory "github.com/ory/kratos-client-go"
 )
 
@@ -38,7 +40,9 @@ func CreateAdminUser(password string, ctx context.Context) (token *string, err e
 
 	if err != nil {
 		//Delete the user to prevent a half-setup situation where admin user is created without admin privileges
-		DeleteUser(identityId, ctx)
+		if err := DeleteUser(identityId, ctx); err != nil {
+			logger.Log.Error(err)
+		}
 
 		return nil, err, nil
 	}

agent/pkg/providers/stats_provider.go

@@ -7,6 +7,7 @@ import (
 
 type GeneralStats struct {
 	EntriesCount        int
+	EntriesVolumeInGB   float64
 	FirstEntryTimestamp int
 	LastEntryTimestamp  int
 }
@@ -21,8 +22,9 @@ func GetGeneralStats() GeneralStats {
 	return generalStats
 }
 
-func EntryAdded() {
+func EntryAdded(size int) {
 	generalStats.EntriesCount++
+	generalStats.EntriesVolumeInGB += float64(size) / (1 << 30)
 
 	currentTimestamp := int(time.Now().Unix())
 
@@ -32,5 +34,3 @@ func EntryAdded() {
 
 	generalStats.LastEntryTimestamp = currentTimestamp
 }
-
-

agent/pkg/providers/stats_provider_test.go

@@ -12,6 +12,10 @@ func TestNoEntryAddedCount(t *testing.T) {
 	if entriesStats.EntriesCount != 0 {
 		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesCount)
 	}
+
+	if entriesStats.EntriesVolumeInGB != 0 {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesVolumeInGB)
+	}
 }
 
 func TestEntryAddedCount(t *testing.T) {
@@ -20,7 +24,7 @@ func TestEntryAddedCount(t *testing.T) {
 	for _, entriesCount := range tests {
 		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
 			for i := 0; i < entriesCount; i++ {
-				providers.EntryAdded()
+				providers.EntryAdded(0)
 			}
 
 			entriesStats := providers.GetGeneralStats()
@@ -29,7 +33,38 @@ func TestEntryAddedCount(t *testing.T) {
 				t.Errorf("unexpected result - expected: %v, actual: %v", entriesCount, entriesStats.EntriesCount)
 			}
 
+			if entriesStats.EntriesVolumeInGB != 0 {
+				t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesVolumeInGB)
+			}
+
 			t.Cleanup(providers.ResetGeneralStats)
 		})
 	}
 }
+
+func TestEntryAddedVolume(t *testing.T) {
+	// 6 bytes + 4 bytes
+	tests := [][]byte{[]byte("volume"), []byte("test")}
+	var expectedEntriesCount int
+	var expectedVolumeInGB float64
+
+	for _, data := range tests {
+		t.Run(fmt.Sprintf("%d", len(data)), func(t *testing.T) {
+			expectedEntriesCount++
+			expectedVolumeInGB += float64(len(data)) / (1 << 30)
+
+			providers.EntryAdded(len(data))
+
+			entriesStats := providers.GetGeneralStats()
+
+			if entriesStats.EntriesCount != expectedEntriesCount {
+				t.Errorf("unexpected result - expected: %v, actual: %v", expectedEntriesCount, entriesStats.EntriesCount)
+			}
+
+			if entriesStats.EntriesVolumeInGB != expectedVolumeInGB {
+				t.Errorf("unexpected result - expected: %v, actual: %v", expectedVolumeInGB, entriesStats.EntriesVolumeInGB)
+			}
+		})
+	}
+
+}

agent/pkg/providers/user_provider.go

@@ -85,11 +85,11 @@ func DeleteUser(identityId string, ctx context.Context) error {
 		return err
 	}
 	if result == nil {
-		return errors.New("unknown error occured during user deletion")
+		return fmt.Errorf("unknown error occured during user deletion %v", identityId)
 	}
 
 	if result.StatusCode < 200 || result.StatusCode > 299 {
-		return errors.New(fmt.Sprintf("user deletion returned bad status %d", result.StatusCode))
+		return fmt.Errorf("user deletion %v returned bad status %d", identityId, result.StatusCode)
 	} else {
 		return nil
 	}

agent/pkg/routes/not_found_route.go

@@ -1,18 +0,0 @@
-package routes
-
-import (
-	"github.com/gin-gonic/gin"
-	"net/http"
-)
-
-// NotFoundRoute defines the 404 Error route.
-func NotFoundRoute(app *gin.Engine) {
-	app.Use(
-		func(c *gin.Context) {
-			c.JSON(http.StatusNotFound, map[string]interface{}{
-				"error": true,
-				"msg":   "sorry, endpoint is not found",
-			})
-		},
-	)
-}

agent/pkg/rules/rulesHTTP.go

@@ -4,15 +4,15 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"mizuserver/pkg/har"
 	"reflect"
 	"regexp"
 	"strings"
 
 	"github.com/up9inc/mizu/shared/logger"
 
-	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
-	jsonpath "github.com/yalp/jsonpath"
+	"github.com/yalp/jsonpath"
 )
 
 type RulesMatched struct {
@@ -108,7 +108,7 @@ func PassedValidationRules(rulesMatched []RulesMatched) (bool, int64, int) {
 	}
 
 	for _, rule := range rulesMatched {
-		if rule.Matched == false {
+		if !rule.Matched {
 			return false, responseTime, numberOfRulesMatched
 		} else {
 			if strings.ToLower(rule.Rule.Type) == "slo" {

agent/pkg/up9/main.go

@@ -3,10 +3,10 @@ package up9
 import (
 	"bytes"
 	"compress/zlib"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"mizuserver/pkg/har"
 	"mizuserver/pkg/utils"
 	"net/http"
 	"net/url"
@@ -15,7 +15,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/google/martian/har"
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
@@ -247,7 +246,7 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 			if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil {
 				continue
 			}
-			harEntry, err := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
+			harEntry, err := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
 			if err != nil {
 				continue
 			}
@@ -259,11 +258,6 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 				harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entry.Destination.Name)
 			}
 
-			// go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64
-			if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil {
-				continue
-			}
-
 			batch = append(batch, *harEntry)
 
 			now := time.Now()

agent/pkg/utils/utils.go

@@ -31,10 +31,14 @@ func StartServer(app *gin.Engine) {
 	}
 
 	go func() {
-		_ = <-signals
+		<-signals
 		logger.Log.Infof("Shutting down...")
-		ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
-		_ = srv.Shutdown(ctx)
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			logger.Log.Errorf("%v", err)
+		}
 		os.Exit(0)
 	}()
 
@@ -91,7 +95,7 @@ func UniqueStringSlice(s []string) []string {
 	uniqueMap := map[string]bool{}
 
 	for _, val := range s {
-		if uniqueMap[val] == true {
+		if uniqueMap[val] {
 			continue
 		}
 		uniqueMap[val] = true

cli/apiserver/provider.go

@@ -23,8 +23,8 @@ type Provider struct {
 	client  *http.Client
 }
 
-const DefaultRetries = 20
-const DefaultTimeout = 5 * time.Second
+const DefaultRetries = 3
+const DefaultTimeout = 2 * time.Second
 
 func NewProvider(url string, retries int, timeout time.Duration) *Provider {
 	return &Provider{
@@ -36,16 +36,6 @@ func NewProvider(url string, retries int, timeout time.Duration) *Provider {
 	}
 }
 
-func NewProviderWithoutRetries(url string, timeout time.Duration) *Provider {
-	return &Provider{
-		url:     url,
-		retries: 1,
-		client: &http.Client{
-			Timeout: timeout,
-		},
-	}
-}
-
 func (provider *Provider) TestConnection() error {
 	retriesLeft := provider.retries
 	for retriesLeft > 0 {

cli/cmd/checkRunner.go

@@ -3,13 +3,14 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"regexp"
+
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared/kubernetes"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/semver"
-	"net/http"
 )
 
 func runMizuCheck() {
@@ -34,7 +35,7 @@ func runMizuCheck() {
 	}
 
 	if checkPassed {
-		checkPassed = checkServerConnection(kubernetesProvider, cancel)
+		checkPassed = checkServerConnection(kubernetesProvider)
 	}
 
 	if checkPassed {
@@ -91,23 +92,75 @@ func checkKubernetesVersion(kubernetesVersion *semver.SemVersion) bool {
 	return true
 }
 
-func checkServerConnection(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) bool {
+func checkServerConnection(kubernetesProvider *kubernetes.Provider) bool {
 	logger.Log.Infof("\nmizu-connectivity\n--------------------")
 
 	serverUrl := GetApiServerUrl()
 
-	if response, err := http.Get(fmt.Sprintf("%s/", serverUrl)); err != nil || response.StatusCode != 200 {
-		startProxyReportErrorIfAny(kubernetesProvider, cancel)
+	apiServerProvider := apiserver.NewProvider(serverUrl, 1, apiserver.DefaultTimeout)
+	if err := apiServerProvider.TestConnection(); err == nil {
+		logger.Log.Infof("%v found Mizu server tunnel available and connected successfully to API server", fmt.Sprintf(uiUtils.Green, "√"))
+		return true
+	}
+
+	connectedToApiServer := false
+
+	if err := checkProxy(serverUrl, kubernetesProvider); err != nil {
+		logger.Log.Errorf("%v couldn't connect to API server using proxy, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+	} else {
+		connectedToApiServer = true
+		logger.Log.Infof("%v connected successfully to API server using proxy", fmt.Sprintf(uiUtils.Green, "√"))
+	}
+
+	if err := checkPortForward(serverUrl, kubernetesProvider); err != nil {
+		logger.Log.Errorf("%v couldn't connect to API server using port-forward, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+	} else {
+		connectedToApiServer = true
+		logger.Log.Infof("%v connected successfully to API server using port-forward", fmt.Sprintf(uiUtils.Green, "√"))
+	}
+
+	return connectedToApiServer
+}
+
+func checkProxy(serverUrl string, kubernetesProvider *kubernetes.Provider) error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	httpServer, err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.ProxyHost, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName, cancel)
+	if err != nil {
+		return err
 	}
 
 	apiServerProvider := apiserver.NewProvider(serverUrl, apiserver.DefaultRetries, apiserver.DefaultTimeout)
 	if err := apiServerProvider.TestConnection(); err != nil {
-		logger.Log.Errorf("%v couldn't connect to API server, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
-		return false
+		return err
 	}
 
-	logger.Log.Infof("%v connected successfully to API server", fmt.Sprintf(uiUtils.Green, "√"))
-	return true
+	if err := httpServer.Shutdown(ctx); err != nil {
+		logger.Log.Debugf("Error occurred while stopping proxy, err: %v", err)
+	}
+
+	return nil
+}
+
+func checkPortForward(serverUrl string, kubernetesProvider *kubernetes.Provider) error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	podRegex, _ := regexp.Compile(kubernetes.ApiServerPodName)
+	forwarder, err := kubernetes.NewPortForward(kubernetesProvider, config.Config.MizuResourcesNamespace, podRegex, config.Config.Tap.GuiPort, ctx, cancel)
+	if err != nil {
+		return err
+	}
+
+	apiServerProvider := apiserver.NewProvider(serverUrl, apiserver.DefaultRetries, apiserver.DefaultTimeout)
+	if err := apiServerProvider.TestConnection(); err != nil {
+		return err
+	}
+
+	forwarder.Close()
+
+	return nil
 }
 
 func checkAllResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.Provider, isInstallCommand bool) bool {
@@ -117,32 +170,32 @@ func checkAllResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.
 	allResourcesExist := checkResourceExist(config.Config.MizuResourcesNamespace, "namespace", exist, err)
 
 	exist, err = kubernetesProvider.DoesConfigMapExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ConfigMapName)
-	allResourcesExist = checkResourceExist(kubernetes.ConfigMapName, "config map", exist, err)
+	allResourcesExist = checkResourceExist(kubernetes.ConfigMapName, "config map", exist, err) && allResourcesExist
 
 	exist, err = kubernetesProvider.DoesServiceAccountExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName)
-	allResourcesExist = checkResourceExist(kubernetes.ServiceAccountName, "service account", exist, err)
+	allResourcesExist = checkResourceExist(kubernetes.ServiceAccountName, "service account", exist, err) && allResourcesExist
 
 	if config.Config.IsNsRestrictedMode() {
 		exist, err = kubernetesProvider.DoesRoleExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleName)
-		allResourcesExist = checkResourceExist(kubernetes.RoleName, "role", exist, err)
+		allResourcesExist = checkResourceExist(kubernetes.RoleName, "role", exist, err) && allResourcesExist
 
 		exist, err = kubernetesProvider.DoesRoleBindingExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleBindingName)
-		allResourcesExist = checkResourceExist(kubernetes.RoleBindingName, "role binding", exist, err)
+		allResourcesExist = checkResourceExist(kubernetes.RoleBindingName, "role binding", exist, err) && allResourcesExist
 	} else {
 		exist, err = kubernetesProvider.DoesClusterRoleExist(ctx, kubernetes.ClusterRoleName)
-		allResourcesExist = checkResourceExist(kubernetes.ClusterRoleName, "cluster role", exist, err)
+		allResourcesExist = checkResourceExist(kubernetes.ClusterRoleName, "cluster role", exist, err) && allResourcesExist
 
 		exist, err = kubernetesProvider.DoesClusterRoleBindingExist(ctx, kubernetes.ClusterRoleBindingName)
-		allResourcesExist = checkResourceExist(kubernetes.ClusterRoleBindingName, "cluster role binding", exist, err)
+		allResourcesExist = checkResourceExist(kubernetes.ClusterRoleBindingName, "cluster role binding", exist, err) && allResourcesExist
 	}
 
 	exist, err = kubernetesProvider.DoesServiceExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
-	allResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "service", exist, err)
+	allResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "service", exist, err) && allResourcesExist
 
 	if isInstallCommand {
-		allResourcesExist = checkInstallResourcesExist(ctx, kubernetesProvider)
+		allResourcesExist = checkInstallResourcesExist(ctx, kubernetesProvider) && allResourcesExist
 	} else {
-		allResourcesExist = checkTapResourcesExist(ctx, kubernetesProvider)
+		allResourcesExist = checkTapResourcesExist(ctx, kubernetesProvider) && allResourcesExist
 	}
 
 	return allResourcesExist
@@ -153,13 +206,13 @@ func checkInstallResourcesExist(ctx context.Context, kubernetesProvider *kuberne
 	installResourcesExist := checkResourceExist(kubernetes.DaemonRoleName, "role", exist, err)
 
 	exist, err = kubernetesProvider.DoesRoleBindingExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleBindingName)
-	installResourcesExist = checkResourceExist(kubernetes.DaemonRoleBindingName, "role binding", exist, err)
+	installResourcesExist = checkResourceExist(kubernetes.DaemonRoleBindingName, "role binding", exist, err) && installResourcesExist
 
 	exist, err = kubernetesProvider.DoesPersistentVolumeClaimExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName)
-	installResourcesExist = checkResourceExist(kubernetes.PersistentVolumeClaimName, "persistent volume claim", exist, err)
+	installResourcesExist = checkResourceExist(kubernetes.PersistentVolumeClaimName, "persistent volume claim", exist, err) && installResourcesExist
 
 	exist, err = kubernetesProvider.DoesDeploymentExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
-	installResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "deployment", exist, err)
+	installResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "deployment", exist, err) && installResourcesExist
 
 	return installResourcesExist
 }

cli/cmd/common.go

@@ -5,6 +5,10 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"path"
+	"regexp"
+	"time"
+
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
@@ -13,8 +17,6 @@ import (
 	"github.com/up9inc/mizu/cli/resources"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
-	"path"
-	"time"
 
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/shared/kubernetes"
@@ -25,7 +27,7 @@ func GetApiServerUrl() string {
 	return fmt.Sprintf("http://%s", kubernetes.GetMizuApiServerProxiedHostAndPath(config.Config.Tap.GuiPort))
 }
 
-func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, ctx context.Context, cancel context.CancelFunc) {
 	httpServer, err := kubernetes.StartProxy(kubernetesProvider, config.Config.Tap.ProxyHost, config.Config.Tap.GuiPort, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName, cancel)
 	if err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running k8s proxy %v\n"+
@@ -34,21 +36,22 @@ func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel
 		return
 	}
 
-	apiProvider = apiserver.NewProviderWithoutRetries(GetApiServerUrl(), time.Second) // short check for proxy
+	apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
 	if err := apiProvider.TestConnection(); err != nil {
 		logger.Log.Debugf("Couldn't connect using proxy, stopping proxy and trying to create port-forward")
-		if err := httpServer.Shutdown(context.Background()); err != nil {
+		if err := httpServer.Shutdown(ctx); err != nil {
 			logger.Log.Debugf("Error occurred while stopping proxy %v", errormessage.FormatError(err))
 		}
 
-		if err := kubernetes.NewPortForward(kubernetesProvider, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName, config.Config.Tap.GuiPort, cancel); err != nil {
+		podRegex, _ := regexp.Compile(kubernetes.ApiServerPodName)
+		if _, err := kubernetes.NewPortForward(kubernetesProvider, config.Config.MizuResourcesNamespace, podRegex, config.Config.Tap.GuiPort, ctx, cancel); err != nil {
 			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error occured while running port forward %v\n"+
 				"Try setting different port by using --%s", errormessage.FormatError(err), configStructs.GuiPortTapName))
 			cancel()
 			return
 		}
 
-		apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout) // long check for port-forward
+		apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
 		if err := apiProvider.TestConnection(); err != nil {
 			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
 			cancel()

cli/cmd/config.go

@@ -50,7 +50,9 @@ func init() {
 	rootCmd.AddCommand(configCmd)
 
 	defaultConfig := config.ConfigStruct{}
-	defaults.Set(&defaultConfig)
+	if err := defaults.Set(&defaultConfig); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	configCmd.Flags().BoolP(configStructs.RegenerateConfigName, "r", defaultConfig.Config.Regenerate, fmt.Sprintf("Regenerate the config file with default values to path %s or to chosen path using --%s", defaultConfig.ConfigFilePath, config.ConfigFilePathCommandName))
 }

cli/cmd/installRunner.go

@@ -4,10 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/shared/kubernetes"
-	core "k8s.io/api/core/v1"
-	"regexp"
-	"time"
 
 	"github.com/creasty/defaults"
 	"github.com/up9inc/mizu/cli/config"
@@ -35,7 +31,9 @@ func runMizuInstall() {
 	var defaultMaxEntriesDBSizeBytes int64 = 200 * 1000 * 1000
 
 	defaultResources := shared.Resources{}
-	defaults.Set(&defaultResources)
+	if err := defaults.Set(&defaultResources); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	mizuAgentConfig := getInstallMizuAgentConfig(defaultMaxEntriesDBSizeBytes, defaultResources)
 	serializedMizuConfig, err := getSerializedMizuAgentConfig(mizuAgentConfig)
@@ -46,7 +44,7 @@ func runMizuInstall() {
 
 	if err = resources.CreateInstallMizuResources(ctx, kubernetesProvider, serializedValidationRules,
 		serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(),
-		config.Config.MizuResourcesNamespace, config.Config.AgentImage, config.Config.BasenineImage,
+		config.Config.MizuResourcesNamespace, config.Config.AgentImage,
 		config.Config.KratosImage, config.Config.KetoImage,
 		nil, defaultMaxEntriesDBSizeBytes, defaultResources, config.Config.ImagePullPolicy(),
 		config.Config.LogLevel(), false); err != nil {
@@ -63,20 +61,6 @@ func runMizuInstall() {
 		return
 	}
 
-	logger.Log.Infof("Waiting for Mizu server to start...")
-	readyChan := make(chan string)
-	readyErrorChan := make(chan error)
-	go watchApiServerPodReady(ctx, kubernetesProvider, readyChan, readyErrorChan)
-
-	select {
-	case readyMessage := <-readyChan:
-		logger.Log.Infof(readyMessage)
-	case err := <-readyErrorChan:
-		defer resources.CleanUpMizuResources(ctx, cancel, kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
-		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("%v", errormessage.FormatError(err)))
-		return
-	}
-
 	logger.Log.Infof(uiUtils.Magenta, "Installation completed, run `mizu view` to connect to the mizu daemon instance")
 }
 
@@ -92,63 +76,8 @@ func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Reso
 		StandaloneMode:         true,
 		ServiceMap:             config.Config.ServiceMap,
 		OAS:                    config.Config.OAS,
+		Elastic:                config.Config.Elastic,
 	}
 
 	return &mizuAgentConfig
 }
-
-func watchApiServerPodReady(ctx context.Context, kubernetesProvider *kubernetes.Provider, readyChan chan string, readyErrorChan chan error) {
-	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", kubernetes.ApiServerPodName))
-	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
-	eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper)
-
-	apiServerTimeoutSec := config.GetIntEnvConfig(config.ApiServerTimeoutSec, 120)
-	timeAfter := time.After(time.Duration(apiServerTimeoutSec) * time.Second)
-	for {
-		select {
-		case wEvent, ok := <-eventChan:
-			if !ok {
-				eventChan = nil
-				continue
-			}
-
-			switch wEvent.Type {
-			case kubernetes.EventAdded:
-				logger.Log.Debugf("Watching API Server pod ready loop, added")
-			case kubernetes.EventDeleted:
-				logger.Log.Debugf("Watching API Server pod ready loop, %s removed", kubernetes.ApiServerPodName)
-			case kubernetes.EventModified:
-				modifiedPod, err := wEvent.ToPod()
-				if err != nil {
-					readyErrorChan <- err
-					return
-				}
-
-				logger.Log.Debugf("Watching API Server pod ready loop, modified: %v", modifiedPod.Status.Phase)
-
-				if modifiedPod.Status.Phase == core.PodRunning {
-					readyChan <- fmt.Sprintf("%v pod is running", modifiedPod.Name)
-					return
-				}
-			case kubernetes.EventBookmark:
-				break
-			case kubernetes.EventError:
-				break
-			}
-		case err, ok := <-errorChan:
-			if !ok {
-				errorChan = nil
-				continue
-			}
-
-			readyErrorChan <- fmt.Errorf("[ERROR] Agent creation, watching %v namespace, error: %v", config.Config.MizuResourcesNamespace, err)
-			return
-		case <-timeAfter:
-			readyErrorChan <- fmt.Errorf("mizu API server was not ready in time")
-			return
-		case <-ctx.Done():
-			logger.Log.Debugf("Watching API Server pod ready loop, ctx done")
-			return
-		}
-	}
-}

cli/cmd/logs.go

@@ -23,7 +23,7 @@ var logsCmd = &cobra.Command{
 		if err != nil {
 			return nil
 		}
-		ctx, _ := context.WithCancel(context.Background())
+		ctx := context.Background()
 
 		if validationErr := config.Config.Logs.Validate(); validationErr != nil {
 			return errormessage.FormatError(validationErr)
@@ -43,7 +43,9 @@ func init() {
 	rootCmd.AddCommand(logsCmd)
 
 	defaultLogsConfig := configStructs.LogsConfig{}
-	defaults.Set(&defaultLogsConfig)
+	if err := defaults.Set(&defaultLogsConfig); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	logsCmd.Flags().StringP(configStructs.FileLogsName, "f", defaultLogsConfig.FileStr, "Path for zip file (default current <pwd>\\mizu_logs.zip)")
 }

cli/cmd/root.go

@@ -30,7 +30,9 @@ Further info is available at https://github.com/up9inc/mizu`,
 
 func init() {
 	defaultConfig := config.ConfigStruct{}
-	defaults.Set(&defaultConfig)
+	if err := defaults.Set(&defaultConfig); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	rootCmd.PersistentFlags().StringSlice(config.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", config.SetCommandName))
 	rootCmd.PersistentFlags().String(config.ConfigFilePathCommandName, defaultConfig.ConfigFilePath, fmt.Sprintf("Override config file path using --%s", config.ConfigFilePathCommandName))

cli/cmd/tap.go

@@ -104,7 +104,9 @@ func init() {
 	rootCmd.AddCommand(tapCmd)
 
 	defaultTapConfig := configStructs.TapConfig{}
-	defaults.Set(&defaultTapConfig)
+	if err := defaults.Set(&defaultTapConfig); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
 	tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")

cli/cmd/tapRunner.go

@@ -124,7 +124,7 @@ func RunMizuTap() {
 	}
 
 	logger.Log.Infof("Waiting for Mizu Agent to start...")
-	if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, config.Config.BasenineImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel()); err != nil {
+	if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel()); err != nil {
 		var statusError *k8serrors.StatusError
 		if errors.As(err, &statusError) {
 			if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists {
@@ -164,6 +164,7 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 		AgentDatabasePath:      shared.DataDirPath,
 		ServiceMap:             config.Config.ServiceMap,
 		OAS:                    config.Config.OAS,
+		Elastic:                config.Config.Elastic,
 	}
 
 	return &mizuAgentConfig
@@ -342,7 +343,7 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 
 				if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
 					isPodReady = true
-					postApiServerStarted(ctx, kubernetesProvider, cancel, err)
+					postApiServerStarted(ctx, kubernetesProvider, cancel)
 				}
 			case kubernetes.EventBookmark:
 				break
@@ -405,7 +406,7 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
 			case "FailedScheduling", "Failed":
 				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Mizu API Server status: %s - %s", event.Reason, event.Note))
 				cancel()
-				break
+				
 			}
 		case err, ok := <-errorChan:
 			if !ok {
@@ -421,11 +422,11 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
 	}
 }
 
-func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, err error) {
-	startProxyReportErrorIfAny(kubernetesProvider, cancel)
+func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+	startProxyReportErrorIfAny(kubernetesProvider, ctx, cancel)
 
 	options, _ := getMizuApiFilteringOptions()
-	if err = startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
+	if err := startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", err))
 		cancel()
 	}

cli/cmd/version.go

@@ -36,7 +36,9 @@ func init() {
 	rootCmd.AddCommand(versionCmd)
 
 	defaultVersionConfig := configStructs.VersionConfig{}
-	defaults.Set(&defaultVersionConfig)
+	if err := defaults.Set(&defaultVersionConfig); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	versionCmd.Flags().BoolP(configStructs.DebugInfoVersionName, "d", defaultVersionConfig.DebugInfo, "Provide all information about version")
 

cli/cmd/view.go

@@ -6,6 +6,7 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/telemetry"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var viewCmd = &cobra.Command{
@@ -22,10 +23,14 @@ func init() {
 	rootCmd.AddCommand(viewCmd)
 
 	defaultViewConfig := configStructs.ViewConfig{}
-	defaults.Set(&defaultViewConfig)
+	if err := defaults.Set(&defaultViewConfig); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	viewCmd.Flags().Uint16P(configStructs.GuiPortViewName, "p", defaultViewConfig.GuiPort, "Provide a custom port for the web interface webserver")
 	viewCmd.Flags().StringP(configStructs.UrlViewName, "u", defaultViewConfig.Url, "Provide a custom host")
 
-	viewCmd.Flags().MarkHidden(configStructs.UrlViewName)
+	if err := viewCmd.Flags().MarkHidden(configStructs.UrlViewName); err != nil {
+		logger.Log.Debug(err)
+	}
 }

cli/cmd/viewRunner.go

@@ -47,7 +47,7 @@ func runMizuView() {
 			return
 		}
 		logger.Log.Infof("Establishing connection to k8s cluster...")
-		startProxyReportErrorIfAny(kubernetesProvider, cancel)
+		startProxyReportErrorIfAny(kubernetesProvider, ctx, cancel)
 	}
 
 	apiServerProvider := apiserver.NewProvider(url, apiserver.DefaultRetries, apiserver.DefaultTimeout)

cli/config/configStruct.go

@@ -28,7 +28,6 @@ type ConfigStruct struct {
 	Auth                   configStructs.AuthConfig    `yaml:"auth"`
 	Config                 configStructs.ConfigConfig  `yaml:"config,omitempty"`
 	AgentImage             string                      `yaml:"agent-image,omitempty" readonly:""`
-	BasenineImage          string                      `yaml:"basenine-image,omitempty" readonly:""`
 	KratosImage            string                      `yaml:"kratos-image,omitempty" readonly:""`
 	KetoImage              string                      `yaml:"keto-image,omitempty" readonly:""`
 	ImagePullPolicyStr     string                      `yaml:"image-pull-policy" default:"Always"`
@@ -39,8 +38,9 @@ type ConfigStruct struct {
 	ConfigFilePath         string                      `yaml:"config-path,omitempty" readonly:""`
 	HeadlessMode           bool                        `yaml:"headless" default:"false"`
 	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
-	ServiceMap             bool                        `yaml:"service-map,omitempty" default:"false" readonly:""`
+	ServiceMap             bool                        `yaml:"service-map" default:"false"`
 	OAS                    bool                        `yaml:"oas,omitempty" default:"false" readonly:""`
+	Elastic                shared.ElasticConfig        `yaml:"elastic"`
 }
 
 func (config *ConfigStruct) validate() error {
@@ -52,7 +52,6 @@ func (config *ConfigStruct) validate() error {
 }
 
 func (config *ConfigStruct) SetDefaults() {
-	config.BasenineImage = fmt.Sprintf("%s:%s", shared.BasenineImageRepo, shared.BasenineImageTag)
 	config.KratosImage = shared.KratosImageDefault
 	config.KetoImage = shared.KetoImageDefault
 	config.AgentImage = fmt.Sprintf("%s:%s", shared.MizuAgentImageRepo, mizu.SemVer)

cli/config/configStructs/tapConfig.go

@@ -60,12 +60,12 @@ func (config *TapConfig) MaxEntriesDBSizeBytes() int64 {
 func (config *TapConfig) Validate() error {
 	_, compileErr := regexp.Compile(config.PodRegexStr)
 	if compileErr != nil {
-		return errors.New(fmt.Sprintf("%s is not a valid regex %s", config.PodRegexStr, compileErr))
+		return fmt.Errorf("%s is not a valid regex %s", config.PodRegexStr, compileErr)
 	}
 
 	_, parseHumanDataSizeErr := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
 	if parseHumanDataSizeErr != nil {
-		return errors.New(fmt.Sprintf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize))
+		return fmt.Errorf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize)
 	}
 
 	if config.Workspace != "" {
@@ -76,7 +76,7 @@ func (config *TapConfig) Validate() error {
 	}
 
 	if config.Analysis && config.Workspace != "" {
-		return errors.New(fmt.Sprintf("Can't run with both --%s and --%s flags", AnalysisTapName, WorkspaceTapName))
+		return fmt.Errorf("Can't run with both --%s and --%s flags", AnalysisTapName, WorkspaceTapName)
 	}
 
 	return nil

cli/mizu/fsUtils/dirUtils.go

@@ -1,7 +1,6 @@
 package fsUtils
 
 import (
-	"errors"
 	"fmt"
 	"os"
 )
@@ -18,7 +17,7 @@ func EnsureDir(dirName string) error {
 			return err
 		}
 		if !info.IsDir() {
-			return errors.New(fmt.Sprintf("path exists but is not a directory: %s", dirName))
+			return fmt.Errorf("path exists but is not a directory: %s", dirName)
 		}
 		return nil
 	}

cli/resources/createResources.go

@@ -15,7 +15,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
-func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, basenineImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level) (bool, error) {
+func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level) (bool, error) {
 	if !isNsRestrictedMode {
 		if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil {
 			return false, err
@@ -42,7 +42,6 @@ func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.
 		Namespace:             mizuResourcesNamespace,
 		PodName:               kubernetes.ApiServerPodName,
 		PodImage:              agentImage,
-		BasenineImage:         basenineImage,
 		KratosImage:           "",
 		KetoImage:             "",
 		ServiceAccountName:    serviceAccountName,
@@ -68,7 +67,7 @@ func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.
 	return mizuServiceAccountExists, nil
 }
 
-func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, basenineImage string, kratosImage string, ketoImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, noPersistentVolumeClaim bool) error {
+func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, kratosImage string, ketoImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, noPersistentVolumeClaim bool) error {
 	if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil {
 		return err
 	}
@@ -98,7 +97,6 @@ func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kuberne
 		Namespace:             mizuResourcesNamespace,
 		PodName:               kubernetes.ApiServerPodName,
 		PodImage:              agentImage,
-		BasenineImage:         basenineImage,
 		KratosImage:           kratosImage,
 		KetoImage:             ketoImage,
 		ServiceAccountName:    serviceAccountName,

cli/telemetry/telemetry.go

@@ -53,6 +53,7 @@ func ReportTapTelemetry(apiProvider *apiserver.Provider, args interface{}, start
 		"args":                   string(argsBytes),
 		"executionTimeInSeconds": int(time.Since(startTime).Seconds()),
 		"apiCallsCount":          generalStats["EntriesCount"],
+		"trafficVolumeInGB":      generalStats["EntriesVolumeInGB"],
 	}
 
 	if err := sendTelemetry(argsMap); err != nil {

cli/up9/provider.go

@@ -23,9 +23,5 @@ func IsTokenValid(tokenString string, envName string) bool {
 	}
 	defer response.Body.Close()
 
-	if response.StatusCode != http.StatusOK {
-		return false
-	}
-
-	return true
+	return response.StatusCode == http.StatusOK
 }

docs/CHANGELOG.md

@@ -2,6 +2,35 @@
 This document summarizes main and fixes changes published in stable (aka `main`) branch of this project.
 Ongoing work and development releases are under `develop` branch.
 
+## 0.24.0
+
+### main features
+* ARM64 support -- Mizu is now available for ARM 64bit architecture
+  * Now you can run Mizu with `minikube` on your Apple M1 laptop or any other ARM-based hosts
+* New command helps user verify Mizu deployment
+  * Run `mizu check` to verify Mizu was deployed successfully
+  * `mizu check` verifies version compatibility, resources and permissions required by Mizu
+* EXPERIMENTAL: Service Map - graph of all service interactions
+  * Arrow direction show client to server connection
+  * Graph edge width reflects volume of traffic captured between the services
+  * to enable this experimental feature use `--set service-map=true` flag
+
+### improvements
+* Mizu container images are now served from [Docker Hub](https://hub.docker.com/r/up9inc/mizu), as multi-architecture images (arm64, amd64)
+* in Mizu GUI the filter query can now be applied by pressing CONTROL/COMMAND + ENTER
+* try port-forwarding if http-proxy connection to Mizu API server is not available
+
+### notable bug fixes
+* Fixed HTTP/1.0 presentation which was shown as HTTP/1.1
+* Fixed handling of long-living TCP connections, improves capturing gRPC and HTTP/2 traffic, and helps in service-mesh setups (istio, linkerd)
+
+
+## 0.23.0
+### notable bug fixes
+* fixed errors in Redis protocol parser (better handling of Array and Bulk String message types)
+
+
+
 ## 0.22.0
 
 ### main features

shared/consts.go

@@ -17,8 +17,6 @@ const (
 	MizuAgentImageRepo               = "docker.io/up9inc/mizu"
 	BasenineHost                     = "127.0.0.1"
 	BaseninePort                     = "9099"
-	BasenineImageRepo                = "docker.io/up9inc/basenine"
-	BasenineImageTag                 = "v0.4.13"
 	KratosImageDefault               = "gcr.io/up9-docker-hub/mizu-kratos/stable:0.0.0"
 	KetoImageDefault                 = "gcr.io/up9-docker-hub/mizu-keto/stable:0.0.0"
 )

shared/debounce/debounce.go

@@ -44,7 +44,7 @@ func (d *Debouncer) SetOn() error {
 	if d.canceled {
 		return fmt.Errorf("debouncer cancelled")
 	}
-	if d.running == true {
+	if d.running {
 		return nil
 	}
 

shared/go.sum

@@ -93,6 +93,7 @@ github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -307,6 +308,7 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

shared/kubernetes/eventWatchHelper.go

@@ -34,7 +34,7 @@ func (wh *EventWatchHelper) Filter(wEvent *WatchEvent) (bool, error) {
 		return false, nil
 	}
 
-	if strings.ToLower(event.Regarding.Kind) != strings.ToLower(wh.Kind) {
+	if strings.EqualFold(event.Regarding.Kind, wh.Kind) {
 		return false, nil
 	}
 

shared/kubernetes/mizuTapperSyncer.go

@@ -3,21 +3,22 @@ package kubernetes
 import (
 	"context"
 	"fmt"
+	"regexp"
+	"time"
+
 	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap/api"
 	core "k8s.io/api/core/v1"
-	"regexp"
-	"time"
 )
 
 const updateTappersDelay = 5 * time.Second
 
 type TappedPodChangeEvent struct {
-	Added                []core.Pod
-	Removed              []core.Pod
+	Added   []core.Pod
+	Removed []core.Pod
 }
 
 // MizuTapperSyncer uses a k8s pod watch to update tapper daemonsets when targeted pods are removed or created
@@ -222,10 +223,14 @@ func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() {
 			switch wEvent.Type {
 			case EventAdded:
 				logger.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
-				restartTappersDebouncer.SetOn()
+				if err := restartTappersDebouncer.SetOn(); err != nil {
+					logger.Log.Error(err)
+				}
 			case EventDeleted:
 				logger.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
-				restartTappersDebouncer.SetOn()
+				if err := restartTappersDebouncer.SetOn(); err != nil {
+					logger.Log.Error(err)
+				}
 			case EventModified:
 				logger.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
 				// Act only if the modified pod has already obtained an IP address.
@@ -235,7 +240,9 @@ func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() {
 				// - Pod reaches ready state
 				// Ready/unready transitions might also trigger this event.
 				if pod.Status.PodIP != "" {
-					restartTappersDebouncer.SetOn()
+					if err := restartTappersDebouncer.SetOn(); err != nil {
+						logger.Log.Error(err)
+					}
 				}
 			case EventBookmark:
 				break
@@ -285,8 +292,8 @@ func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, ch
 			tapperSyncer.CurrentlyTappedPods = podsToTap
 			tapperSyncer.nodeToTappedPodMap = GetNodeHostToTappedPodsMap(tapperSyncer.CurrentlyTappedPods)
 			tapperSyncer.TapPodChangesOut <- TappedPodChangeEvent{
-				Added:                addedPods,
-				Removed:              removedPods,
+				Added:   addedPods,
+				Removed: removedPods,
 			}
 			return nil, true
 		}

shared/kubernetes/provider.go

@@ -169,7 +169,6 @@ type ApiServerOptions struct {
 	Namespace             string
 	PodName               string
 	PodImage              string
-	BasenineImage         string
 	KratosImage           string
 	KetoImage             string
 	ServiceAccountName    string
@@ -195,19 +194,19 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 
 	cpuLimit, err := resource.ParseQuantity(opts.Resources.CpuLimit)
 	if err != nil {
-		return nil, errors.New(fmt.Sprintf("invalid cpu limit for %s container", opts.PodName))
+		return nil, fmt.Errorf("invalid cpu limit for %s container", opts.PodName)
 	}
 	memLimit, err := resource.ParseQuantity(opts.Resources.MemoryLimit)
 	if err != nil {
-		return nil, errors.New(fmt.Sprintf("invalid memory limit for %s container", opts.PodName))
+		return nil, fmt.Errorf("invalid memory limit for %s container", opts.PodName)
 	}
 	cpuRequests, err := resource.ParseQuantity(opts.Resources.CpuRequests)
 	if err != nil {
-		return nil, errors.New(fmt.Sprintf("invalid cpu request for %s container", opts.PodName))
+		return nil, fmt.Errorf("invalid cpu request for %s container", opts.PodName)
 	}
 	memRequests, err := resource.ParseQuantity(opts.Resources.MemoryRequests)
 	if err != nil {
-		return nil, errors.New(fmt.Sprintf("invalid memory request for %s container", opts.PodName))
+		return nil, fmt.Errorf("invalid memory request for %s container", opts.PodName)
 	}
 
 	command := []string{"./mizuagent", "--api-server"}
@@ -275,7 +274,7 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 		},
 		{
 			Name:            "basenine",
-			Image:           opts.BasenineImage,
+			Image:           opts.PodImage,
 			ImagePullPolicy: opts.ImagePullPolicy,
 			VolumeMounts:    volumeMounts,
 			ReadinessProbe: &core.Probe{
@@ -299,7 +298,7 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 					"memory": memRequests,
 				},
 			},
-			Command:    []string{"/basenine"},
+			Command:    []string{"basenine"},
 			Args:       []string{"-addr", "0.0.0.0", "-port", shared.BaseninePort, "-persistent"},
 			WorkingDir: shared.DataDirPath,
 		},
@@ -396,7 +395,7 @@ func (provider *Provider) CreatePod(ctx context.Context, namespace string, podSp
 }
 
 func (provider *Provider) CreateDeployment(ctx context.Context, namespace string, deploymentName string, podSpec *core.Pod) (*v1.Deployment, error) {
-	if _, keyExists := podSpec.ObjectMeta.Labels["app"]; keyExists == false {
+	if _, keyExists := podSpec.ObjectMeta.Labels["app"]; !keyExists {
 		return nil, errors.New("pod spec must contain 'app' label")
 	}
 	podTemplate := &core.PodTemplateSpec{
@@ -765,7 +764,7 @@ func (provider *Provider) handleRemovalError(err error) error {
 }
 
 func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error {
-	configMapData := make(map[string]string, 0)
+	configMapData := make(map[string]string)
 	if serializedValidationRules != "" {
 		configMapData[shared.ValidationRulesFileName] = serializedValidationRules
 	}
@@ -855,19 +854,19 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	)
 	cpuLimit, err := resource.ParseQuantity(resources.CpuLimit)
 	if err != nil {
-		return errors.New(fmt.Sprintf("invalid cpu limit for %s container", tapperPodName))
+		return fmt.Errorf("invalid cpu limit for %s container", tapperPodName)
 	}
 	memLimit, err := resource.ParseQuantity(resources.MemoryLimit)
 	if err != nil {
-		return errors.New(fmt.Sprintf("invalid memory limit for %s container", tapperPodName))
+		return fmt.Errorf("invalid memory limit for %s container", tapperPodName)
 	}
 	cpuRequests, err := resource.ParseQuantity(resources.CpuRequests)
 	if err != nil {
-		return errors.New(fmt.Sprintf("invalid cpu request for %s container", tapperPodName))
+		return fmt.Errorf("invalid cpu request for %s container", tapperPodName)
 	}
 	memRequests, err := resource.ParseQuantity(resources.MemoryRequests)
 	if err != nil {
-		return errors.New(fmt.Sprintf("invalid memory request for %s container", tapperPodName))
+		return fmt.Errorf("invalid memory request for %s container", tapperPodName)
 	}
 	agentResourceLimits := core.ResourceList{
 		"cpu":    cpuLimit,

shared/kubernetes/proxy.go

@@ -11,6 +11,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strings"
 	"time"
 
@@ -84,12 +85,21 @@ func getRerouteHttpHandlerMizuStatic(proxyHandler http.Handler, mizuNamespace st
 	})
 }
 
-func NewPortForward(kubernetesProvider *Provider, namespace string, podName string, localPort uint16, cancel context.CancelFunc) error {
-	logger.Log.Debugf("Starting proxy using port-forward method. namespace: [%v], service name: [%s], port: [%v]", namespace, podName, localPort)
+func NewPortForward(kubernetesProvider *Provider, namespace string, podRegex *regexp.Regexp, localPort uint16, ctx context.Context, cancel context.CancelFunc) (*portforward.PortForwarder, error) {
+	pods, err := kubernetesProvider.ListAllRunningPodsMatchingRegex(ctx, podRegex, []string{namespace})
+	if err != nil {
+		return nil, err
+	} else if len(pods) == 0 {
+		return nil, fmt.Errorf("didn't find pod to port-forward")
+	}
+
+	podName := pods[0].Name
+
+	logger.Log.Debugf("Starting proxy using port-forward method. namespace: [%v], pod name: [%s], port: [%v]", namespace, podName, localPort)
 
 	dialer, err := getHttpDialer(kubernetesProvider, namespace, podName)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	stopChan, readyChan := make(chan struct{}, 1), make(chan struct{}, 1)
@@ -97,7 +107,7 @@ func NewPortForward(kubernetesProvider *Provider, namespace string, podName stri
 
 	forwarder, err := portforward.New(dialer, []string{fmt.Sprintf("%d:%d", localPort, shared.DefaultApiServerPort)}, stopChan, readyChan, out, errOut)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	go func() {
@@ -107,7 +117,7 @@ func NewPortForward(kubernetesProvider *Provider, namespace string, podName stri
 		}
 	}()
 
-	return nil
+	return forwarder, nil
 }
 
 func getHttpDialer(kubernetesProvider *Provider, namespace string, podName string) (httpstream.Dialer, error) {
@@ -116,6 +126,7 @@ func getHttpDialer(kubernetesProvider *Provider, namespace string, podName strin
 		logger.Log.Errorf("Error creating http dialer")
 		return nil, err
 	}
+
 	path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward", namespace, podName)
 	hostIP := strings.TrimLeft(kubernetesProvider.clientConfig.Host, "htps:/") // no need specify "t" twice
 	serverURL := url.URL{Scheme: "https", Path: path, Host: hostIP}

shared/kubernetes/utils.go

@@ -10,7 +10,7 @@ import (
 )
 
 func GetNodeHostToTappedPodsMap(tappedPods []core.Pod) map[string][]core.Pod {
-	nodeToTappedPodMap := make(map[string][]core.Pod, 0)
+	nodeToTappedPodMap := make(map[string][]core.Pod)
 	for _, pod := range tappedPods {
 		minimizedPod := getMinimizedPod(pod)
 

shared/kubernetes/watch.go

@@ -4,11 +4,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/shared/debounce"
-	"github.com/up9inc/mizu/shared/logger"
 	"sync"
 	"time"
 
+	"github.com/up9inc/mizu/shared/debounce"
+	"github.com/up9inc/mizu/shared/logger"
+
 	"k8s.io/apimachinery/pkg/watch"
 )
 
@@ -31,7 +32,7 @@ func FilteredWatch(ctx context.Context, watcherCreator WatchCreator, targetNames
 
 		go func(targetNamespace string) {
 			defer wg.Done()
-			watchRestartDebouncer := debounce.NewDebouncer(1 * time.Minute, func() {})
+			watchRestartDebouncer := debounce.NewDebouncer(1*time.Minute, func() {})
 
 			for {
 				watcher, err := watcherCreator.NewWatcher(ctx, targetNamespace)
@@ -44,7 +45,7 @@ func FilteredWatch(ctx context.Context, watcherCreator WatchCreator, targetNames
 				watcher.Stop()
 
 				select {
-				case <- ctx.Done():
+				case <-ctx.Done():
 					return
 				default:
 					break
@@ -55,7 +56,9 @@ func FilteredWatch(ctx context.Context, watcherCreator WatchCreator, targetNames
 					break
 				} else {
 					if !watchRestartDebouncer.IsOn() {
-						watchRestartDebouncer.SetOn()
+						if err := watchRestartDebouncer.SetOn(); err != nil {
+							logger.Log.Error(err)
+						}
 						logger.Log.Debug("k8s watch channel closed, restarting watcher")
 						time.Sleep(time.Second * 5)
 						continue

shared/models.go

@@ -43,6 +43,13 @@ type MizuAgentConfig struct {
 	StandaloneMode         bool          `json:"standaloneMode"`
 	ServiceMap             bool          `json:"serviceMap"`
 	OAS                    bool          `json:"oas"`
+	Elastic                ElasticConfig `json:"elastic"`
+}
+
+type ElasticConfig struct {
+	User     string `yaml:"user,omitempty" default:"" readonly:""`
+	Password string `yaml:"password,omitempty" default:"" readonly:""`
+	Url      string `yaml:"url,omitempty" default:"" readonly:""`
 }
 
 type WebSocketMessageMetadata struct {

tap/api/api.go

@@ -283,7 +283,7 @@ func (h HTTPPayload) MarshalJSON() ([]byte, error) {
 			RawResponse: &HTTPResponseWrapper{Response: h.Data.(*http.Response)},
 		})
 	default:
-		panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s", h.Type))
+		panic(fmt.Sprintf("HTTP payload cannot be marshaled: %v", h.Type))
 	}
 }
 
@@ -313,7 +313,7 @@ type HTTPRequestWrapper struct {
 func (r *HTTPRequestWrapper) MarshalJSON() ([]byte, error) {
 	body, _ := ioutil.ReadAll(r.Request.Body)
 	r.Request.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-	return json.Marshal(&struct {
+	return json.Marshal(&struct { //nolint
 		Body    string `json:"Body,omitempty"`
 		GetBody string `json:"GetBody,omitempty"`
 		Cancel  string `json:"Cancel,omitempty"`
@@ -331,7 +331,7 @@ type HTTPResponseWrapper struct {
 func (r *HTTPResponseWrapper) MarshalJSON() ([]byte, error) {
 	body, _ := ioutil.ReadAll(r.Response.Body)
 	r.Response.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-	return json.Marshal(&struct {
+	return json.Marshal(&struct { //nolint
 		Body    string `json:"Body,omitempty"`
 		GetBody string `json:"GetBody,omitempty"`
 		Cancel  string `json:"Cancel,omitempty"`

tap/extensions/amqp/helpers.go

@@ -24,14 +24,14 @@ var connectionMethodMap = map[int]string{
 	61: "connection unblocked",
 }
 
-var channelMethodMap = map[int]string{
-	10: "channel open",
-	11: "channel open-ok",
-	20: "channel flow",
-	21: "channel flow-ok",
-	40: "channel close",
-	41: "channel close-ok",
-}
+// var channelMethodMap = map[int]string{
+// 	10: "channel open",
+// 	11: "channel open-ok",
+// 	20: "channel flow",
+// 	21: "channel flow-ok",
+// 	40: "channel close",
+// 	41: "channel close-ok",
+// }
 
 var exchangeMethodMap = map[int]string{
 	10: "exchange declare",
@@ -78,14 +78,14 @@ var basicMethodMap = map[int]string{
 	120: "basic nack",
 }
 
-var txMethodMap = map[int]string{
-	10: "tx select",
-	11: "tx select-ok",
-	20: "tx commit",
-	21: "tx commit-ok",
-	30: "tx rollback",
-	31: "tx rollback-ok",
-}
+// var txMethodMap = map[int]string{
+// 	10: "tx select",
+// 	11: "tx select-ok",
+// 	20: "tx commit",
+// 	21: "tx commit-ok",
+// 	30: "tx rollback",
+// 	31: "tx rollback-ok",
+// }
 
 type AMQPWrapper struct {
 	Method  string      `json:"method"`
@@ -550,14 +550,12 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
 		headers := make([]api.TableData, 0)
 		for name, value := range event["serverProperties"].(map[string]interface{}) {
 			var outcome string
-			switch value.(type) {
+			switch v := value.(type) {
 			case string:
-				outcome = value.(string)
-				break
+				outcome = v
 			case map[string]interface{}:
 				x, _ := json.Marshal(value)
 				outcome = string(x)
-				break
 			default:
 				panic("Unknown data type for the server property!")
 			}

tap/extensions/amqp/main.go

@@ -47,7 +47,6 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 
 	var remaining int
 	var header *HeaderFrame
-	var body []byte
 
 	connectionInfo := &api.ConnectionInfo{
 		ClientIP:   tcpID.SrcIP,
@@ -102,13 +101,10 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 				eventBasicPublish.Properties = header.Properties
 			case *BasicDeliver:
 				eventBasicDeliver.Properties = header.Properties
-			default:
-				frame = nil
 			}
 
 		case *BodyFrame:
 			// continue until terminated
-			body = append(body, f.Body...)
 			remaining -= len(f.Body)
 			switch lastMethodFrameMessage.(type) {
 			case *BasicPublish:
@@ -119,9 +115,6 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 				eventBasicDeliver.Body = f.Body
 				superIdentifier.Protocol = &protocol
 				emitAMQP(*eventBasicDeliver, amqpRequest, basicMethodMap[60], connectionInfo, superTimer.CaptureTime, emitter)
-			default:
-				body = nil
-				frame = nil
 			}
 
 		case *MethodFrame:
@@ -211,10 +204,6 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 				}
 				superIdentifier.Protocol = &protocol
 				emitAMQP(*eventConnectionClose, amqpRequest, connectionMethodMap[50], connectionInfo, superTimer.CaptureTime, emitter)
-
-			default:
-				frame = nil
-
 			}
 
 		default:
@@ -231,32 +220,24 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 	switch request["method"] {
 	case basicMethodMap[40]:
 		summary = reqDetails["exchange"].(string)
-		break
 	case basicMethodMap[60]:
 		summary = reqDetails["exchange"].(string)
-		break
 	case exchangeMethodMap[10]:
 		summary = reqDetails["exchange"].(string)
-		break
 	case queueMethodMap[10]:
 		summary = reqDetails["queue"].(string)
-		break
 	case connectionMethodMap[10]:
 		summary = fmt.Sprintf(
 			"%s.%s",
 			strconv.Itoa(int(reqDetails["versionMajor"].(float64))),
 			strconv.Itoa(int(reqDetails["versionMinor"].(float64))),
 		)
-		break
 	case connectionMethodMap[50]:
 		summary = reqDetails["replyText"].(string)
-		break
 	case queueMethodMap[20]:
 		summary = reqDetails["queue"].(string)
-		break
 	case basicMethodMap[20]:
 		summary = reqDetails["queue"].(string)
-		break
 	}
 
 	request["url"] = summary
@@ -288,33 +269,25 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 
 func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
 	bodySize = 0
-	representation := make(map[string]interface{}, 0)
+	representation := make(map[string]interface{})
 	var repRequest []interface{}
 	switch request["method"].(string) {
 	case basicMethodMap[40]:
 		repRequest = representBasicPublish(request)
-		break
 	case basicMethodMap[60]:
 		repRequest = representBasicDeliver(request)
-		break
 	case queueMethodMap[10]:
 		repRequest = representQueueDeclare(request)
-		break
 	case exchangeMethodMap[10]:
 		repRequest = representExchangeDeclare(request)
-		break
 	case connectionMethodMap[10]:
 		repRequest = representConnectionStart(request)
-		break
 	case connectionMethodMap[50]:
 		repRequest = representConnectionClose(request)
-		break
 	case queueMethodMap[20]:
 		repRequest = representQueueBind(request)
-		break
 	case basicMethodMap[20]:
 		repRequest = representBasicConsume(request)
-		break
 	}
 	representation["request"] = repRequest
 	object, err = json.Marshal(representation)

tap/extensions/amqp/read.go

@@ -135,20 +135,6 @@ func readDecimal(r io.Reader) (v Decimal, err error) {
 	return
 }
 
-func readFloat32(r io.Reader) (v float32, err error) {
-	if err = binary.Read(r, binary.BigEndian, &v); err != nil {
-		return
-	}
-	return
-}
-
-func readFloat64(r io.Reader) (v float64, err error) {
-	if err = binary.Read(r, binary.BigEndian, &v); err != nil {
-		return
-	}
-	return
-}
-
 func readTimestamp(r io.Reader) (v time.Time, err error) {
 	var sec int64
 	if err = binary.Read(r, binary.BigEndian, &sec); err != nil {

tap/extensions/amqp/types.go

@@ -78,15 +78,6 @@ type Error struct {
 	Recover bool   // true when this error can be recovered by retrying later or with different parameters
 }
 
-func newError(code uint16, text string) *Error {
-	return &Error{
-		Code:    int(code),
-		Reason:  text,
-		Recover: isSoftExceptionCode(int(code)),
-		Server:  true,
-	}
-}
-
 func (e Error) Error() string {
 	return fmt.Sprintf("Exception (%d) Reason: %q", e.Code, e.Reason)
 }
@@ -262,19 +253,6 @@ func (t Table) Validate() error {
 	return validateField(t)
 }
 
-// Heap interface for maintaining delivery tags
-type tagSet []uint64
-
-func (set tagSet) Len() int              { return len(set) }
-func (set tagSet) Less(i, j int) bool    { return (set)[i] < (set)[j] }
-func (set tagSet) Swap(i, j int)         { (set)[i], (set)[j] = (set)[j], (set)[i] }
-func (set *tagSet) Push(tag interface{}) { *set = append(*set, tag.(uint64)) }
-func (set *tagSet) Pop() interface{} {
-	val := (*set)[len(*set)-1]
-	*set = (*set)[:len(*set)-1]
-	return val
-}
-
 type Message interface {
 	id() (uint16, uint16)
 	wait() bool
@@ -282,12 +260,6 @@ type Message interface {
 	write(io.Writer) error
 }
 
-type messageWithContent interface {
-	Message
-	getContent() (Properties, []byte)
-	setContent(Properties, []byte)
-}
-
 /*
 The base interface implemented as:
 
@@ -322,22 +294,6 @@ type AmqpReader struct {
 	R io.Reader
 }
 
-type writer struct {
-	w io.Writer
-}
-
-// Implements the frame interface for Connection RPC
-type protocolHeader struct{}
-
-func (protocolHeader) write(w io.Writer) error {
-	_, err := w.Write([]byte{'A', 'M', 'Q', 'P', 0, 0, 9, 1})
-	return err
-}
-
-func (protocolHeader) channel() uint16 {
-	panic("only valid as initial handshake")
-}
-
 /*
 Method frames carry the high-level protocol commands (which we call "methods").
 One method frame carries one command.  The method frame payload has this format:

tap/extensions/amqp/write.go

@@ -6,7 +6,6 @@
 package amqp
 
 import (
-	"bufio"
 	"bytes"
 	"encoding/binary"
 	"errors"
@@ -15,18 +14,6 @@ import (
 	"time"
 )
 
-func (w *writer) WriteFrame(frame frame) (err error) {
-	if err = frame.write(w.w); err != nil {
-		return
-	}
-
-	if buf, ok := w.w.(*bufio.Writer); ok {
-		err = buf.Flush()
-	}
-
-	return
-}
-
 func (f *MethodFrame) write(w io.Writer) (err error) {
 	var payload bytes.Buffer
 
@@ -412,5 +399,5 @@ func writeTable(w io.Writer, table Table) (err error) {
 		}
 	}
 
-	return writeLongstr(w, string(buf.Bytes()))
+	return writeLongstr(w, buf.String())
 }

tap/extensions/http/helpers.go

@@ -81,7 +81,7 @@ func representSliceAsTable(slice []interface{}, selectorPrefix string) (represen
 		selector := fmt.Sprintf("%s[%d]", selectorPrefix, i)
 		table = append(table, api.TableData{
 			Name:     strconv.Itoa(i),
-			Value:    item.(interface{}),
+			Value:    item,
 			Selector: selector,
 		})
 	}

tap/extensions/http/main.go

@@ -92,21 +92,30 @@ func (d dissecting) Ping() {
 }
 
 func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
-	isHTTP2, err := checkIsHTTP2Connection(b, isClient)
+	var err error
+	isHTTP2, _ := checkIsHTTP2Connection(b, isClient)
 
 	var http2Assembler *Http2Assembler
 	if isHTTP2 {
-		prepareHTTP2Connection(b, isClient)
+		err = prepareHTTP2Connection(b, isClient)
+		if err != nil {
+			return err
+		}
 		http2Assembler = createHTTP2Assembler(b)
 	}
 
-	dissected := false
 	switchingProtocolsHTTP2 := false
 	for {
 		if switchingProtocolsHTTP2 {
 			switchingProtocolsHTTP2 = false
 			isHTTP2, err = checkIsHTTP2Connection(b, isClient)
-			prepareHTTP2Connection(b, isClient)
+			if err != nil {
+				break
+			}
+			err = prepareHTTP2Connection(b, isClient)
+			if err != nil {
+				break
+			}
 			http2Assembler = createHTTP2Assembler(b)
 		}
 
@@ -121,7 +130,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			} else if err != nil {
 				continue
 			}
-			dissected = true
+			superIdentifier.Protocol = &http11protocol
 		} else if isClient {
 			var req *http.Request
 			switchingProtocolsHTTP2, req, err = handleHTTP1ClientStream(b, tcpID, counterPair, superTimer, emitter, options)
@@ -130,7 +139,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			} else if err != nil {
 				continue
 			}
-			dissected = true
+			superIdentifier.Protocol = &http11protocol
 
 			// In case of an HTTP2 upgrade, duplicate the HTTP1 request into HTTP2 with stream ID 1
 			if switchingProtocolsHTTP2 {
@@ -161,14 +170,14 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			} else if err != nil {
 				continue
 			}
-			dissected = true
+			superIdentifier.Protocol = &http11protocol
 		}
 	}
 
-	if !dissected {
+	if superIdentifier.Protocol == nil {
 		return err
 	}
-	superIdentifier.Protocol = &http11protocol
+
 	return nil
 }
 
@@ -341,11 +350,11 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
 	})
 
 	postData, _ := request["postData"].(map[string]interface{})
-	mimeType, _ := postData["mimeType"]
+	mimeType := postData["mimeType"]
 	if mimeType == nil || len(mimeType.(string)) == 0 {
 		mimeType = "text/html"
 	}
-	text, _ := postData["text"]
+	text := postData["text"]
 	if text != nil {
 		repRequest = append(repRequest, api.SectionData{
 			Type:     api.BODY,
@@ -425,12 +434,12 @@ func representResponse(response map[string]interface{}) (repResponse []interface
 	})
 
 	content, _ := response["content"].(map[string]interface{})
-	mimeType, _ := content["mimeType"]
+	mimeType := content["mimeType"]
 	if mimeType == nil || len(mimeType.(string)) == 0 {
 		mimeType = "text/html"
 	}
-	encoding, _ := content["encoding"]
-	text, _ := content["text"]
+	encoding := content["encoding"]
+	text := content["text"]
 	if text != nil {
 		repResponse = append(repResponse, api.SectionData{
 			Type:     api.BODY,
@@ -446,7 +455,7 @@ func representResponse(response map[string]interface{}) (repResponse []interface
 }
 
 func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
-	representation := make(map[string]interface{}, 0)
+	representation := make(map[string]interface{})
 	repRequest := representRequest(request)
 	repResponse, bodySize := representResponse(response)
 	representation["request"] = repRequest

tap/extensions/http/sensitive_data_cleaner.go

@@ -89,7 +89,7 @@ func filterResponseBody(response *http.Response, options *api.TrafficFilteringOp
 }
 
 func filterHeaders(headers *http.Header) {
-	for key, _ := range *headers {
+	for key := range *headers {
 		if strings.ToLower(key) == userAgent {
 			continue
 		}
@@ -103,7 +103,7 @@ func filterHeaders(headers *http.Header) {
 }
 
 func getContentTypeHeaderValue(headers http.Header) string {
-	for key, _ := range headers {
+	for key := range headers {
 		if strings.ToLower(key) == "content-type" {
 			return headers.Get(key)
 		}

tap/extensions/kafka/buffer.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"math"
 	"sync"
 	"sync/atomic"
 )
@@ -201,25 +200,6 @@ func newPageBuffer() *pageBuffer {
 	return b
 }
 
-func (pb *pageBuffer) refTo(ref *pageRef, begin, end int64) {
-	length := end - begin
-
-	if length > math.MaxUint32 {
-		panic("reference to contiguous buffer pages exceeds the maximum size of 4 GB")
-	}
-
-	ref.pages = append(ref.buffer[:0], pb.pages.slice(begin, end)...)
-	ref.pages.ref()
-	ref.offset = begin
-	ref.length = uint32(length)
-}
-
-func (pb *pageBuffer) ref(begin, end int64) *pageRef {
-	ref := new(pageRef)
-	pb.refTo(ref, begin, end)
-	return ref
-}
-
 func (pb *pageBuffer) unref() {
 	pb.refc.unref(func() {
 		pb.pages.unref()
@@ -353,12 +333,12 @@ func (pb *pageBuffer) Write(b []byte) (int, error) {
 		free := tail.Cap() - tail.Len()
 
 		if len(b) <= free {
-			tail.Write(b)
+			_, _ = tail.Write(b)
 			pb.length += len(b)
 			break
 		}
 
-		tail.Write(b[:free])
+		_, _ = tail.Write(b[:free])
 		b = b[free:]
 
 		pb.length += free
@@ -374,7 +354,7 @@ func (pb *pageBuffer) WriteAt(b []byte, off int64) (int, error) {
 		return n, err
 	}
 	if n < len(b) {
-		pb.Write(b[n:])
+		_, _ = pb.Write(b[n:])
 	}
 	return len(b), nil
 }
@@ -406,12 +386,6 @@ var (
 
 type contiguousPages []*page
 
-func (pages contiguousPages) ref() {
-	for _, p := range pages {
-		p.ref()
-	}
-}
-
 func (pages contiguousPages) unref() {
 	for _, p := range pages {
 		p.unref()
@@ -480,7 +454,6 @@ var (
 )
 
 type pageRef struct {
-	buffer [2]*page
 	pages  contiguousPages
 	offset int64
 	cursor int64
@@ -590,28 +563,6 @@ var (
 	_ io.WriterTo = (*pageRef)(nil)
 )
 
-type pageRefAllocator struct {
-	refs []pageRef
-	head int
-	size int
-}
-
-func (a *pageRefAllocator) newPageRef() *pageRef {
-	if a.head == len(a.refs) {
-		a.refs = make([]pageRef, a.size)
-		a.head = 0
-	}
-	ref := &a.refs[a.head]
-	a.head++
-	return ref
-}
-
-func unref(x interface{}) {
-	if r, _ := x.(interface{ unref() }); r != nil {
-		r.unref()
-	}
-}
-
 func seek(cursor, limit, offset int64, whence int) (int64, error) {
 	switch whence {
 	case io.SeekStart:
@@ -631,15 +582,3 @@ func seek(cursor, limit, offset int64, whence int) (int64, error) {
 	}
 	return offset, nil
 }
-
-func closeBytes(b Bytes) {
-	if b != nil {
-		b.Close()
-	}
-}
-
-func resetBytes(b Bytes) {
-	if r, _ := b.(interface{ Reset() }); r != nil {
-		r.Reset()
-	}
-}

tap/extensions/kafka/compression.go

@@ -1,8 +1,6 @@
 package kafka
 
 import (
-	"errors"
-
 	"github.com/segmentio/kafka-go/compress"
 )
 
@@ -12,19 +10,3 @@ type CompressionCodec = compress.Codec
 
 // TODO: this file should probably go away once the internals of the package
 // have moved to use the protocol package.
-const (
-	compressionCodecMask = 0x07
-)
-
-var (
-	errUnknownCodec = errors.New("the compression code is invalid or its codec has not been imported")
-)
-
-// resolveCodec looks up a codec by Code()
-func resolveCodec(code int8) (CompressionCodec, error) {
-	codec := compress.Compression(code).Codec()
-	if codec == nil {
-		return nil, errUnknownCodec
-	}
-	return codec, nil
-}

tap/extensions/kafka/decode.go

@@ -58,14 +58,6 @@ func (d *decoder) ReadByte() (byte, error) {
 	return c, d.err
 }
 
-func (d *decoder) done() bool {
-	return d.remain == 0 || d.err != nil
-}
-
-func (d *decoder) setCRC(table *crc32.Table) {
-	d.table, d.crc32 = table, 0
-}
-
 func (d *decoder) decodeBool(v value) {
 	v.setBool(d.readBool())
 }
@@ -199,19 +191,6 @@ func (d *decoder) read(n int) []byte {
 	return b
 }
 
-func (d *decoder) writeTo(w io.Writer, n int) {
-	limit := d.remain
-	if n < limit {
-		d.remain = n
-	}
-	c, err := io.Copy(w, d)
-	if int(c) < n && err == nil {
-		err = io.ErrUnexpectedEOF
-	}
-	d.remain = limit - int(c)
-	d.setError(err)
-}
-
 func (d *decoder) setError(err error) {
 	if d.err == nil && err != nil {
 		d.err = err
@@ -272,14 +251,6 @@ func (d *decoder) readString() string {
 	}
 }
 
-func (d *decoder) readVarString() string {
-	if n := d.readVarInt(); n < 0 {
-		return ""
-	} else {
-		return bytesToString(d.read(int(n)))
-	}
-}
-
 func (d *decoder) readCompactString() string {
 	if n := d.readUnsignedVarInt(); n < 1 {
 		return ""
@@ -296,32 +267,6 @@ func (d *decoder) readBytes() []byte {
 	}
 }
 
-func (d *decoder) readBytesTo(w io.Writer) bool {
-	if n := d.readInt32(); n < 0 {
-		return false
-	} else {
-		d.writeTo(w, int(n))
-		return d.err == nil
-	}
-}
-
-func (d *decoder) readVarBytes() []byte {
-	if n := d.readVarInt(); n < 0 {
-		return nil
-	} else {
-		return d.read(int(n))
-	}
-}
-
-func (d *decoder) readVarBytesTo(w io.Writer) bool {
-	if n := d.readVarInt(); n < 0 {
-		return false
-	} else {
-		d.writeTo(w, int(n))
-		return d.err == nil
-	}
-}
-
 func (d *decoder) readCompactBytes() []byte {
 	if n := d.readUnsignedVarInt(); n < 1 {
 		return nil
@@ -330,15 +275,6 @@ func (d *decoder) readCompactBytes() []byte {
 	}
 }
 
-func (d *decoder) readCompactBytesTo(w io.Writer) bool {
-	if n := d.readUnsignedVarInt(); n < 1 {
-		return false
-	} else {
-		d.writeTo(w, int(n-1))
-		return d.err == nil
-	}
-}
-
 func (d *decoder) readVarInt() int64 {
 	n := 11 // varints are at most 11 bytes
 

tap/extensions/kafka/discard.go

@@ -14,37 +14,3 @@ func discardN(r *bufio.Reader, sz int, n int) (int, error) {
 	}
 	return sz - n, err
 }
-
-func discardInt8(r *bufio.Reader, sz int) (int, error) {
-	return discardN(r, sz, 1)
-}
-
-func discardInt16(r *bufio.Reader, sz int) (int, error) {
-	return discardN(r, sz, 2)
-}
-
-func discardInt32(r *bufio.Reader, sz int) (int, error) {
-	return discardN(r, sz, 4)
-}
-
-func discardInt64(r *bufio.Reader, sz int) (int, error) {
-	return discardN(r, sz, 8)
-}
-
-func discardString(r *bufio.Reader, sz int) (int, error) {
-	return readStringWith(r, sz, func(r *bufio.Reader, sz int, n int) (int, error) {
-		if n < 0 {
-			return sz, nil
-		}
-		return discardN(r, sz, n)
-	})
-}
-
-func discardBytes(r *bufio.Reader, sz int) (int, error) {
-	return readBytesWith(r, sz, func(r *bufio.Reader, sz int, n int) (int, error) {
-		if n < 0 {
-			return sz, nil
-		}
-		return discardN(r, sz, n)
-	})
-}

tap/extensions/kafka/encode.go

@@ -3,7 +3,6 @@ package kafka
 import (
 	"bytes"
 	"encoding/binary"
-	"fmt"
 	"hash/crc32"
 	"io"
 	"reflect"
@@ -95,10 +94,6 @@ func (e *encoder) WriteString(s string) (int, error) {
 	return n, nil
 }
 
-func (e *encoder) setCRC(table *crc32.Table) {
-	e.table, e.crc32 = table, 0
-}
-
 func (e *encoder) update(b []byte) {
 	if e.table != nil {
 		e.crc32 = crc32.Update(e.crc32, e.table, b)
@@ -133,10 +128,6 @@ func (e *encoder) encodeString(v value) {
 	e.writeString(v.string())
 }
 
-func (e *encoder) encodeVarString(v value) {
-	e.writeVarString(v.string())
-}
-
 func (e *encoder) encodeCompactString(v value) {
 	e.writeCompactString(v.string())
 }
@@ -145,10 +136,6 @@ func (e *encoder) encodeNullString(v value) {
 	e.writeNullString(v.string())
 }
 
-func (e *encoder) encodeVarNullString(v value) {
-	e.writeVarNullString(v.string())
-}
-
 func (e *encoder) encodeCompactNullString(v value) {
 	e.writeCompactNullString(v.string())
 }
@@ -157,10 +144,6 @@ func (e *encoder) encodeBytes(v value) {
 	e.writeBytes(v.bytes())
 }
 
-func (e *encoder) encodeVarBytes(v value) {
-	e.writeVarBytes(v.bytes())
-}
-
 func (e *encoder) encodeCompactBytes(v value) {
 	e.writeCompactBytes(v.bytes())
 }
@@ -169,10 +152,6 @@ func (e *encoder) encodeNullBytes(v value) {
 	e.writeNullBytes(v.bytes())
 }
 
-func (e *encoder) encodeVarNullBytes(v value) {
-	e.writeVarNullBytes(v.bytes())
-}
-
 func (e *encoder) encodeCompactNullBytes(v value) {
 	e.writeCompactNullBytes(v.bytes())
 }
@@ -228,37 +207,32 @@ func (e *encoder) encodeCompactNullArray(v value, elemType reflect.Type, encodeE
 
 func (e *encoder) writeInt8(i int8) {
 	writeInt8(e.buffer[:1], i)
-	e.Write(e.buffer[:1])
+	_, _ = e.Write(e.buffer[:1])
 }
 
 func (e *encoder) writeInt16(i int16) {
 	writeInt16(e.buffer[:2], i)
-	e.Write(e.buffer[:2])
+	_, _ = e.Write(e.buffer[:2])
 }
 
 func (e *encoder) writeInt32(i int32) {
 	writeInt32(e.buffer[:4], i)
-	e.Write(e.buffer[:4])
+	_, _ = e.Write(e.buffer[:4])
 }
 
 func (e *encoder) writeInt64(i int64) {
 	writeInt64(e.buffer[:8], i)
-	e.Write(e.buffer[:8])
+	_, _ = e.Write(e.buffer[:8])
 }
 
 func (e *encoder) writeString(s string) {
 	e.writeInt16(int16(len(s)))
-	e.WriteString(s)
-}
-
-func (e *encoder) writeVarString(s string) {
-	e.writeVarInt(int64(len(s)))
-	e.WriteString(s)
+	_, _ = e.WriteString(s)
 }
 
 func (e *encoder) writeCompactString(s string) {
 	e.writeUnsignedVarInt(uint64(len(s)) + 1)
-	e.WriteString(s)
+	_, _ = e.WriteString(s)
 }
 
 func (e *encoder) writeNullString(s string) {
@@ -266,16 +240,7 @@ func (e *encoder) writeNullString(s string) {
 		e.writeInt16(-1)
 	} else {
 		e.writeInt16(int16(len(s)))
-		e.WriteString(s)
-	}
-}
-
-func (e *encoder) writeVarNullString(s string) {
-	if s == "" {
-		e.writeVarInt(-1)
-	} else {
-		e.writeVarInt(int64(len(s)))
-		e.WriteString(s)
+		_, _ = e.WriteString(s)
 	}
 }
 
@@ -284,23 +249,18 @@ func (e *encoder) writeCompactNullString(s string) {
 		e.writeUnsignedVarInt(0)
 	} else {
 		e.writeUnsignedVarInt(uint64(len(s)) + 1)
-		e.WriteString(s)
+		_, _ = e.WriteString(s)
 	}
 }
 
 func (e *encoder) writeBytes(b []byte) {
 	e.writeInt32(int32(len(b)))
-	e.Write(b)
-}
-
-func (e *encoder) writeVarBytes(b []byte) {
-	e.writeVarInt(int64(len(b)))
-	e.Write(b)
+	_, _ = e.Write(b)
 }
 
 func (e *encoder) writeCompactBytes(b []byte) {
 	e.writeUnsignedVarInt(uint64(len(b)) + 1)
-	e.Write(b)
+	_, _ = e.Write(b)
 }
 
 func (e *encoder) writeNullBytes(b []byte) {
@@ -308,16 +268,7 @@ func (e *encoder) writeNullBytes(b []byte) {
 		e.writeInt32(-1)
 	} else {
 		e.writeInt32(int32(len(b)))
-		e.Write(b)
-	}
-}
-
-func (e *encoder) writeVarNullBytes(b []byte) {
-	if b == nil {
-		e.writeVarInt(-1)
-	} else {
-		e.writeVarInt(int64(len(b)))
-		e.Write(b)
+		_, _ = e.Write(b)
 	}
 }
 
@@ -326,69 +277,10 @@ func (e *encoder) writeCompactNullBytes(b []byte) {
 		e.writeUnsignedVarInt(0)
 	} else {
 		e.writeUnsignedVarInt(uint64(len(b)) + 1)
-		e.Write(b)
+		_, _ = e.Write(b)
 	}
 }
 
-func (e *encoder) writeBytesFrom(b Bytes) error {
-	size := int64(b.Len())
-	e.writeInt32(int32(size))
-	n, err := io.Copy(e, b)
-	if err == nil && n != size {
-		err = fmt.Errorf("size of bytes does not match the number of bytes that were written (size=%d, written=%d): %w", size, n, io.ErrUnexpectedEOF)
-	}
-	return err
-}
-
-func (e *encoder) writeNullBytesFrom(b Bytes) error {
-	if b == nil {
-		e.writeInt32(-1)
-		return nil
-	} else {
-		size := int64(b.Len())
-		e.writeInt32(int32(size))
-		n, err := io.Copy(e, b)
-		if err == nil && n != size {
-			err = fmt.Errorf("size of nullable bytes does not match the number of bytes that were written (size=%d, written=%d): %w", size, n, io.ErrUnexpectedEOF)
-		}
-		return err
-	}
-}
-
-func (e *encoder) writeVarNullBytesFrom(b Bytes) error {
-	if b == nil {
-		e.writeVarInt(-1)
-		return nil
-	} else {
-		size := int64(b.Len())
-		e.writeVarInt(size)
-		n, err := io.Copy(e, b)
-		if err == nil && n != size {
-			err = fmt.Errorf("size of nullable bytes does not match the number of bytes that were written (size=%d, written=%d): %w", size, n, io.ErrUnexpectedEOF)
-		}
-		return err
-	}
-}
-
-func (e *encoder) writeCompactNullBytesFrom(b Bytes) error {
-	if b == nil {
-		e.writeUnsignedVarInt(0)
-		return nil
-	} else {
-		size := int64(b.Len())
-		e.writeUnsignedVarInt(uint64(size + 1))
-		n, err := io.Copy(e, b)
-		if err == nil && n != size {
-			err = fmt.Errorf("size of compact nullable bytes does not match the number of bytes that were written (size=%d, written=%d): %w", size, n, io.ErrUnexpectedEOF)
-		}
-		return err
-	}
-}
-
-func (e *encoder) writeVarInt(i int64) {
-	e.writeUnsignedVarInt(uint64((i << 1) ^ (i >> 63)))
-}
-
 func (e *encoder) writeUnsignedVarInt(i uint64) {
 	b := e.buffer[:]
 	n := 0
@@ -404,7 +296,7 @@ func (e *encoder) writeUnsignedVarInt(i uint64) {
 		n++
 	}
 
-	e.Write(b[:n])
+	_, _ = e.Write(b[:n])
 }
 
 type encodeFunc func(*encoder, value)
@@ -530,7 +422,7 @@ func structEncodeFuncOf(typ reflect.Type, version int16, flexible bool) encodeFu
 				se := &encoder{writer: buf}
 				f.encode(se, v.fieldByIndex(f.index))
 				e.writeUnsignedVarInt(uint64(buf.Len()))
-				e.Write(buf.Bytes())
+				_, _ = e.Write(buf.Bytes())
 			}
 		}
 	}

tap/extensions/kafka/helpers.go

@@ -346,7 +346,13 @@ func representProduceRequest(data map[string]interface{}) []interface{} {
 			})
 
 			obj, err := oj.ParseString(string(partitionsJson))
+			if err != nil {
+				return rep
+			}
 			recordBatchPath, err := jp.ParseString(`partitionData.records.recordBatch`)
+			if err != nil {
+				return rep
+			}
 			recordBatchresults := recordBatchPath.Get(obj)
 			if len(recordBatchresults) > 0 {
 				rep = append(rep, api.SectionData{
@@ -357,6 +363,9 @@ func representProduceRequest(data map[string]interface{}) []interface{} {
 			}
 
 			recordsPath, err := jp.ParseString(`partitionData.records.recordBatch.record`)
+			if err != nil {
+				return rep
+			}
 			recordsResults := recordsPath.Get(obj)
 			if len(recordsResults) > 0 {
 				records := recordsResults[0].([]interface{})

tap/extensions/kafka/main.go

@@ -81,10 +81,8 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
-		break
 	case ApiVersions:
 		summary = reqDetails["clientID"].(string)
-		break
 	case Produce:
 		_topics := reqDetails["payload"].(map[string]interface{})["topicData"]
 		if _topics == nil {
@@ -97,7 +95,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
-		break
 	case Fetch:
 		_topics := reqDetails["payload"].(map[string]interface{})["topics"]
 		if _topics == nil {
@@ -110,7 +107,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
-		break
 	case ListOffsets:
 		_topics := reqDetails["payload"].(map[string]interface{})["topics"]
 		if _topics == nil {
@@ -123,7 +119,6 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
-		break
 	case CreateTopics:
 		topics := reqDetails["payload"].(map[string]interface{})["topics"].([]interface{})
 		for _, topic := range topics {
@@ -132,13 +127,11 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 		if len(summary) > 0 {
 			summary = summary[:len(summary)-2]
 		}
-		break
 	case DeleteTopics:
 		topicNames := reqDetails["topicNames"].([]string)
 		for _, name := range topicNames {
 			summary += fmt.Sprintf("%s, ", name)
 		}
-		break
 	}
 
 	request["url"] = summary
@@ -173,7 +166,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 
 func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
 	bodySize = 0
-	representation := make(map[string]interface{}, 0)
+	representation := make(map[string]interface{})
 
 	apiKey := ApiKey(request["apiKey"].(float64))
 
@@ -183,31 +176,24 @@ func (d dissecting) Represent(request map[string]interface{}, response map[strin
 	case Metadata:
 		repRequest = representMetadataRequest(request)
 		repResponse = representMetadataResponse(response)
-		break
 	case ApiVersions:
 		repRequest = representApiVersionsRequest(request)
 		repResponse = representApiVersionsResponse(response)
-		break
 	case Produce:
 		repRequest = representProduceRequest(request)
 		repResponse = representProduceResponse(response)
-		break
 	case Fetch:
 		repRequest = representFetchRequest(request)
 		repResponse = representFetchResponse(response)
-		break
 	case ListOffsets:
 		repRequest = representListOffsetsRequest(request)
 		repResponse = representListOffsetsResponse(response)
-		break
 	case CreateTopics:
 		repRequest = representCreateTopicsRequest(request)
 		repResponse = representCreateTopicsResponse(response)
-		break
 	case DeleteTopics:
 		repRequest = representDeleteTopicsRequest(request)
 		repResponse = representDeleteTopicsResponse(response)
-		break
 	}
 
 	representation["request"] = repRequest

tap/extensions/kafka/matcher.go

@@ -26,9 +26,9 @@ func CreateResponseRequestMatcher() requestResponseMatcher {
 func (matcher *requestResponseMatcher) registerRequest(key string, request *Request) *RequestResponsePair {
 	if response, found := matcher.openMessagesMap.LoadAndDelete(key); found {
 		// Check for a situation that only occurs when a Kafka broker is initiating
-		switch response.(type) {
+		switch v := response.(type) {
 		case *Response:
-			return matcher.preparePair(request, response.(*Response))
+			return matcher.preparePair(request, v)
 		}
 	}
 

tap/extensions/kafka/protocol.go

@@ -167,10 +167,6 @@ type messageType struct {
 	encode   encodeFunc
 }
 
-func (t *messageType) new() Message {
-	return reflect.New(t.gotype).Interface().(Message)
-}
-
 type apiType struct {
 	requests  []messageType
 	responses []messageType
@@ -389,16 +385,16 @@ func (b Broker) String() string {
 func (b Broker) Format(w fmt.State, v rune) {
 	switch v {
 	case 'd':
-		io.WriteString(w, itoa(b.ID))
+		_, _ = io.WriteString(w, itoa(b.ID))
 	case 's':
-		io.WriteString(w, b.String())
+		_, _ = io.WriteString(w, b.String())
 	case 'v':
-		io.WriteString(w, itoa(b.ID))
-		io.WriteString(w, " ")
-		io.WriteString(w, b.String())
+		_, _ = io.WriteString(w, itoa(b.ID))
+		_, _ = io.WriteString(w, " ")
+		_, _ = io.WriteString(w, b.String())
 		if b.Rack != "" {
-			io.WriteString(w, " ")
-			io.WriteString(w, b.Rack)
+			_, _ = io.WriteString(w, " ")
+			_, _ = io.WriteString(w, b.Rack)
 		}
 	}
 }

tap/extensions/kafka/protocol_make.go

@@ -99,10 +99,8 @@ func (k apiKey) String() string {
 	return strconv.Itoa(int(k))
 }
 
-type apiVersion int16
-
 const (
-	v0  = 0
+	// v0  = 0
 	v1  = 1
 	v2  = 2
 	v3  = 3
@@ -113,6 +111,7 @@ const (
 	v8  = 8
 	v9  = 9
 	v10 = 10
+	v11 = 11
 )
 
 var apiKeyStrings = [...]string{
@@ -166,35 +165,6 @@ var apiKeyStrings = [...]string{
 	offsetDelete:                "OffsetDelete",
 }
 
-type requestHeader struct {
-	Size          int32
-	ApiKey        int16
-	ApiVersion    int16
-	CorrelationID int32
-	ClientID      string
-}
-
-func sizeofString(s string) int32 {
-	return 2 + int32(len(s))
-}
-
-func (h requestHeader) size() int32 {
-	return 4 + 2 + 2 + 4 + sizeofString(h.ClientID)
-}
-
-// func (h requestHeader) writeTo(wb *writeBuffer) {
-// 	wb.writeInt32(h.Size)
-// 	wb.writeInt16(h.ApiKey)
-// 	wb.writeInt16(h.ApiVersion)
-// 	wb.writeInt32(h.CorrelationID)
-// 	wb.writeString(h.ClientID)
-// }
-
-type request interface {
-	size() int32
-	// writable
-}
-
 func makeInt8(b []byte) int8 {
 	return int8(b[0])
 }
@@ -210,10 +180,3 @@ func makeInt32(b []byte) int32 {
 func makeInt64(b []byte) int64 {
 	return int64(binary.BigEndian.Uint64(b))
 }
-
-func expectZeroSize(sz int, err error) error {
-	if err == nil && sz != 0 {
-		err = fmt.Errorf("reading a response left %d unread bytes", sz)
-	}
-	return err
-}

tap/extensions/kafka/read.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"reflect"
 )
 
 type readable interface {
@@ -42,59 +41,6 @@ func readInt64(r *bufio.Reader, sz int, v *int64) (int, error) {
 	return peekRead(r, sz, 8, func(b []byte) { *v = makeInt64(b) })
 }
 
-func readVarInt(r *bufio.Reader, sz int, v *int64) (remain int, err error) {
-	// Optimistically assume that most of the time, there will be data buffered
-	// in the reader. If this is not the case, the buffer will be refilled after
-	// consuming zero bytes from the input.
-	input, _ := r.Peek(r.Buffered())
-	x := uint64(0)
-	s := uint(0)
-
-	for {
-		if len(input) > sz {
-			input = input[:sz]
-		}
-
-		for i, b := range input {
-			if b < 0x80 {
-				x |= uint64(b) << s
-				*v = int64(x>>1) ^ -(int64(x) & 1)
-				n, err := r.Discard(i + 1)
-				return sz - n, err
-			}
-
-			x |= uint64(b&0x7f) << s
-			s += 7
-		}
-
-		// Make room in the input buffer to load more data from the underlying
-		// stream. The x and s variables are left untouched, ensuring that the
-		// varint decoding can continue on the next loop iteration.
-		n, _ := r.Discard(len(input))
-		sz -= n
-		if sz == 0 {
-			return 0, errShortRead
-		}
-
-		// Fill the buffer: ask for one more byte, but in practice the reader
-		// will load way more from the underlying stream.
-		if _, err := r.Peek(1); err != nil {
-			if err == io.EOF {
-				err = errShortRead
-			}
-			return sz, err
-		}
-
-		// Grab as many bytes as possible from the buffer, then go on to the
-		// next loop iteration which is going to consume it.
-		input, _ = r.Peek(r.Buffered())
-	}
-}
-
-func readBool(r *bufio.Reader, sz int, v *bool) (int, error) {
-	return peekRead(r, sz, 1, func(b []byte) { *v = b[0] != 0 })
-}
-
 func readString(r *bufio.Reader, sz int, v *string) (int, error) {
 	return readStringWith(r, sz, func(r *bufio.Reader, sz int, n int) (remain int, err error) {
 		*v, remain, err = readNewString(r, sz, n)
@@ -179,102 +125,6 @@ func readArrayLen(r *bufio.Reader, sz int, n *int) (int, error) {
 	return sz, nil
 }
 
-func readArrayWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int) (int, error)) (int, error) {
-	var err error
-	var len int32
-
-	if sz, err = readInt32(r, sz, &len); err != nil {
-		return sz, err
-	}
-
-	for n := int(len); n > 0; n-- {
-		if sz, err = cb(r, sz); err != nil {
-			break
-		}
-	}
-
-	return sz, err
-}
-
-func readStringArray(r *bufio.Reader, sz int, v *[]string) (remain int, err error) {
-	var content []string
-	fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {
-		var value string
-		if fnRemain, fnErr = readString(r, size, &value); fnErr != nil {
-			return
-		}
-		content = append(content, value)
-		return
-	}
-	if remain, err = readArrayWith(r, sz, fn); err != nil {
-		return
-	}
-
-	*v = content
-	return
-}
-
-func readMapStringInt32(r *bufio.Reader, sz int, v *map[string][]int32) (remain int, err error) {
-	var len int32
-	if remain, err = readInt32(r, sz, &len); err != nil {
-		return
-	}
-
-	content := make(map[string][]int32, len)
-	for i := 0; i < int(len); i++ {
-		var key string
-		var values []int32
-
-		if remain, err = readString(r, remain, &key); err != nil {
-			return
-		}
-
-		fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {
-			var value int32
-			if fnRemain, fnErr = readInt32(r, size, &value); fnErr != nil {
-				return
-			}
-			values = append(values, value)
-			return
-		}
-		if remain, err = readArrayWith(r, remain, fn); err != nil {
-			return
-		}
-
-		content[key] = values
-	}
-	*v = content
-
-	return
-}
-
-func read(r *bufio.Reader, sz int, a interface{}) (int, error) {
-	switch v := a.(type) {
-	case *int8:
-		return readInt8(r, sz, v)
-	case *int16:
-		return readInt16(r, sz, v)
-	case *int32:
-		return readInt32(r, sz, v)
-	case *int64:
-		return readInt64(r, sz, v)
-	case *bool:
-		return readBool(r, sz, v)
-	case *string:
-		return readString(r, sz, v)
-	case *[]byte:
-		return readBytes(r, sz, v)
-	}
-	switch v := reflect.ValueOf(a).Elem(); v.Kind() {
-	case reflect.Struct:
-		return readStruct(r, sz, v)
-	case reflect.Slice:
-		return readSlice(r, sz, v)
-	default:
-		panic(fmt.Sprintf("unsupported type: %T", a))
-	}
-}
-
 func ReadAll(r *bufio.Reader, sz int, ptrs ...interface{}) (int, error) {
 	var err error
 
@@ -307,333 +157,3 @@ func readPtr(r *bufio.Reader, sz int, ptr interface{}) (int, error) {
 		panic(fmt.Sprintf("unsupported type: %T", v))
 	}
 }
-
-func readStruct(r *bufio.Reader, sz int, v reflect.Value) (int, error) {
-	var err error
-	for i, n := 0, v.NumField(); i != n; i++ {
-		if sz, err = read(r, sz, v.Field(i).Addr().Interface()); err != nil {
-			return sz, err
-		}
-	}
-	return sz, nil
-}
-
-func readSlice(r *bufio.Reader, sz int, v reflect.Value) (int, error) {
-	var err error
-	var len int32
-
-	if sz, err = readInt32(r, sz, &len); err != nil {
-		return sz, err
-	}
-
-	if n := int(len); n < 0 {
-		v.Set(reflect.Zero(v.Type()))
-	} else {
-		v.Set(reflect.MakeSlice(v.Type(), n, n))
-
-		for i := 0; i != n; i++ {
-			if sz, err = read(r, sz, v.Index(i).Addr().Interface()); err != nil {
-				return sz, err
-			}
-		}
-	}
-
-	return sz, nil
-}
-
-func readFetchResponseHeaderV2(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) {
-	var n int32
-	var p struct {
-		Partition           int32
-		ErrorCode           int16
-		HighwaterMarkOffset int64
-		MessageSetSize      int32
-	}
-
-	if remain, err = readInt32(r, size, &throttle); err != nil {
-		return
-	}
-
-	if remain, err = readInt32(r, remain, &n); err != nil {
-		return
-	}
-
-	// This error should never trigger, unless there's a bug in the kafka client
-	// or server.
-	if n != 1 {
-		err = fmt.Errorf("1 kafka topic was expected in the fetch response but the client received %d", n)
-		return
-	}
-
-	// We ignore the topic name because we've requests messages for a single
-	// topic, unless there's a bug in the kafka server we will have received
-	// the name of the topic that we requested.
-	if remain, err = discardString(r, remain); err != nil {
-		return
-	}
-
-	if remain, err = readInt32(r, remain, &n); err != nil {
-		return
-	}
-
-	// This error should never trigger, unless there's a bug in the kafka client
-	// or server.
-	if n != 1 {
-		err = fmt.Errorf("1 kafka partition was expected in the fetch response but the client received %d", n)
-		return
-	}
-
-	if remain, err = read(r, remain, &p); err != nil {
-		return
-	}
-
-	if p.ErrorCode != 0 {
-		err = Error(p.ErrorCode)
-		return
-	}
-
-	// This error should never trigger, unless there's a bug in the kafka client
-	// or server.
-	if remain != int(p.MessageSetSize) {
-		err = fmt.Errorf("the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)", p.MessageSetSize, remain)
-		return
-	}
-
-	watermark = p.HighwaterMarkOffset
-	return
-}
-
-func readFetchResponseHeaderV5(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) {
-	var n int32
-	type AbortedTransaction struct {
-		ProducerId  int64
-		FirstOffset int64
-	}
-	var p struct {
-		Partition           int32
-		ErrorCode           int16
-		HighwaterMarkOffset int64
-		LastStableOffset    int64
-		LogStartOffset      int64
-	}
-	var messageSetSize int32
-	var abortedTransactions []AbortedTransaction
-
-	if remain, err = readInt32(r, size, &throttle); err != nil {
-		return
-	}
-
-	if remain, err = readInt32(r, remain, &n); err != nil {
-		return
-	}
-
-	// This error should never trigger, unless there's a bug in the kafka client
-	// or server.
-	if n != 1 {
-		err = fmt.Errorf("1 kafka topic was expected in the fetch response but the client received %d", n)
-		return
-	}
-
-	// We ignore the topic name because we've requests messages for a single
-	// topic, unless there's a bug in the kafka server we will have received
-	// the name of the topic that we requested.
-	if remain, err = discardString(r, remain); err != nil {
-		return
-	}
-
-	if remain, err = readInt32(r, remain, &n); err != nil {
-		return
-	}
-
-	// This error should never trigger, unless there's a bug in the kafka client
-	// or server.
-	if n != 1 {
-		err = fmt.Errorf("1 kafka partition was expected in the fetch response but the client received %d", n)
-		return
-	}
-
-	if remain, err = read(r, remain, &p); err != nil {
-		return
-	}
-
-	var abortedTransactionLen int
-	if remain, err = readArrayLen(r, remain, &abortedTransactionLen); err != nil {
-		return
-	}
-
-	if abortedTransactionLen == -1 {
-		abortedTransactions = nil
-	} else {
-		abortedTransactions = make([]AbortedTransaction, abortedTransactionLen)
-		for i := 0; i < abortedTransactionLen; i++ {
-			if remain, err = read(r, remain, &abortedTransactions[i]); err != nil {
-				return
-			}
-		}
-	}
-
-	if p.ErrorCode != 0 {
-		err = Error(p.ErrorCode)
-		return
-	}
-
-	remain, err = readInt32(r, remain, &messageSetSize)
-	if err != nil {
-		return
-	}
-
-	// This error should never trigger, unless there's a bug in the kafka client
-	// or server.
-	if remain != int(messageSetSize) {
-		err = fmt.Errorf("the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)", messageSetSize, remain)
-		return
-	}
-
-	watermark = p.HighwaterMarkOffset
-	return
-
-}
-
-func readFetchResponseHeaderV10(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) {
-	var n int32
-	var errorCode int16
-	type AbortedTransaction struct {
-		ProducerId  int64
-		FirstOffset int64
-	}
-	var p struct {
-		Partition           int32
-		ErrorCode           int16
-		HighwaterMarkOffset int64
-		LastStableOffset    int64
-		LogStartOffset      int64
-	}
-	var messageSetSize int32
-	var abortedTransactions []AbortedTransaction
-
-	if remain, err = readInt32(r, size, &throttle); err != nil {
-		return
-	}
-
-	if remain, err = readInt16(r, remain, &errorCode); err != nil {
-		return
-	}
-	if errorCode != 0 {
-		err = Error(errorCode)
-		return
-	}
-
-	if remain, err = discardInt32(r, remain); err != nil {
-		return
-	}
-
-	if remain, err = readInt32(r, remain, &n); err != nil {
-		return
-	}
-
-	// This error should never trigger, unless there's a bug in the kafka client
-	// or server.
-	if n != 1 {
-		err = fmt.Errorf("1 kafka topic was expected in the fetch response but the client received %d", n)
-		return
-	}
-
-	// We ignore the topic name because we've requests messages for a single
-	// topic, unless there's a bug in the kafka server we will have received
-	// the name of the topic that we requested.
-	if remain, err = discardString(r, remain); err != nil {
-		return
-	}
-
-	if remain, err = readInt32(r, remain, &n); err != nil {
-		return
-	}
-
-	// This error should never trigger, unless there's a bug in the kafka client
-	// or server.
-	if n != 1 {
-		err = fmt.Errorf("1 kafka partition was expected in the fetch response but the client received %d", n)
-		return
-	}
-
-	if remain, err = read(r, remain, &p); err != nil {
-		return
-	}
-
-	var abortedTransactionLen int
-	if remain, err = readArrayLen(r, remain, &abortedTransactionLen); err != nil {
-		return
-	}
-
-	if abortedTransactionLen == -1 {
-		abortedTransactions = nil
-	} else {
-		abortedTransactions = make([]AbortedTransaction, abortedTransactionLen)
-		for i := 0; i < abortedTransactionLen; i++ {
-			if remain, err = read(r, remain, &abortedTransactions[i]); err != nil {
-				return
-			}
-		}
-	}
-
-	if p.ErrorCode != 0 {
-		err = Error(p.ErrorCode)
-		return
-	}
-
-	remain, err = readInt32(r, remain, &messageSetSize)
-	if err != nil {
-		return
-	}
-
-	// This error should never trigger, unless there's a bug in the kafka client
-	// or server.
-	if remain != int(messageSetSize) {
-		err = fmt.Errorf("the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)", messageSetSize, remain)
-		return
-	}
-
-	watermark = p.HighwaterMarkOffset
-	return
-
-}
-
-func readMessageHeader(r *bufio.Reader, sz int) (offset int64, attributes int8, timestamp int64, remain int, err error) {
-	var version int8
-
-	if remain, err = readInt64(r, sz, &offset); err != nil {
-		return
-	}
-
-	// On discarding the message size and CRC:
-	// ---------------------------------------
-	//
-	// - Not sure why kafka gives the message size here, we already have the
-	// number of remaining bytes in the response and kafka should only truncate
-	// the trailing message.
-	//
-	// - TCP is already taking care of ensuring data integrity, no need to
-	// waste resources doing it a second time so we just skip the message CRC.
-	//
-	if remain, err = discardN(r, remain, 8); err != nil {
-		return
-	}
-
-	if remain, err = readInt8(r, remain, &version); err != nil {
-		return
-	}
-
-	if remain, err = readInt8(r, remain, &attributes); err != nil {
-		return
-	}
-
-	switch version {
-	case 0:
-	case 1:
-		remain, err = readInt64(r, remain, &timestamp)
-	default:
-		err = fmt.Errorf("unsupported message version %d found in fetch response", version)
-	}
-
-	return
-}

tap/extensions/kafka/record.go

@@ -110,25 +110,6 @@ type RecordSet struct {
 	Records RecordReader
 }
 
-// bufferedReader is an interface implemented by types like bufio.Reader, which
-// we use to optimize prefix reads by accessing the internal buffer directly
-// through calls to Peek.
-type bufferedReader interface {
-	Discard(int) (int, error)
-	Peek(int) ([]byte, error)
-}
-
-// bytesBuffer is an interface implemented by types like bytes.Buffer, which we
-// use to optimize prefix reads by accessing the internal buffer directly
-// through calls to Bytes.
-type bytesBuffer interface {
-	Bytes() []byte
-}
-
-// magicByteOffset is the position of the magic byte in all versions of record
-// sets in the kafka protocol.
-const magicByteOffset = 16
-
 // ReadFrom reads the representation of a record set from r into rs, returning
 // the number of bytes consumed from r, and an non-nil error if the record set
 // could not be read.
@@ -292,23 +273,7 @@ func (rs *RecordSet) WriteTo(w io.Writer) (int64, error) {
 	return 0, nil
 }
 
-func makeTime(t int64) time.Time {
-	return time.Unix(t/1000, (t%1000)*int64(time.Millisecond))
-}
-
-func timestamp(t time.Time) int64 {
-	if t.IsZero() {
-		return 0
-	}
-	return t.UnixNano() / int64(time.Millisecond)
-}
-
 func packUint32(u uint32) (b [4]byte) {
 	binary.BigEndian.PutUint32(b[:], u)
 	return
 }
-
-func packUint64(u uint64) (b [8]byte) {
-	binary.BigEndian.PutUint64(b[:], u)
-	return
-}

tap/extensions/kafka/reflect.go

@@ -34,10 +34,6 @@ func valueOf(x interface{}) value {
 	return value{val: reflect.ValueOf(x).Elem()}
 }
 
-func makeValue(t reflect.Type) value {
-	return value{val: reflect.New(t).Elem()}
-}
-
 func (v value) bool() bool { return v.val.Bool() }
 
 func (v value) int8() int8 { return int8(v.int64()) }
@@ -54,7 +50,7 @@ func (v value) bytes() []byte { return v.val.Bytes() }
 
 func (v value) iface(t reflect.Type) interface{} { return v.val.Addr().Interface() }
 
-func (v value) array(t reflect.Type) array { return array{val: v.val} }
+func (v value) array(t reflect.Type) array { return array{val: v.val} } //nolint
 
 func (v value) setBool(b bool) { v.val.SetBool(b) }
 

tap/extensions/kafka/request.go

@@ -64,19 +64,19 @@ func ReadRequest(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer) (api
 	case Metadata:
 		var mt interface{}
 		var metadataRequest interface{}
-		if apiVersion >= 11 {
+		if apiVersion >= v11 {
 			types := makeTypes(reflect.TypeOf(&MetadataRequestV11{}).Elem())
 			mt = types[0]
 			metadataRequest = &MetadataRequestV11{}
-		} else if apiVersion >= 10 {
+		} else if apiVersion >= v10 {
 			types := makeTypes(reflect.TypeOf(&MetadataRequestV10{}).Elem())
 			mt = types[0]
 			metadataRequest = &MetadataRequestV10{}
-		} else if apiVersion >= 8 {
+		} else if apiVersion >= v8 {
 			types := makeTypes(reflect.TypeOf(&MetadataRequestV8{}).Elem())
 			mt = types[0]
 			metadataRequest = &MetadataRequestV8{}
-		} else if apiVersion >= 4 {
+		} else if apiVersion >= v4 {
 			types := makeTypes(reflect.TypeOf(&MetadataRequestV4{}).Elem())
 			mt = types[0]
 			metadataRequest = &MetadataRequestV4{}
@@ -87,11 +87,10 @@ func ReadRequest(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer) (api
 		}
 		mt.(messageType).decode(d, valueOf(metadataRequest))
 		payload = metadataRequest
-		break
 	case ApiVersions:
 		var mt interface{}
 		var apiVersionsRequest interface{}
-		if apiVersion >= 3 {
+		if apiVersion >= v3 {
 			types := makeTypes(reflect.TypeOf(&ApiVersionsRequestV3{}).Elem())
 			mt = types[0]
 			apiVersionsRequest = &ApiVersionsRequestV3{}
@@ -102,11 +101,10 @@ func ReadRequest(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer) (api
 		}
 		mt.(messageType).decode(d, valueOf(apiVersionsRequest))
 		payload = apiVersionsRequest
-		break
 	case Produce:
 		var mt interface{}
 		var produceRequest interface{}
-		if apiVersion >= 3 {
+		if apiVersion >= v3 {
 			types := makeTypes(reflect.TypeOf(&ProduceRequestV3{}).Elem())
 			mt = types[0]
 			produceRequest = &ProduceRequestV3{}
@@ -117,7 +115,6 @@ func ReadRequest(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer) (api
 		}
 		mt.(messageType).decode(d, valueOf(produceRequest))
 		payload = produceRequest
-		break
 	case Fetch:
 		var mt interface{}
 		var fetchRequest interface{}
@@ -125,23 +122,23 @@ func ReadRequest(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer) (api
 			types := makeTypes(reflect.TypeOf(&FetchRequestV11{}).Elem())
 			mt = types[0]
 			fetchRequest = &FetchRequestV11{}
-		} else if apiVersion >= 9 {
+		} else if apiVersion >= v9 {
 			types := makeTypes(reflect.TypeOf(&FetchRequestV9{}).Elem())
 			mt = types[0]
 			fetchRequest = &FetchRequestV9{}
-		} else if apiVersion >= 7 {
+		} else if apiVersion >= v7 {
 			types := makeTypes(reflect.TypeOf(&FetchRequestV7{}).Elem())
 			mt = types[0]
 			fetchRequest = &FetchRequestV7{}
-		} else if apiVersion >= 5 {
+		} else if apiVersion >= v5 {
 			types := makeTypes(reflect.TypeOf(&FetchRequestV5{}).Elem())
 			mt = types[0]
 			fetchRequest = &FetchRequestV5{}
-		} else if apiVersion >= 4 {
+		} else if apiVersion >= v4 {
 			types := makeTypes(reflect.TypeOf(&FetchRequestV4{}).Elem())
 			mt = types[0]
 			fetchRequest = &FetchRequestV4{}
-		} else if apiVersion >= 3 {
+		} else if apiVersion >= v3 {
 			types := makeTypes(reflect.TypeOf(&FetchRequestV3{}).Elem())
 			mt = types[0]
 			fetchRequest = &FetchRequestV3{}
@@ -152,19 +149,18 @@ func ReadRequest(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer) (api
 		}
 		mt.(messageType).decode(d, valueOf(fetchRequest))
 		payload = fetchRequest
-		break
 	case ListOffsets:
 		var mt interface{}
 		var listOffsetsRequest interface{}
-		if apiVersion >= 4 {
+		if apiVersion >= v4 {
 			types := makeTypes(reflect.TypeOf(&ListOffsetsRequestV4{}).Elem())
 			mt = types[0]
 			listOffsetsRequest = &ListOffsetsRequestV4{}
-		} else if apiVersion >= 2 {
+		} else if apiVersion >= v2 {
 			types := makeTypes(reflect.TypeOf(&ListOffsetsRequestV2{}).Elem())
 			mt = types[0]
 			listOffsetsRequest = &ListOffsetsRequestV2{}
-		} else if apiVersion >= 1 {
+		} else if apiVersion >= v1 {
 			types := makeTypes(reflect.TypeOf(&ListOffsetsRequestV1{}).Elem())
 			mt = types[0]
 			listOffsetsRequest = &ListOffsetsRequestV1{}
@@ -175,11 +171,10 @@ func ReadRequest(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer) (api
 		}
 		mt.(messageType).decode(d, valueOf(listOffsetsRequest))
 		payload = listOffsetsRequest
-		break
 	case CreateTopics:
 		var mt interface{}
 		var createTopicsRequest interface{}
-		if apiVersion >= 1 {
+		if apiVersion >= v1 {
 			types := makeTypes(reflect.TypeOf(&CreateTopicsRequestV1{}).Elem())
 			mt = types[0]
 			createTopicsRequest = &CreateTopicsRequestV1{}
@@ -190,11 +185,10 @@ func ReadRequest(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer) (api
 		}
 		mt.(messageType).decode(d, valueOf(createTopicsRequest))
 		payload = createTopicsRequest
-		break
 	case DeleteTopics:
 		var mt interface{}
 		var deleteTopicsRequest interface{}
-		if apiVersion >= 6 {
+		if apiVersion >= v6 {
 			types := makeTypes(reflect.TypeOf(&DeleteTopicsRequestV6{}).Elem())
 			mt = types[0]
 			deleteTopicsRequest = &DeleteTopicsRequestV6{}
@@ -285,7 +279,7 @@ func WriteRequest(w io.Writer, apiVersion int16, correlationID int32, clientID s
 
 	if err == nil {
 		size := packUint32(uint32(b.Size()) - 4)
-		b.WriteAt(size[:], 0)
+		_, _ = b.WriteAt(size[:], 0)
 		_, err = b.WriteTo(w)
 	}
 

tap/extensions/kafka/response.go

@@ -62,35 +62,35 @@ func ReadResponse(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer, emi
 	case Metadata:
 		var mt interface{}
 		var metadataResponse interface{}
-		if apiVersion >= 11 {
+		if apiVersion >= v11 {
 			types := makeTypes(reflect.TypeOf(&MetadataResponseV11{}).Elem())
 			mt = types[0]
 			metadataResponse = &MetadataResponseV11{}
-		} else if apiVersion >= 10 {
+		} else if apiVersion >= v10 {
 			types := makeTypes(reflect.TypeOf(&MetadataResponseV10{}).Elem())
 			mt = types[0]
 			metadataResponse = &MetadataResponseV10{}
-		} else if apiVersion >= 8 {
+		} else if apiVersion >= v8 {
 			types := makeTypes(reflect.TypeOf(&MetadataResponseV8{}).Elem())
 			mt = types[0]
 			metadataResponse = &MetadataResponseV8{}
-		} else if apiVersion >= 7 {
+		} else if apiVersion >= v7 {
 			types := makeTypes(reflect.TypeOf(&MetadataResponseV7{}).Elem())
 			mt = types[0]
 			metadataResponse = &MetadataResponseV7{}
-		} else if apiVersion >= 5 {
+		} else if apiVersion >= v5 {
 			types := makeTypes(reflect.TypeOf(&MetadataResponseV5{}).Elem())
 			mt = types[0]
 			metadataResponse = &MetadataResponseV5{}
-		} else if apiVersion >= 3 {
+		} else if apiVersion >= v3 {
 			types := makeTypes(reflect.TypeOf(&MetadataResponseV3{}).Elem())
 			mt = types[0]
 			metadataResponse = &MetadataResponseV3{}
-		} else if apiVersion >= 2 {
+		} else if apiVersion >= v2 {
 			types := makeTypes(reflect.TypeOf(&MetadataResponseV2{}).Elem())
 			mt = types[0]
 			metadataResponse = &MetadataResponseV2{}
-		} else if apiVersion >= 1 {
+		} else if apiVersion >= v1 {
 			types := makeTypes(reflect.TypeOf(&MetadataResponseV1{}).Elem())
 			mt = types[0]
 			metadataResponse = &MetadataResponseV1{}
@@ -101,11 +101,10 @@ func ReadResponse(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer, emi
 		}
 		mt.(messageType).decode(d, valueOf(metadataResponse))
 		reqResPair.Response.Payload = metadataResponse
-		break
 	case ApiVersions:
 		var mt interface{}
 		var apiVersionsResponse interface{}
-		if apiVersion >= 1 {
+		if apiVersion >= v1 {
 			types := makeTypes(reflect.TypeOf(&ApiVersionsResponseV1{}).Elem())
 			mt = types[0]
 			apiVersionsResponse = &ApiVersionsResponseV1{}
@@ -116,23 +115,22 @@ func ReadResponse(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer, emi
 		}
 		mt.(messageType).decode(d, valueOf(apiVersionsResponse))
 		reqResPair.Response.Payload = apiVersionsResponse
-		break
 	case Produce:
 		var mt interface{}
 		var produceResponse interface{}
-		if apiVersion >= 8 {
+		if apiVersion >= v8 {
 			types := makeTypes(reflect.TypeOf(&ProduceResponseV8{}).Elem())
 			mt = types[0]
 			produceResponse = &ProduceResponseV8{}
-		} else if apiVersion >= 5 {
+		} else if apiVersion >= v5 {
 			types := makeTypes(reflect.TypeOf(&ProduceResponseV5{}).Elem())
 			mt = types[0]
 			produceResponse = &ProduceResponseV5{}
-		} else if apiVersion >= 2 {
+		} else if apiVersion >= v2 {
 			types := makeTypes(reflect.TypeOf(&ProduceResponseV2{}).Elem())
 			mt = types[0]
 			produceResponse = &ProduceResponseV2{}
-		} else if apiVersion >= 1 {
+		} else if apiVersion >= v1 {
 			types := makeTypes(reflect.TypeOf(&ProduceResponseV1{}).Elem())
 			mt = types[0]
 			produceResponse = &ProduceResponseV1{}
@@ -143,27 +141,26 @@ func ReadResponse(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer, emi
 		}
 		mt.(messageType).decode(d, valueOf(produceResponse))
 		reqResPair.Response.Payload = produceResponse
-		break
 	case Fetch:
 		var mt interface{}
 		var fetchResponse interface{}
-		if apiVersion >= 11 {
+		if apiVersion >= v11 {
 			types := makeTypes(reflect.TypeOf(&FetchResponseV11{}).Elem())
 			mt = types[0]
 			fetchResponse = &FetchResponseV11{}
-		} else if apiVersion >= 7 {
+		} else if apiVersion >= v7 {
 			types := makeTypes(reflect.TypeOf(&FetchResponseV7{}).Elem())
 			mt = types[0]
 			fetchResponse = &FetchResponseV7{}
-		} else if apiVersion >= 5 {
+		} else if apiVersion >= v5 {
 			types := makeTypes(reflect.TypeOf(&FetchResponseV5{}).Elem())
 			mt = types[0]
 			fetchResponse = &FetchResponseV5{}
-		} else if apiVersion >= 4 {
+		} else if apiVersion >= v4 {
 			types := makeTypes(reflect.TypeOf(&FetchResponseV4{}).Elem())
 			mt = types[0]
 			fetchResponse = &FetchResponseV4{}
-		} else if apiVersion >= 1 {
+		} else if apiVersion >= v1 {
 			types := makeTypes(reflect.TypeOf(&FetchResponseV1{}).Elem())
 			mt = types[0]
 			fetchResponse = &FetchResponseV1{}
@@ -174,19 +171,18 @@ func ReadResponse(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer, emi
 		}
 		mt.(messageType).decode(d, valueOf(fetchResponse))
 		reqResPair.Response.Payload = fetchResponse
-		break
 	case ListOffsets:
 		var mt interface{}
 		var listOffsetsResponse interface{}
-		if apiVersion >= 4 {
+		if apiVersion >= v4 {
 			types := makeTypes(reflect.TypeOf(&ListOffsetsResponseV4{}).Elem())
 			mt = types[0]
 			listOffsetsResponse = &ListOffsetsResponseV4{}
-		} else if apiVersion >= 2 {
+		} else if apiVersion >= v2 {
 			types := makeTypes(reflect.TypeOf(&ListOffsetsResponseV2{}).Elem())
 			mt = types[0]
 			listOffsetsResponse = &ListOffsetsResponseV2{}
-		} else if apiVersion >= 1 {
+		} else if apiVersion >= v1 {
 			types := makeTypes(reflect.TypeOf(&ListOffsetsResponseV1{}).Elem())
 			mt = types[0]
 			listOffsetsResponse = &ListOffsetsResponseV1{}
@@ -200,19 +196,19 @@ func ReadResponse(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer, emi
 	case CreateTopics:
 		var mt interface{}
 		var createTopicsResponse interface{}
-		if apiVersion >= 7 {
+		if apiVersion >= v7 {
 			types := makeTypes(reflect.TypeOf(&CreateTopicsResponseV0{}).Elem())
 			mt = types[0]
 			createTopicsResponse = &CreateTopicsResponseV0{}
-		} else if apiVersion >= 5 {
+		} else if apiVersion >= v5 {
 			types := makeTypes(reflect.TypeOf(&CreateTopicsResponseV5{}).Elem())
 			mt = types[0]
 			createTopicsResponse = &CreateTopicsResponseV5{}
-		} else if apiVersion >= 2 {
+		} else if apiVersion >= v2 {
 			types := makeTypes(reflect.TypeOf(&CreateTopicsResponseV2{}).Elem())
 			mt = types[0]
 			createTopicsResponse = &CreateTopicsResponseV2{}
-		} else if apiVersion >= 1 {
+		} else if apiVersion >= v1 {
 			types := makeTypes(reflect.TypeOf(&CreateTopicsResponseV1{}).Elem())
 			mt = types[0]
 			createTopicsResponse = &CreateTopicsResponseV1{}
@@ -223,19 +219,18 @@ func ReadResponse(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer, emi
 		}
 		mt.(messageType).decode(d, valueOf(createTopicsResponse))
 		reqResPair.Response.Payload = createTopicsResponse
-		break
 	case DeleteTopics:
 		var mt interface{}
 		var deleteTopicsResponse interface{}
-		if apiVersion >= 6 {
+		if apiVersion >= v6 {
 			types := makeTypes(reflect.TypeOf(&DeleteTopicsReponseV6{}).Elem())
 			mt = types[0]
 			deleteTopicsResponse = &DeleteTopicsReponseV6{}
-		} else if apiVersion >= 5 {
+		} else if apiVersion >= v5 {
 			types := makeTypes(reflect.TypeOf(&DeleteTopicsReponseV5{}).Elem())
 			mt = types[0]
 			deleteTopicsResponse = &DeleteTopicsReponseV5{}
-		} else if apiVersion >= 1 {
+		} else if apiVersion >= v1 {
 			types := makeTypes(reflect.TypeOf(&DeleteTopicsReponseV1{}).Elem())
 			mt = types[0]
 			deleteTopicsResponse = &DeleteTopicsReponseV1{}
@@ -251,10 +246,10 @@ func ReadResponse(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer, emi
 	}
 
 	connectionInfo := &api.ConnectionInfo{
-		ClientIP:   tcpID.SrcIP,
-		ClientPort: tcpID.SrcPort,
-		ServerIP:   tcpID.DstIP,
-		ServerPort: tcpID.DstPort,
+		ClientIP:   tcpID.DstIP,
+		ClientPort: tcpID.DstPort,
+		ServerIP:   tcpID.SrcIP,
+		ServerPort: tcpID.SrcPort,
 		IsOutgoing: true,
 	}
 
@@ -337,7 +332,7 @@ func WriteResponse(w io.Writer, apiVersion int16, correlationID int32, msg Messa
 
 	if err == nil {
 		size := packUint32(uint32(b.Size()) - 4)
-		b.WriteAt(size[:], 0)
+		_, _ = b.WriteAt(size[:], 0)
 		_, err = b.WriteTo(w)
 	}
 

tap/extensions/kafka/structs.go

@@ -296,8 +296,6 @@ type MessageV0 struct {
 	Set              *MessageSet `json:"set"`              // the message set a message might wrap
 	Version          int8        `json:"version"`          // v1 requires Kafka 0.10
 	Timestamp        time.Time   `json:"timestamp"`        // the timestamp of the message (version 1+ only)
-
-	compressedSize int // used for computing the compression ratio metrics
 }
 
 // MessageBlock represents a part of request with message

tap/extensions/redis/main.go

@@ -52,9 +52,13 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 		}
 
 		if isClient {
-			handleClientStream(tcpID, counterPair, superTimer, emitter, redisPacket)
+			err = handleClientStream(tcpID, counterPair, superTimer, emitter, redisPacket)
 		} else {
-			handleServerStream(tcpID, counterPair, superTimer, emitter, redisPacket)
+			err = handleServerStream(tcpID, counterPair, superTimer, emitter, redisPacket)
+		}
+
+		if err != nil {
+			return err
 		}
 	}
 }
@@ -108,7 +112,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 
 func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
 	bodySize = 0
-	representation := make(map[string]interface{}, 0)
+	representation := make(map[string]interface{})
 	repRequest := representGeneric(request, `request.`)
 	repResponse := representGeneric(response, `response.`)
 	representation["request"] = repRequest