Changed OAS generator to get entries by push (#1103)

* Fix `checkFilterByMethod` in acceptance tests

* #run_acceptance_tests

* Add `numberOfRecords` parameter to test specs #run_acceptance_tests

* Fix the values #run_acceptance_tests

* Fix the values #run_acceptance_tests

* Fix #run_acceptance_tests

* #run_acceptance_tests

* Reduce value duplication #run_acceptance_tests

Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>
Co-authored-by: Roee Gadot <roee.gadot@up9.com>

.github/workflows/acceptance_tests.yml

@@ -8,6 +8,9 @@ on:
     branches:
       - 'develop'
 
+env:
+  MIZU_CI_IMAGE: mizu/ci:0.0
+
 jobs:
   run-acceptance-tests:
     name: Run acceptance tests
@@ -21,6 +24,19 @@ jobs:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          load: true
+          tags: ${{ env.MIZU_CI_IMAGE }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Setup acceptance test
         run: ./acceptanceTests/setup.sh
 
@@ -43,7 +59,7 @@ jobs:
         with:
           status: ${{ job.status }}
           notification_title: 'Mizu {workflow} has {status_message}'
-          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit <{commit_url}|{commit_sha}> by ${{ github.event.head_commit.committer.name }} <${{ github.event.head_commit.committer.email }}> ```${{ github.event.head_commit.message }}```'
+          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit <{commit_url}|{commit_sha}> by ${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}> ```${{ github.event.head_commit.message }}```'
           footer: 'Linked Repo <{repo_url}|{repo}>'
           notify_when: 'failure'
         env:

.github/workflows/acceptance_tests_on_pr.yml

@@ -0,0 +1,54 @@
+name: Acceptance tests on PR
+
+on: push
+
+env:
+  MIZU_CI_IMAGE: mizu/ci:0.0
+
+concurrency:
+  group: acceptance-tests-on-pr-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-tests:
+    name: Run tests
+    runs-on: ubuntu-latest
+    if: ${{ contains(github.event.head_commit.message, '#run_acceptance_tests') }}
+
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          load: true
+          tags: ${{ env.MIZU_CI_IMAGE }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Setup acceptance test
+        run: ./acceptanceTests/setup.sh
+
+      - name: Create k8s users and change context
+        env:
+          USERNAME_UNRESTRICTED: user-with-clusterwide-access
+          USERNAME_RESTRICTED: user-with-restricted-access
+        run: |
+          ./acceptanceTests/create_user.sh "${USERNAME_UNRESTRICTED}"
+          ./acceptanceTests/create_user.sh "${USERNAME_RESTRICTED}"
+          kubectl apply -f cli/cmd/permissionFiles/permissions-all-namespaces-tap.yaml
+          kubectl config use-context ${USERNAME_UNRESTRICTED}
+
+      - name: Test
+        run: make acceptance-test

.github/workflows/inactive-issues-close.yaml

@@ -13,10 +13,11 @@ jobs:
       - uses: actions/stale@v3
         with:
           days-before-issue-stale: 30
-          days-before-issue-close: 14
+          days-before-issue-close: -1
+          exempt-issue-labels: "enhancement"
           stale-issue-label: "stale"
           stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
-          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+          close-issue-message: ""
           days-before-pr-stale: -1
           days-before-pr-close: -1
           repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/static_code_analysis.yml

@@ -10,7 +10,7 @@ permissions:
   contents: read
 
 jobs:
-  golangci:
+  go-lint:
     name: Go lint
     runs-on: ubuntu-latest
     steps:
@@ -141,3 +141,61 @@ jobs:
         with:
           version: latest
           working-directory: tap/extensions/redis
+
+      - name: Check logger modified files
+        id: logger_modified_files
+        run: devops/check_modified_files.sh logger/
+
+      - name: Go lint - logger
+        uses: golangci/golangci-lint-action@v2
+        if: steps.logger_modified_files.outputs.matched == 'true'
+        with:
+          version: latest
+          working-directory: logger
+
+  es-lint:
+    name: ES lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16
+        
+      - name: Check modified UI files
+        id: ui_modified_files
+        run: devops/check_modified_files.sh ui/
+      
+      - name: ESLint prerequisites ui
+        if: steps.ui_modified_files.outputs.matched == 'true'
+        run: |
+          sudo npm install -g eslint
+          cd ui
+          npm run prestart
+          npm i
+
+      - name: ESLint ui
+        if: steps.ui_modified_files.outputs.matched == 'true'
+        run: |
+          cd ui
+          npm run eslint
+      
+      - name: Check modified ui-common files
+        id: ui_common_modified_files
+        run: devops/check_modified_files.sh ui-common/
+      
+      - name: ESLint prerequisites ui-common
+        if: steps.ui_common_modified_files.outputs.matched == 'true'
+        run: |
+          sudo npm install -g eslint
+          cd ui-common
+          npm i
+
+      - name: ESLint ui-common
+        if: steps.ui_common_modified_files.outputs.matched == 'true'
+        run: |
+          cd ui-common
+          npm run eslint

.github/workflows/test.yml

@@ -56,7 +56,7 @@ jobs:
       
       - name: Check extensions modified files
         id: ext_modified_files
-        run: devops/check_modified_files.sh tap/extensions/
+        run: devops/check_modified_files.sh tap/extensions/ tap/api/
 
       - name: Extensions Test
         if: github.event_name == 'push' || steps.ext_modified_files.outputs.matched == 'true'
@@ -64,4 +64,3 @@ jobs:
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
-

.gitignore

@@ -52,4 +52,7 @@ tap/extensions/*/expect
 # UI folders to ignore
 **/node_modules/**
 **/dist/**
-*.editorconfig
+*.editorconfig
+
+# Ignore *.log files
+*.log

Dockerfile

@@ -44,11 +44,18 @@ ENV CGO_ENABLED=1 GOOS=linux
 ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap"
 
 
+### Builder image for AArch64 to x86-64 cross-compilation
+FROM up9inc/linux-x86_64-musl-go-libpcap AS builder-from-arm64v8-to-amd64
+ENV CGO_ENABLED=1 GOOS=linux
+ENV GOARCH=amd64 CGO_CFLAGS="-I/libpcap"
+
+
 ### Final builder image where the build happens
 # Possible build strategies:
 # BUILDARCH=amd64 TARGETARCH=amd64
 # BUILDARCH=arm64v8 TARGETARCH=arm64v8
 # BUILDARCH=amd64 TARGETARCH=arm64v8
+# BUILDARCH=arm64v8 TARGETARCH=amd64
 ARG BUILDARCH=amd64
 ARG TARGETARCH=amd64
 FROM builder-from-${BUILDARCH}-to-${TARGETARCH} AS builder
@@ -58,18 +65,19 @@ WORKDIR /app/agent-build
 
 COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
+COPY logger/go.mod logger/go.mod ../logger/
 COPY tap/go.mod tap/go.mod ../tap/
 COPY tap/api/go.mod ../tap/api/
+COPY tap/dbgctl/go.mod ../tap/dbgctl/
 COPY tap/extensions/amqp/go.mod ../tap/extensions/amqp/
 COPY tap/extensions/http/go.mod ../tap/extensions/http/
 COPY tap/extensions/kafka/go.mod ../tap/extensions/kafka/
 COPY tap/extensions/redis/go.mod ../tap/extensions/redis/
 RUN go mod download
-# cheap trick to make the build faster (as long as go.mod did not change)
-RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
 
 # Copy and build agent code
 COPY shared ../shared
+COPY logger ../logger
 COPY tap ../tap
 COPY agent .
 
@@ -87,8 +95,8 @@ RUN go build -ldflags="-extldflags=-static -s -w \
     -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum
-ADD https://github.com/up9inc/basenine/releases/download/v0.6.6/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
-ADD https://github.com/up9inc/basenine/releases/download/v0.6.6/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.8.2/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
+ADD https://github.com/up9inc/basenine/releases/download/v0.8.2/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
 
 RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
     chmod +x ./basenine_linux_"${GOARCH}" && \

Makefile

@@ -83,6 +83,7 @@ test-lint:  ## Run lint on all modules
 	cd cli && golangci-lint run
 	cd acceptanceTests && golangci-lint run
 	cd tap/api && golangci-lint run
+	cd tap/dbgctl && golangci-lint run
 	cd tap/extensions/ && for D in */; do cd $$D && golangci-lint run && cd ..; done
 
 test-cli:  ## Run cli tests

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -39,13 +39,13 @@ export function verifyMinimumEntries() {
     });
 }
 
-export function leftTextCheck(entryNum, path, expectedText) {
-    cy.get(`#list #entry-${entryNum} ${path}`).invoke('text').should('eq', expectedText);
+export function leftTextCheck(entryId, path, expectedText) {
+    cy.get(`#list #entry-${entryId} ${path}`).invoke('text').should('eq', expectedText);
 }
 
-export function leftOnHoverCheck(entryNum, path, filterName) {
-    cy.get(`#list #entry-${entryNum} ${path}`).trigger('mouseover');
-    cy.get(`#list #entry-${entryNum} [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(filterName));
+export function leftOnHoverCheck(entryId, path, filterName) {
+    cy.get(`#list #entry-${entryId} ${path}`).trigger('mouseover');
+    cy.get(`#list #entry-${entryId} [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(filterName));
 }
 
 export function rightTextCheck(path, expectedText) {
@@ -57,15 +57,8 @@ export function rightOnHoverCheck(path, expectedText) {
     cy.get(`#rightSideContainer [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(expectedText));
 }
 
-export function checkThatAllEntriesShown() {
-    cy.get('#entries-length').then(number => {
-        if (number.text() === '1')
-            cy.get('[title="Fetch old records"]').click();
-    });
-}
-
 export function checkFilterByMethod(funcDict) {
-    const {protocol, method, methodQuery, summary, summaryQuery} = funcDict;
+    const {protocol, method, methodQuery, summary, summaryQuery, numberOfRecords} = funcDict;
     const summaryDict = getSummaryDict(summary, summaryQuery);
     const methodDict = getMethodDict(method, methodQuery);
     const protocolDict = getProtocolDict(protocol.name, protocol.text);
@@ -76,57 +69,62 @@ export function checkFilterByMethod(funcDict) {
         cy.get('[type="submit"]').click();
         cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
 
-        cy.get('#entries-length').then(number => {
-            // if the entries list isn't expanded it expands here
-            if (number.text() === '0' || number.text() === '1') // todo change when TRA-4262 is fixed
-                cy.get('[title="Fetch old records"]').click();
+        waitForFetch(numberOfRecords);
+        pauseStream();
 
-            cy.get('#entries-length').should('not.have.text', '0').and('not.have.text', '1').then(() => {
-                cy.get(`#list [id]`).then(elements => {
-                    const listElmWithIdAttr = Object.values(elements);
-                    let doneCheckOnFirst = false;
+        cy.get(`#list [id^=entry]`).then(elements => {
+            const listElmWithIdAttr = Object.values(elements);
+            let doneCheckOnFirst = false;
 
-                    cy.get('#entries-length').invoke('text').then(len => {
-                        resizeIfNeeded(len);
-                        listElmWithIdAttr.forEach(entry => {
-                            if (entry?.id && entry.id.match(RegExp(/entry-(\d{2}|\d{1})$/gm))) {
-                                const entryNum = getEntryNumById(entry.id);
+            cy.get('#entries-length').invoke('text').then(len => {
+                listElmWithIdAttr.forEach(entry => {
+                    if (entry?.id && entry.id.match(RegExp(/entry-(\d{24})$/gm))) {
+                        const entryId = getEntryId(entry.id);
 
-                                leftTextCheck(entryNum, methodDict.pathLeft, methodDict.expectedText);
-                                leftTextCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedTextLeft);
-                                if (summaryDict)
-                                    leftTextCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedText);
+                        leftTextCheck(entryId, methodDict.pathLeft, methodDict.expectedText);
+                        leftTextCheck(entryId, protocolDict.pathLeft, protocolDict.expectedTextLeft);
+                        if (summaryDict)
+                            leftTextCheck(entryId, summaryDict.pathLeft, summaryDict.expectedText);
 
-                                if (!doneCheckOnFirst) {
-                                    deepCheck(funcDict, protocolDict, methodDict, entry);
-                                    doneCheckOnFirst = true;
-                                }
-                            }
-                        });
-                        resizeIfNeeded(len);
-                    });
+                        if (!doneCheckOnFirst) {
+                            deepCheck(funcDict, protocolDict, methodDict, entry);
+                            doneCheckOnFirst = true;
+                        }
+                    }
                 });
             });
         });
     });
 }
 
-function resizeIfNeeded(entriesLen) {
-    if (entriesLen > maxEntriesInDom){
-        Cypress.config().viewportHeight === Cypress.env('normalMizuHeight') ?
-            resizeToHugeMizu() : resizeToNormalMizu()
-    }
+export const refreshWaitTimeout = 10000;
+
+export function waitForFetch(gt) {
+    cy.get('#entries-length', {timeout: refreshWaitTimeout}).should((el) => {
+        expect(parseInt(el.text().trim(), 10)).to.be.greaterThan(gt);
+    });
+}
+
+export function pauseStream() {
+    cy.get('#pause-icon').click();
+    cy.get('#pause-icon').should('not.be.visible');
+}
+
+
+export function getEntryId(id) {
+    // take the second part from the string (entry-<ID>)
+    return id.split('-')[1];
 }
 
 function deepCheck(generalDict, protocolDict, methodDict, entry) {
-    const entryNum = getEntryNumById(entry.id);
+    const entryId = getEntryId(entry.id);
     const {summary, value} = generalDict;
     const summaryDict = getSummaryDict(summary);
 
-    leftOnHoverCheck(entryNum, methodDict.pathLeft, methodDict.expectedOnHover);
-    leftOnHoverCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedOnHover);
+    leftOnHoverCheck(entryId, methodDict.pathLeft, methodDict.expectedOnHover);
+    leftOnHoverCheck(entryId, protocolDict.pathLeft, protocolDict.expectedOnHover);
     if (summaryDict)
-        leftOnHoverCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedOnHover);
+        leftOnHoverCheck(entryId, summaryDict.pathLeft, summaryDict.expectedOnHover);
 
     cy.get(`#${entry.id}`).click();
 
@@ -183,7 +181,3 @@ function getProtocolDict(protocol, protocolText) {
         expectedOnHover: protocol.toLowerCase()
     };
 }
-
-function getEntryNumById (id) {
-    return parseInt(id.split('-')[1]);
-}

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -1,5 +1,4 @@
 import {
-    checkThatAllEntriesShown,
     isValueExistsInElement,
     resizeToHugeMizu,
 } from "../testHelpers/TrafficHelper";
@@ -12,18 +11,15 @@ checkEntries();
 
 function checkEntries() {
     it('checking all entries', function () {
-        checkThatAllEntriesShown();
-        resizeToHugeMizu();
+        cy.get('#entries-length').should('not.have.text', '0').then(() => {
+            resizeToHugeMizu();
 
-        cy.get('#total-entries').then(number => {
-            const numOfEntries = parseInt(number.text());
-            [...Array(numOfEntries).keys()].map(checkEntry);
+            cy.get('#list [id^=entry]').each(entryElement => {
+                entryElement.click();
+                cy.get('#tbody-Headers').should('be.visible');
+                isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
+            });
         });
     });
 }
 
-function checkEntry(entryIndex) {
-    cy.get(`#entry-${entryIndex}`).click();
-    cy.get('#tbody-Headers').should('be.visible');
-    isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
-}

acceptanceTests/cypress/integration/tests/Rabbit.js

@@ -5,6 +5,7 @@ it('opening mizu', function () {
 });
 
 const rabbitProtocolDetails = {name: 'AMQP', text: 'Advanced Message Queuing Protocol 0-9-1'};
+const numberOfRecords = 5;
 
 checkFilterByMethod({
     protocol: rabbitProtocolDetails,
@@ -12,6 +13,7 @@ checkFilterByMethod({
     methodQuery: 'request.method == "exchange declare"',
     summary: 'exchange',
     summaryQuery: 'request.exchange == "exchange"',
+    numberOfRecords: numberOfRecords,
     value: null
 });
 
@@ -21,6 +23,7 @@ checkFilterByMethod({
     methodQuery: 'request.method == "queue declare"',
     summary: 'queue',
     summaryQuery: 'request.queue == "queue"',
+    numberOfRecords: numberOfRecords,
     value: null
 });
 
@@ -30,6 +33,7 @@ checkFilterByMethod({
     methodQuery: 'request.method == "queue bind"',
     summary: 'queue',
     summaryQuery: 'request.queue == "queue"',
+    numberOfRecords: numberOfRecords,
     value: null
 });
 
@@ -39,6 +43,7 @@ checkFilterByMethod({
     methodQuery: 'request.method == "basic publish"',
     summary: 'exchange',
     summaryQuery: 'request.exchange == "exchange"',
+    numberOfRecords: numberOfRecords,
     value: {tab: valueTabs.request, regex: /^message$/mg}
 });
 
@@ -48,6 +53,7 @@ checkFilterByMethod({
     methodQuery: 'request.method == "basic consume"',
     summary: 'queue',
     summaryQuery: 'request.queue == "queue"',
+    numberOfRecords: numberOfRecords,
     value: null
 });
 
@@ -57,5 +63,6 @@ checkFilterByMethod({
     methodQuery: 'request.method == "basic deliver"',
     summary: 'exchange',
     summaryQuery: 'request.queue == "exchange"',
+    numberOfRecords: numberOfRecords,
     value: {tab: valueTabs.request, regex: /^message$/mg}
 });

acceptanceTests/cypress/integration/tests/Redis.js

@@ -5,6 +5,7 @@ it('opening mizu', function () {
 });
 
 const redisProtocolDetails = {name: 'redis', text: 'Redis Serialization Protocol'};
+const numberOfRecords = 5;
 
 checkFilterByMethod({
     protocol: redisProtocolDetails,
@@ -12,6 +13,7 @@ checkFilterByMethod({
     methodQuery: 'request.command == "PING"',
     summary: null,
     summaryQuery: '',
+    numberOfRecords: numberOfRecords,
     value: null
 })
 
@@ -21,6 +23,7 @@ checkFilterByMethod({
     methodQuery: 'request.command == "SET"',
     summary: 'key',
     summaryQuery: 'request.key == "key"',
+    numberOfRecords: numberOfRecords,
     value: {tab: valueTabs.request, regex: /^\[value, keepttl]$/mg}
 })
 
@@ -30,6 +33,7 @@ checkFilterByMethod({
     methodQuery: 'request.command == "EXISTS"',
     summary: 'key',
     summaryQuery: 'request.key == "key"',
+    numberOfRecords: numberOfRecords,
     value: {tab: valueTabs.response, regex: /^1$/mg}
 })
 
@@ -39,6 +43,7 @@ checkFilterByMethod({
     methodQuery: 'request.command == "GET"',
     summary: 'key',
     summaryQuery: 'request.key == "key"',
+    numberOfRecords: numberOfRecords,
     value: {tab: valueTabs.response, regex: /^value$/mg}
 })
 
@@ -48,5 +53,6 @@ checkFilterByMethod({
     methodQuery: 'request.command == "DEL"',
     summary: 'key',
     summaryQuery: 'request.key == "key"',
+    numberOfRecords: numberOfRecords,
     value: {tab: valueTabs.response, regex: /^1$|^0$/mg}
 })

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -1,17 +1,18 @@
 import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
 import {
+    getEntryId,
     leftOnHoverCheck,
     leftTextCheck,
     resizeToHugeMizu,
     resizeToNormalMizu,
     rightOnHoverCheck,
     rightTextCheck,
-    verifyMinimumEntries
+    verifyMinimumEntries,
+    refreshWaitTimeout,
+    waitForFetch,
+    pauseStream
 } from "../testHelpers/TrafficHelper";
 
-const refreshWaitTimeout = 10000;
-
-
 const fullParam = Cypress.env('arrayDict'); // "Name:fooNamespace:barName:foo1Namespace:bar1"
 const podsArray = fullParam.split('Name:').slice(1); // ["fooNamespace:bar", "foo1Namespace:bar1"]
 podsArray.forEach((podStr, index) => {
@@ -64,70 +65,77 @@ it('right side sanity test', function () {
 checkIllegalFilter('invalid filter');
 
 checkFilter({
-    name: 'http',
+    filter: 'http',
     leftSidePath: '> :nth-child(1) > :nth-child(1)',
     leftSideExpectedText: 'HTTP',
     rightSidePath: '[title=HTTP]',
     rightSideExpectedText: 'Hypertext Transfer Protocol -- HTTP/1.1',
-    applyByEnter: true
+    applyByCtrlEnter: true,
+    numberOfRecords: 20,
 });
 
 checkFilter({
-    name: 'response.status == 200',
+    filter: 'response.status == 200',
     leftSidePath: '[title="Status Code"]',
     leftSideExpectedText: '200',
     rightSidePath: '> :nth-child(2) [title="Status Code"]',
     rightSideExpectedText: '200',
-    applyByEnter: false
+    applyByCtrlEnter: false,
+    numberOfRecords: 20
 });
 
 if (Cypress.env('shouldCheckSrcAndDest')) {
     serviceMapCheck();
 
     checkFilter({
-        name: 'src.name == ""',
+        filter: 'src.name == ""',
         leftSidePath: '[title="Source Name"]',
         leftSideExpectedText: '[Unresolved]',
         rightSidePath: '> :nth-child(2) [title="Source Name"]',
         rightSideExpectedText: '[Unresolved]',
-        applyByEnter: false
+        applyByCtrlEnter: false,
+        numberOfRecords: 20
     });
 
     checkFilter({
-        name: `dst.name == "httpbin.mizu-tests"`,
+        filter: `dst.name == "httpbin.mizu-tests"`,
         leftSidePath: '> :nth-child(3) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
         leftSideExpectedText: 'httpbin.mizu-tests',
         rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
         rightSideExpectedText: 'httpbin.mizu-tests',
-        applyByEnter: false
+        applyByCtrlEnter: false,
+        numberOfRecords: 20
     });
 }
 
 checkFilter({
-    name: 'request.method == "GET"',
+    filter: 'request.method == "GET"',
     leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
     leftSideExpectedText: 'GET',
     rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
     rightSideExpectedText: 'GET',
-    applyByEnter: true
+    applyByCtrlEnter: true,
+    numberOfRecords: 20
 });
 
 checkFilter({
-    name: 'request.path == "/get"',
+    filter: 'request.path == "/get"',
     leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
     leftSideExpectedText: '/get',
     rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
     rightSideExpectedText: '/get',
-    applyByEnter: false
+    applyByCtrlEnter: false,
+    numberOfRecords: 20
 });
 
 checkFilter({
-    name: 'src.ip == "127.0.0.1"',
+    filter: 'src.ip == "127.0.0.1"',
     leftSidePath: '[title="Source IP"]',
     leftSideExpectedText: '127.0.0.1',
     rightSidePath: '> :nth-child(2) [title="Source IP"]',
     rightSideExpectedText: '127.0.0.1',
-    applyByEnter: false
+    applyByCtrlEnter: false,
+    numberOfRecords: 20
 });
 
 checkFilterNoResults('request.method == "POST"');
@@ -148,9 +156,6 @@ function checkFilterNoResults(filterName) {
             // the DOM should show 0 entries
             cy.get('#entries-length').should('have.text', '0');
 
-            // going through every potential entry and verifies that it doesn't exist
-            [...Array(parseInt(totalEntries)).keys()].map(shouldNotExist);
-
             cy.get('[title="Fetch old records"]').click();
             cy.get('#noMoreDataTop', {timeout: refreshWaitTimeout}).should('be.visible');
             cy.get('#entries-length').should('have.text', '0'); // after loading all entries there should still be 0 entries
@@ -162,10 +167,6 @@ function checkFilterNoResults(filterName) {
     });
 }
 
-function shouldNotExist(entryNum) {
-    cy.get(`entry-${entryNum}`).should('not.exist');
-}
-
 function checkIllegalFilter(illegalFilterName) {
     it(`should show red search bar with the input: ${illegalFilterName}`, function () {
         cy.reload();
@@ -185,60 +186,74 @@ function checkIllegalFilter(illegalFilterName) {
         });
     });
 }
-function checkFilter(filterDetails){
-    const {name, leftSidePath, rightSidePath, rightSideExpectedText, leftSideExpectedText, applyByEnter} = filterDetails;
+
+function checkFilter(filterDetails) {
+    const {
+        filter,
+        leftSidePath,
+        rightSidePath,
+        rightSideExpectedText,
+        leftSideExpectedText,
+        applyByCtrlEnter,
+        numberOfRecords
+    } = filterDetails;
+
     const entriesForDeeperCheck = 5;
 
-    it(`checking the filter: ${name}`, function () {
-        cy.get('#total-entries').should('not.have.text', '0').then(number => {
-            const totalEntries = number.text();
+    it(`checking the filter: ${filter}`, function () {
+        cy.get('.w-tc-editor-text').clear();
+        // applying the filter with alt+enter or with the button
+        cy.get('.w-tc-editor-text').type(`${filter}${applyByCtrlEnter ? '{ctrl+enter}' : ''}`);
+        cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
+        if (!applyByCtrlEnter)
+            cy.get('[type="submit"]').click();
 
-            // checks the hover on the last entry (the only one in DOM at the beginning)
-            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
+        waitForFetch(numberOfRecords);
+        pauseStream();
 
-            cy.get('.w-tc-editor-text').clear();
-            // applying the filter with alt+enter or with the button
-            cy.get('.w-tc-editor-text').type(`${name}${applyByEnter ? '{alt+enter}' : ''}`);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
-            if (!applyByEnter)
-                cy.get('[type="submit"]').click();
+        cy.get(`#list [id^=entry]`).last().then(elem => {
+            const element = elem[0];
+            const entryId = getEntryId(element.id);
 
             // only one entry in DOM after filtering, checking all checks on it
-            leftTextCheck(totalEntries - 1, leftSidePath, leftSideExpectedText);
-            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
+            leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
+            leftOnHoverCheck(entryId, leftSidePath, filter);
+
             rightTextCheck(rightSidePath, rightSideExpectedText);
-            rightOnHoverCheck(rightSidePath, name);
+            rightOnHoverCheck(rightSidePath, filter);
             checkRightSideResponseBody();
-
-            cy.get('[title="Fetch old records"]').click();
-            resizeToHugeMizu();
-
-            // waiting for the entries number to load
-            cy.get('#entries-length', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-
-            // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
-            [...Array(parseInt(totalEntries)).keys()].forEach(entryNum => {
-                leftTextCheck(entryNum, leftSidePath, leftSideExpectedText);
-            });
-
-            // making the other 3 checks on the first X entries (longer time for each check)
-            deeperChcek(leftSidePath, rightSidePath, name, leftSideExpectedText, rightSideExpectedText, entriesForDeeperCheck);
-
-            // reloading then waiting for the entries number to load
-            resizeToNormalMizu();
-            cy.reload();
-            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
         });
+
+        resizeToHugeMizu();
+
+        // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
+        cy.get(`#list [id^=entry]`).each(elem => {
+            const element = elem[0];
+            let entryId = getEntryId(element.id);
+            leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
+        });
+
+        // making the other 3 checks on the first X entries (longer time for each check)
+        deeperCheck(leftSidePath, rightSidePath, filter, rightSideExpectedText, entriesForDeeperCheck);
+
+        // reloading then waiting for the entries number to load
+        resizeToNormalMizu();
+        cy.reload();
+        waitForFetch(numberOfRecords);
+        pauseStream();
     });
 }
 
-function deeperChcek(leftSidePath, rightSidePath, filterName, leftSideExpectedText, rightSideExpectedText, entriesNumToCheck) {
-    [...Array(entriesNumToCheck).keys()].forEach(entryNum => {
-        leftOnHoverCheck(entryNum, leftSidePath, filterName);
+function deeperCheck(leftSidePath, rightSidePath, filterName, rightSideExpectedText, entriesNumToCheck) {
+    cy.get(`#list [id^=entry]`).each((element, index) => {
+        if (index < entriesNumToCheck) {
+            const entryId = getEntryId(element[0].id);
+            leftOnHoverCheck(entryId, leftSidePath, filterName);
 
-        cy.get(`#list #entry-${entryNum}`).click();
-        rightTextCheck(rightSidePath, rightSideExpectedText);
-        rightOnHoverCheck(rightSidePath, filterName);
+            cy.get(`#list #entry-${entryId}`).click();
+            rightTextCheck(rightSidePath, rightSideExpectedText);
+            rightOnHoverCheck(rightSidePath, filterName);
+        }
     });
 }
 

acceptanceTests/go.mod

@@ -29,6 +29,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/up9inc/mizu/logger v0.0.0 // indirect
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
@@ -48,6 +49,10 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
+replace github.com/up9inc/mizu/logger v0.0.0 => ../logger
+
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
+
+replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../tap/dbgctl

acceptanceTests/setup.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
+set -e
 
 PREFIX=$HOME/local/bin
 VERSION=v1.22.0
+TUNNEL_LOG="tunnel.log"
+PROXY_LOG="proxy.log"
 
 echo "Attempting to install minikube and assorted tools to $PREFIX"
 
@@ -11,7 +14,7 @@ if ! [ -x "$(command -v kubectl)" ]; then
   chmod +x kubectl
   mv kubectl "$PREFIX"
 else
-  echo "kubetcl is already installed"
+  echo "kubectl is already installed"
 fi
 
 if ! [ -x "$(command -v minikube)" ]; then
@@ -27,44 +30,57 @@ echo "Starting minikube..."
 minikube start
 
 echo "Creating mizu tests namespaces"
-kubectl create namespace mizu-tests
-kubectl create namespace mizu-tests2
+kubectl create namespace mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl create namespace mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating httpbin deployments"
-kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests
-kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
-kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating redis deployment"
-kubectl create deployment redis --image=redis -n mizu-tests
+kubectl create deployment redis --image=redis -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating rabbitmq deployment"
-kubectl create deployment rabbitmq --image=rabbitmq -n mizu-tests
+kubectl create deployment rabbitmq --image=rabbitmq -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating httpbin services"
-kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
-kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
-kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating redis service"
-kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests
+kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating rabbitmq service"
-kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests
+kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
+# TODO: need to understand how to fail if address already in use
 echo "Starting proxy"
-kubectl proxy --port=8080 &
+rm -f ${PROXY_LOG}
+kubectl proxy --port=8080 > ${PROXY_LOG} &
+PID1=$!
+echo "kubectl proxy process id is ${PID1} and log of proxy in ${PROXY_LOG}"
 
-echo "Setting minikube docker env"
-eval $(minikube docker-env)
+if [[ -z "${CI}" ]]; then
+  echo "Setting env var of mizu ci image"
+  export MIZU_CI_IMAGE="mizu/ci:0.0"
+  echo "Build agent image"
+  docker build -t "${MIZU_CI_IMAGE}" .
+else
+  echo "not building docker image in CI because it is created as separate step"
+fi
 
-echo "Build agent image"
-docker build -t mizu/ci:0.0 .
+minikube image load "${MIZU_CI_IMAGE}"
 
 echo "Build cli"
 cd cli && make build GIT_BRANCH=ci SUFFIX=ci
 
+# TODO: need to understand how to fail if password is asked (sudo)
 echo "Starting tunnel"
-minikube tunnel &
+rm -f ${TUNNEL_LOG}
+minikube tunnel > ${TUNNEL_LOG} &
+PID2=$!
+echo "Minikube tunnel process id is ${PID2} and log of tunnel in ${TUNNEL_LOG}"

acceptanceTests/testsUtils.go

@@ -212,9 +212,10 @@ func DeleteKubeFile(kubeContext string, namespace string, filename string) error
 }
 
 func getDefaultCommandArgs() []string {
+	agentImageValue := os.Getenv("MIZU_CI_IMAGE")
 	setFlag := "--set"
 	telemetry := "telemetry=false"
-	agentImage := "agent-image=mizu/ci:0.0"
+	agentImage := fmt.Sprintf("agent-image=%s", agentImageValue)
 	imagePullPolicy := "image-pull-policy=IfNotPresent"
 	headless := "headless=true"
 

agent/go.mod

@@ -6,8 +6,8 @@ require (
 	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
 	github.com/chanced/openapi v0.0.8
 	github.com/djherbis/atime v1.1.0
-	github.com/elastic/go-elasticsearch/v7 v7.17.0
 	github.com/getkin/kin-openapi v0.89.0
+	github.com/gin-contrib/pprof v1.3.0
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.7.7
 	github.com/go-playground/locales v0.14.0
@@ -19,12 +19,13 @@ require (
 	github.com/nav-inc/datetime v0.1.3
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v1.0.0
-	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/stretchr/testify v1.7.0
-	github.com/up9inc/basenine/client/go v0.0.0-20220326121918-785f3061c8ce
+	github.com/up9inc/basenine/client/go v0.0.0-20220509204026-c37adfc587f4
+	github.com/up9inc/mizu/logger v0.0.0
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
+	github.com/up9inc/mizu/tap/dbgctl v0.0.0
 	github.com/up9inc/mizu/tap/extensions/amqp v0.0.0
 	github.com/up9inc/mizu/tap/extensions/http v0.0.0
 	github.com/up9inc/mizu/tap/extensions/kafka v0.0.0
@@ -49,7 +50,6 @@ require (
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beevik/etree v1.1.0 // indirect
-	github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 // indirect
 	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
 	github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect
 	github.com/cilium/ebpf v0.8.0 // indirect
@@ -62,6 +62,7 @@ require (
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.6 // indirect
 	github.com/go-openapi/swag v0.21.1 // indirect
@@ -77,6 +78,7 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -86,6 +88,7 @@ require (
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mertyildiran/gqlparser/v2 v2.4.6 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
@@ -101,17 +104,22 @@ require (
 	github.com/russross/blackfriday v1.6.0 // indirect
 	github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
 	github.com/segmentio/kafka-go v0.4.27 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/spf13/cobra v1.3.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/struCoder/pidusage v0.2.1 // indirect
 	github.com/tidwall/gjson v1.14.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/sjson v1.2.4 // indirect
+	github.com/tklauser/go-sysconf v0.3.10 // indirect
+	github.com/tklauser/numcpus v0.4.0 // indirect
 	github.com/ugorji/go/codec v1.2.6 // indirect
 	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.starlark.net v0.0.0-20220203230714-bb14e151c28f // indirect
-	golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab // indirect
+	golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b // indirect
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
@@ -136,6 +144,8 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
+replace github.com/up9inc/mizu/logger v0.0.0 => ../logger
+
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 
 replace github.com/up9inc/mizu/tap v0.0.0 => ../tap
@@ -149,3 +159,5 @@ replace github.com/up9inc/mizu/tap/extensions/http v0.0.0 => ../tap/extensions/h
 replace github.com/up9inc/mizu/tap/extensions/kafka v0.0.0 => ../tap/extensions/kafka
 
 replace github.com/up9inc/mizu/tap/extensions/redis v0.0.0 => ../tap/extensions/redis
+
+replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../tap/dbgctl

agent/go.sum

@@ -83,6 +83,7 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -108,8 +109,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M=
-github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -167,8 +166,6 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
-github.com/elastic/go-elasticsearch/v7 v7.17.0 h1:0fcSh4qeC/i1+7QU1KXpmq2iUAdMk4l0/vmbtW1+KJM=
-github.com/elastic/go-elasticsearch/v7 v7.17.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -216,10 +213,13 @@ github.com/getkin/kin-openapi v0.89.0 h1:p4nagHchUKGn85z/f+pse4aSh50nIBOYjOhMIku
 github.com/getkin/kin-openapi v0.89.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gin-contrib/pprof v1.3.0 h1:G9eK6HnbkSqDZBYbzG4wrjCsA4e+cvYAHUZw6W+W9K0=
+github.com/gin-contrib/pprof v1.3.0/go.mod h1:waMjT1H9b179t3CxuG1cV3DHpga6ybizwfBaM5OXaB0=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-contrib/static v0.0.1 h1:JVxuvHPuUfkoul12N7dtQw7KRn/pSMq7Ue1Va9Swm1U=
 github.com/gin-contrib/static v0.0.1/go.mod h1:CSxeF+wep05e0kCOsqWdAWbSszmc31zTIbD8TvWl7Hs=
+github.com/gin-gonic/gin v1.6.2/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs=
 github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=
@@ -241,6 +241,8 @@ github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
 github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -406,6 +408,7 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
@@ -495,6 +498,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mertyildiran/gqlparser/v2 v2.4.6 h1:enAq4F5PYgW/rYExPNzYt7IYrrZnzrfqdywMA1QdFtM=
+github.com/mertyildiran/gqlparser/v2 v2.4.6/go.mod h1:XZId58F+XqRSmoLrdsOLgqA918oNvBzuOORruJWBjDo=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
@@ -556,8 +561,6 @@ github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HD
 github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
-github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
@@ -620,6 +623,8 @@ github.com/segmentio/kafka-go v0.4.27 h1:sIhEozeL/TLN2mZ5dkG462vcGEWYKS+u31sXPjK
 github.com/segmentio/kafka-go v0.4.27/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
+github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -661,6 +666,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/struCoder/pidusage v0.2.1 h1:dFiEgUDkubeIj0XA1NpQ6+8LQmKrLi7NiIQl86E6BoY=
+github.com/struCoder/pidusage v0.2.1/go.mod h1:bewtP2KUA1TBUyza5+/PCpSQ6sc/H6jJbIKAzqW86BA=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.12.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
@@ -674,6 +681,10 @@ github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso
 github.com/tidwall/sjson v1.2.3/go.mod h1:5WdjKx3AQMvCJ4RG6/2UYT7dLrGvJUV1x4jdTAyGvZs=
 github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc=
 github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
+github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
+github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
+github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
+github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
@@ -683,8 +694,8 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
 github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
-github.com/up9inc/basenine/client/go v0.0.0-20220326121918-785f3061c8ce h1:vMTCpKItc9OyTLJXocNaq2NcBU5EnurJgTVOYb8W8dw=
-github.com/up9inc/basenine/client/go v0.0.0-20220326121918-785f3061c8ce/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220509204026-c37adfc587f4 h1:nNOrU1HVH0fnaG7GNhxCc8kNPVL035Iix7ihUF6lZT8=
+github.com/up9inc/basenine/client/go v0.0.0-20220509204026-c37adfc587f4/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=
@@ -704,6 +715,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
+github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
@@ -755,8 +768,9 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab h1:lnZ4LoV0UMdibeCUfIB2a4uFwRu491WX/VB2reB8xNc=
 golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b h1:Qwe1rC8PSniVfAFPFJeyUkB+zcysC3RgJBAGk7eqBEU=
+golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -895,6 +909,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -963,6 +978,7 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220207234003-57398862261d h1:Bm7BNOQt2Qv7ZqysjeLjgCBanX+88Z/OtdvsrEv1Djc=
 golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

agent/main.go

@@ -14,10 +14,10 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/gin-contrib/pprof"
 	"github.com/gin-contrib/static"
 	"github.com/gin-gonic/gin"
 	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/agent/pkg/elastic"
 	"github.com/up9inc/mizu/agent/pkg/entries"
 	"github.com/up9inc/mizu/agent/pkg/middlewares"
 	"github.com/up9inc/mizu/agent/pkg/models"
@@ -33,10 +33,11 @@ import (
 
 	"github.com/gorilla/websocket"
 	"github.com/op/go-logging"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap"
 	tapApi "github.com/up9inc/mizu/tap/api"
+	"github.com/up9inc/mizu/tap/dbgctl"
 )
 
 var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API")
@@ -46,6 +47,7 @@ var apiServerAddress = flag.String("api-server-address", "", "Address of mizu AP
 var namespace = flag.String("namespace", "", "Resolve IPs if they belong to resources in this namespace (default is all)")
 var harsReaderMode = flag.Bool("hars-read", false, "Run in hars-read mode")
 var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
+var profiler = flag.Bool("profiler", false, "Run pprof server")
 
 const (
 	socketConnectionRetries    = 30
@@ -62,7 +64,7 @@ func main() {
 	app.LoadExtensions()
 
 	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode {
-		panic("One of the flags --tap, --api or --standalone or --hars-read must be provided")
+		panic("One of the flags --tap, --api-server, --standalone or --hars-read must be provided")
 	}
 
 	if *standaloneMode {
@@ -70,7 +72,14 @@ func main() {
 	} else if *tapperMode {
 		runInTapperMode()
 	} else if *apiServerMode {
-		utils.StartServer(runInApiServerMode(*namespace))
+		app := runInApiServerMode(*namespace)
+
+		if *profiler {
+			pprof.Register(app)
+		}
+
+		utils.StartServer(app)
+
 	} else if *harsReaderMode {
 		runInHarReaderMode()
 	}
@@ -153,7 +162,9 @@ func runInTapperMode() {
 	}
 
 	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-	tapOpts := &tap.TapOpts{HostMode: hostMode}
+	tapOpts := &tap.TapOpts{
+		HostMode:     hostMode,
+	}
 
 	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
 
@@ -199,13 +210,12 @@ func runInHarReaderMode() {
 func enableExpFeatureIfNeeded() {
 	if config.Config.OAS {
 		oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
-		oasGenerator.Start(nil)
+		oasGenerator.Start()
 	}
 	if config.Config.ServiceMap {
 		serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap)
 		serviceMapGenerator.Enable()
 	}
-	elastic.GetInstance().Configure(config.Config.Elastic)
 }
 
 func getSyncEntriesConfig() *shared.SyncEntriesConfig {
@@ -283,6 +293,10 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
 			continue
 		}
 
+		if dbgctl.MizuTapperDisableSending {
+			continue
+		}
+
 		// NOTE: This is where the `*tapApi.OutputChannelItem` leaves the code
 		// and goes into the intermediate WebSocket.
 		err = connection.WriteMessage(websocket.TextMessage, marshaledData)
@@ -320,6 +334,7 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time.
 	for i := 1; i < retryAmount; i++ {
 		socketConnection, _, err := dialer.Dial(socketAddress, nil)
 		if err != nil {
+			lastErr = err
 			if i < retryAmount {
 				logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay/time.Second)
 				time.Sleep(retryDelay)
@@ -372,6 +387,7 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
 func initializeDependencies() {
 	dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
 	dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} { return oas.GetDefaultOasGeneratorInstance() })
+	dependency.RegisterGenerator(dependency.EntriesInserter, func() interface{} { return api.GetBasenineEntryInserterInstance() })
 	dependency.RegisterGenerator(dependency.EntriesProvider, func() interface{} { return &entries.BasenineEntriesProvider{} })
 	dependency.RegisterGenerator(dependency.EntriesSocketStreamer, func() interface{} { return &api.BasenineEntryStreamer{} })
 	dependency.RegisterGenerator(dependency.EntryStreamerSocketConnector, func() interface{} { return &api.DefaultEntryStreamerSocketConnector{} })

agent/pkg/api/entry_streamer_socket_connector.go

@@ -5,20 +5,19 @@ import (
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 type EntryStreamerSocketConnector interface {
-	SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams)
-	SendMetadata(socketId int, metadata *basenine.Metadata)
-	SendToastError(socketId int, err error)
+	SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error
+	SendMetadata(socketId int, metadata *basenine.Metadata) error
+	SendToastError(socketId int, err error) error
 	CleanupSocket(socketId int)
 }
 
 type DefaultEntryStreamerSocketConnector struct{}
 
-func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) {
+func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error {
 	var message []byte
 	if params.EnableFullEntries {
 		message, _ = models.CreateFullEntryWebSocketMessage(entry)
@@ -29,26 +28,32 @@ func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tap
 	}
 
 	if err := SendToSocket(socketId, message); err != nil {
-		logger.Log.Error(err)
+		return err
 	}
+
+	return nil
 }
 
-func (e *DefaultEntryStreamerSocketConnector) SendMetadata(socketId int, metadata *basenine.Metadata) {
+func (e *DefaultEntryStreamerSocketConnector) SendMetadata(socketId int, metadata *basenine.Metadata) error {
 	metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
 	if err := SendToSocket(socketId, metadataBytes); err != nil {
-		logger.Log.Error(err)
+		return err
 	}
+
+	return nil
 }
 
-func (e *DefaultEntryStreamerSocketConnector) SendToastError(socketId int, err error) {
+func (e *DefaultEntryStreamerSocketConnector) SendToastError(socketId int, err error) error {
 	toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
 		Type:      "error",
 		AutoClose: 5000,
 		Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
 	})
 	if err := SendToSocket(socketId, toastBytes); err != nil {
-		logger.Log.Error(err)
+		return err
 	}
+
+	return nil
 }
 
 func (e *DefaultEntryStreamerSocketConnector) CleanupSocket(socketId int) {

agent/pkg/api/main.go

@@ -5,29 +5,27 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/up9inc/mizu/agent/pkg/models"
 	"os"
 	"path"
 	"sort"
 	"strings"
 	"time"
 
+	"github.com/up9inc/mizu/agent/pkg/models"
+
 	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/agent/pkg/elastic"
 	"github.com/up9inc/mizu/agent/pkg/har"
 	"github.com/up9inc/mizu/agent/pkg/holder"
 	"github.com/up9inc/mizu/agent/pkg/providers"
 
+	"github.com/up9inc/mizu/agent/pkg/oas"
 	"github.com/up9inc/mizu/agent/pkg/servicemap"
 
 	"github.com/up9inc/mizu/agent/pkg/resolver"
 	"github.com/up9inc/mizu/agent/pkg/utils"
 
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
-
-	basenine "github.com/up9inc/basenine/client/go"
 )
 
 var k8sResolver *resolver.Resolver
@@ -102,12 +100,6 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		panic("Channel of captured messages is nil")
 	}
 
-	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
-	if err != nil {
-		panic(err)
-	}
-	connection.InsertMode()
-
 	disableOASValidation := false
 	ctx := context.Background()
 	doc, contractContent, router, err := loadOAS(ctx)
@@ -119,19 +111,24 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 	for item := range outputItems {
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation, namespace := resolveIP(item.ConnectionInfo)
+
+		if namespace == "" && item.Namespace != tapApi.UNKNOWN_NAMESPACE {
+			namespace = item.Namespace
+		}
+
 		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation, namespace)
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
 				var httpPair tapApi.HTTPRequestResponsePair
 				if err := json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair); err != nil {
 					logger.Log.Error(err)
+				} else {
+					contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
+					mizuEntry.ContractStatus = contract.Status
+					mizuEntry.ContractRequestReason = contract.RequestReason
+					mizuEntry.ContractResponseReason = contract.ResponseReason
+					mizuEntry.ContractContent = contract.Content
 				}
-
-				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
-				mizuEntry.ContractStatus = contract.Status
-				mizuEntry.ContractRequestReason = contract.RequestReason
-				mizuEntry.ContractResponseReason = contract.ResponseReason
-				mizuEntry.ContractContent = contract.Content
 			}
 
 			harEntry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
@@ -143,17 +140,22 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 
 		data, err := json.Marshal(mizuEntry)
 		if err != nil {
-			panic(err)
+			logger.Log.Errorf("Error while marshaling entry: %v", err)
+			continue
 		}
 
 		providers.EntryAdded(len(data))
 
-		connection.SendText(string(data))
+		entryInserter := dependency.GetInstance(dependency.EntriesInserter).(EntryInserter)
+		if err := entryInserter.Insert(mizuEntry); err != nil {
+			logger.Log.Errorf("Error inserting entry, err: %v", err)
+		}
 
 		serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMapSink)
 		serviceMapGenerator.NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
 
-		elastic.GetInstance().PushEntry(mizuEntry)
+		oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGeneratorSink)
+		oasGenerator.HandleEntry(mizuEntry)
 	}
 }
 

agent/pkg/api/socket_data_inserter.go

@@ -0,0 +1,71 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap/api"
+	"sync"
+	"time"
+)
+
+type EntryInserter interface {
+	Insert(entry *api.Entry) error
+}
+
+type BasenineEntryInserter struct {
+	connection *basenine.Connection
+}
+
+var instance *BasenineEntryInserter
+var once sync.Once
+
+func GetBasenineEntryInserterInstance() *BasenineEntryInserter {
+	once.Do(func() {
+		instance = &BasenineEntryInserter{}
+	})
+
+	return instance
+}
+
+func (e *BasenineEntryInserter) Insert(entry *api.Entry) error {
+	if e.connection == nil {
+		e.connection = initializeConnection()
+	}
+
+	data, err := json.Marshal(entry)
+	if err != nil {
+		return fmt.Errorf("error marshling entry, err: %v", err)
+	}
+
+	if err := e.connection.SendText(string(data)); err != nil {
+		e.connection.Close()
+		e.connection = nil
+
+		return fmt.Errorf("error sending text to database, err: %v", err)
+	}
+
+	return nil
+}
+
+func initializeConnection() *basenine.Connection{
+	for {
+		connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+		if err != nil {
+			logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
+			time.Sleep(shared.BasenineReconnectInterval * time.Second)
+			continue
+		}
+
+		if err = connection.InsertMode(); err != nil {
+			logger.Log.Errorf("Insert mode call failed: %v", err)
+			connection.Close()
+			time.Sleep(shared.BasenineReconnectInterval * time.Second)
+			continue
+		}
+
+		return connection
+	}
+}

agent/pkg/api/socket_data_streamer.go

@@ -3,11 +3,12 @@ package api
 import (
 	"context"
 	"encoding/json"
+	"time"
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
@@ -24,7 +25,7 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 
 	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
 	if err != nil {
-		logger.Log.Errorf("failed to establish a connection to Basenine: %v", err)
+		logger.Log.Errorf("Failed to establish a connection to Basenine: %v", err)
 		entryStreamerSocketConnector.CleanupSocket(socketId)
 		return err
 	}
@@ -33,9 +34,18 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 	meta := make(chan []byte)
 
 	query := params.Query
-	err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
+	if err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query); err != nil {
+		if err := entryStreamerSocketConnector.SendToastError(socketId, err); err != nil {
+			return err
+		}
+
+		entryStreamerSocketConnector.CleanupSocket(socketId)
+		return err
+	}
+
+	leftOff, err := e.fetch(socketId, params, entryStreamerSocketConnector)
 	if err != nil {
-		entryStreamerSocketConnector.SendToastError(socketId, err)
+		logger.Log.Errorf("Fetch error: %v", err)
 	}
 
 	handleDataChannel := func(c *basenine.Connection, data chan []byte) {
@@ -47,13 +57,15 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 			}
 
 			var entry *tapApi.Entry
-			err = json.Unmarshal(bytes, &entry)
-			if err != nil {
-				logger.Log.Debugf("error unmarshalling entry: %v", err.Error())
+			if err = json.Unmarshal(bytes, &entry); err != nil {
+				logger.Log.Debugf("Error unmarshalling entry: %v", err)
 				continue
 			}
 
-			entryStreamerSocketConnector.SendEntry(socketId, entry, params)
+			if err := entryStreamerSocketConnector.SendEntry(socketId, entry, params); err != nil {
+				logger.Log.Errorf("Error sending entry to socket, err: %v", err)
+				return
+			}
 		}
 	}
 
@@ -66,20 +78,26 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 			}
 
 			var metadata *basenine.Metadata
-			err = json.Unmarshal(bytes, &metadata)
-			if err != nil {
-				logger.Log.Debugf("Error unmarshalling metadata: %v", err.Error())
+			if err = json.Unmarshal(bytes, &metadata); err != nil {
+				logger.Log.Debugf("Error unmarshalling metadata: %v", err)
 				continue
 			}
 
-			entryStreamerSocketConnector.SendMetadata(socketId, metadata)
+			if err := entryStreamerSocketConnector.SendMetadata(socketId, metadata); err != nil {
+				logger.Log.Errorf("Error sending metadata to socket, err: %v", err)
+				return
+			}
 		}
 	}
 
 	go handleDataChannel(connection, data)
 	go handleMetaChannel(connection, meta)
 
-	connection.Query(query, data, meta)
+	if err = connection.Query(leftOff, query, data, meta); err != nil {
+		logger.Log.Errorf("Query mode call failed: %v", err)
+		entryStreamerSocketConnector.CleanupSocket(socketId)
+		return err
+	}
 
 	go func() {
 		<-ctx.Done()
@@ -90,3 +108,64 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 
 	return nil
 }
+
+// Reverses a []byte slice.
+func (e *BasenineEntryStreamer) fetch(socketId int, params *WebSocketParams, connector EntryStreamerSocketConnector) (leftOff string, err error) {
+	if params.Fetch <= 0 {
+		leftOff = params.LeftOff
+		return
+	}
+
+	var data [][]byte
+	var firstMeta []byte
+	var lastMeta []byte
+	data, firstMeta, lastMeta, err = basenine.Fetch(
+		shared.BasenineHost,
+		shared.BaseninePort,
+		params.LeftOff,
+		-1,
+		params.Query,
+		params.Fetch,
+		time.Duration(params.TimeoutMs)*time.Millisecond,
+	)
+	if err != nil {
+		return
+	}
+
+	var firstMetadata *basenine.Metadata
+	if err = json.Unmarshal(firstMeta, &firstMetadata); err != nil {
+		return
+	}
+
+	leftOff = firstMetadata.LeftOff
+
+	var lastMetadata *basenine.Metadata
+	if err = json.Unmarshal(lastMeta, &lastMetadata); err != nil {
+		return
+	}
+
+	if err = connector.SendMetadata(socketId, lastMetadata); err != nil {
+		return
+	}
+
+	data = e.reverseBytesSlice(data)
+	for _, row := range data {
+		var entry *tapApi.Entry
+		if err = json.Unmarshal(row, &entry); err != nil {
+			break
+		}
+
+		if err = connector.SendEntry(socketId, entry, params); err != nil {
+			return
+		}
+	}
+	return
+}
+
+// Reverses a []byte slice.
+func (e *BasenineEntryStreamer) reverseBytesSlice(arr [][]byte) (newArr [][]byte) {
+	for i := len(arr) - 1; i >= 0; i-- {
+		newArr = append(newArr, arr[i])
+	}
+	return newArr
+}

agent/pkg/api/socket_routes.go

@@ -10,7 +10,7 @@ import (
 	"github.com/gorilla/websocket"
 	"github.com/up9inc/mizu/agent/pkg/models"
 	"github.com/up9inc/mizu/agent/pkg/utils"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
@@ -34,8 +34,11 @@ type SocketConnection struct {
 }
 
 type WebSocketParams struct {
+	LeftOff           string `json:"leftOff"`
 	Query             string `json:"query"`
 	EnableFullEntries bool   `json:"enableFullEntries"`
+	Fetch             int    `json:"fetch"`
+	TimeoutMs         int    `json:"timeoutMs"`
 }
 
 var (
@@ -123,22 +126,23 @@ func SendToSocket(socketId int, message []byte) error {
 		return fmt.Errorf("socket %v is disconnected", socketId)
 	}
 
-	var sent = false
-	time.AfterFunc(time.Second*5, func() {
-		if !sent {
-			logger.Log.Error("socket timed out")
-			socketCleanup(socketId, socketObj)
-		}
-	})
-
 	socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
-	err := socketObj.connection.WriteMessage(1, message)
-	socketObj.lock.Unlock()
-	sent = true
+	defer socketObj.lock.Unlock()
 
-	if err != nil {
-		return fmt.Errorf("failed to write message to socket %v, err: %w", socketId, err)
+	if connectedWebsockets[socketId] == nil {
+		return fmt.Errorf("socket %v is disconnected", socketId)
 	}
+
+	if err := socketObj.connection.SetWriteDeadline(time.Now().Add(time.Second * 10)); err != nil {
+		socketCleanup(socketId, socketObj)
+		return fmt.Errorf("error setting timeout to socket %v, err: %v", socketId, err)
+	}
+
+	if err := socketObj.connection.WriteMessage(websocket.TextMessage, message); err != nil {
+		socketCleanup(socketId, socketObj)
+		return fmt.Errorf("failed to write message to socket %v, err: %v", socketId, err)
+	}
+
 	return nil
 }
 

agent/pkg/api/socket_server_handlers.go

@@ -14,8 +14,8 @@ import (
 
 	tapApi "github.com/up9inc/mizu/tap/api"
 
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 type BrowserClient struct {

agent/pkg/api/utils.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 
 	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 func BroadcastTappedPodsStatus() {

agent/pkg/app/main.go

@@ -10,7 +10,8 @@ import (
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/api"
 	"github.com/up9inc/mizu/agent/pkg/utils"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/tap/dbgctl"
 	tapApi "github.com/up9inc/mizu/tap/api"
 	amqpExt "github.com/up9inc/mizu/tap/extensions/amqp"
 	httpExt "github.com/up9inc/mizu/tap/extensions/http"
@@ -24,36 +25,38 @@ var (
 )
 
 func LoadExtensions() {
-	Extensions = make([]*tapApi.Extension, 4)
+	Extensions = make([]*tapApi.Extension, 0)
 	ExtensionsMap = make(map[string]*tapApi.Extension)
 
-	extensionAmqp := &tapApi.Extension{}
-	dissectorAmqp := amqpExt.NewDissector()
-	dissectorAmqp.Register(extensionAmqp)
-	extensionAmqp.Dissector = dissectorAmqp
-	Extensions[0] = extensionAmqp
-	ExtensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
-
 	extensionHttp := &tapApi.Extension{}
 	dissectorHttp := httpExt.NewDissector()
 	dissectorHttp.Register(extensionHttp)
 	extensionHttp.Dissector = dissectorHttp
-	Extensions[1] = extensionHttp
+	Extensions = append(Extensions, extensionHttp)
 	ExtensionsMap[extensionHttp.Protocol.Name] = extensionHttp
 
-	extensionKafka := &tapApi.Extension{}
-	dissectorKafka := kafkaExt.NewDissector()
-	dissectorKafka.Register(extensionKafka)
-	extensionKafka.Dissector = dissectorKafka
-	Extensions[2] = extensionKafka
-	ExtensionsMap[extensionKafka.Protocol.Name] = extensionKafka
+	if !dbgctl.MizuTapperDisableNonHttpExtensions {
+		extensionAmqp := &tapApi.Extension{}
+		dissectorAmqp := amqpExt.NewDissector()
+		dissectorAmqp.Register(extensionAmqp)
+		extensionAmqp.Dissector = dissectorAmqp
+		Extensions = append(Extensions, extensionAmqp)
+		ExtensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
 
-	extensionRedis := &tapApi.Extension{}
-	dissectorRedis := redisExt.NewDissector()
-	dissectorRedis.Register(extensionRedis)
-	extensionRedis.Dissector = dissectorRedis
-	Extensions[3] = extensionRedis
-	ExtensionsMap[extensionRedis.Protocol.Name] = extensionRedis
+		extensionKafka := &tapApi.Extension{}
+		dissectorKafka := kafkaExt.NewDissector()
+		dissectorKafka.Register(extensionKafka)
+		extensionKafka.Dissector = dissectorKafka
+		Extensions = append(Extensions, extensionKafka)
+		ExtensionsMap[extensionKafka.Protocol.Name] = extensionKafka
+
+		extensionRedis := &tapApi.Extension{}
+		dissectorRedis := redisExt.NewDissector()
+		dissectorRedis.Register(extensionRedis)
+		extensionRedis.Dissector = dissectorRedis
+		Extensions = append(Extensions, extensionRedis)
+		ExtensionsMap[extensionRedis.Protocol.Name] = extensionRedis
+	}
 
 	sort.Slice(Extensions, func(i, j int) bool {
 		return Extensions[i].Protocol.Priority < Extensions[j].Protocol.Priority
@@ -67,7 +70,7 @@ func ConfigureBasenineServer(host string, port string, dbSize int64, logLevel lo
 		wait.WithProto("tcp"),
 		wait.WithWait(200*time.Millisecond),
 		wait.WithBreak(50*time.Millisecond),
-		wait.WithDeadline(5*time.Second),
+		wait.WithDeadline(20*time.Second),
 		wait.WithDebug(logLevel == logging.DEBUG),
 	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
 		logger.Log.Panicf("Basenine is not available!")

agent/pkg/controllers/entries_controller.go

@@ -2,7 +2,6 @@ package controllers
 
 import (
 	"net/http"
-	"strconv"
 
 	"github.com/up9inc/mizu/agent/pkg/dependency"
 	"github.com/up9inc/mizu/agent/pkg/entries"
@@ -11,7 +10,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 func HandleEntriesError(c *gin.Context, err error) bool {
@@ -69,7 +68,7 @@ func GetEntry(c *gin.Context) {
 		c.JSON(http.StatusBadRequest, validationError)
 	}
 
-	id, _ := strconv.Atoi(c.Param("id"))
+	id := c.Param("id")
 
 	entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
 	entry, err := entriesProvider.GetEntry(singleEntryRequest, id)

agent/pkg/controllers/oas_controller.go

@@ -7,7 +7,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/up9inc/mizu/agent/pkg/dependency"
 	"github.com/up9inc/mizu/agent/pkg/oas"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 func GetOASServers(c *gin.Context) {

agent/pkg/controllers/oas_controller_test.go

@@ -1,17 +1,12 @@
 package controllers
 
 import (
-	"bytes"
-	basenine "github.com/up9inc/basenine/client/go"
-	"net"
 	"net/http/httptest"
 	"testing"
-	"time"
-
-	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/agent/pkg/oas"
 
 	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/oas"
 )
 
 func TestGetOASServers(t *testing.T) {
@@ -37,33 +32,14 @@ func TestGetOASSpec(t *testing.T) {
 	t.Logf("Written body: %s", recorder.Body.String())
 }
 
-type fakeConn struct {
-	sendBuffer    *bytes.Buffer
-	receiveBuffer *bytes.Buffer
-}
-
-func (f fakeConn) Read(p []byte) (int, error)       { return f.sendBuffer.Read(p) }
-func (f fakeConn) Write(p []byte) (int, error)      { return f.receiveBuffer.Write(p) }
-func (fakeConn) Close() error                       { return nil }
-func (fakeConn) LocalAddr() net.Addr                { return nil }
-func (fakeConn) RemoteAddr() net.Addr               { return nil }
-func (fakeConn) SetDeadline(t time.Time) error      { return nil }
-func (fakeConn) SetReadDeadline(t time.Time) error  { return nil }
-func (fakeConn) SetWriteDeadline(t time.Time) error { return nil }
-
 func getRecorderAndContext() (*httptest.ResponseRecorder, *gin.Context) {
-	dummyConn := new(basenine.Connection)
-	dummyConn.Conn = fakeConn{
-		sendBuffer:    bytes.NewBufferString("\n"),
-		receiveBuffer: bytes.NewBufferString("\n"),
-	}
 	dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} {
 		return oas.GetDefaultOasGeneratorInstance()
 	})
 
 	recorder := httptest.NewRecorder()
 	c, _ := gin.CreateTestContext(recorder)
-	oas.GetDefaultOasGeneratorInstance().Start(dummyConn)
+	oas.GetDefaultOasGeneratorInstance().Start()
 	oas.GetDefaultOasGeneratorInstance().GetServiceSpecs().Store("some", oas.NewGen("some"))
 	return recorder, c
 }

agent/pkg/controllers/status_controller.go

@@ -2,6 +2,7 @@ package controllers
 
 import (
 	"net/http"
+
 	core "k8s.io/api/core/v1"
 
 	"github.com/gin-gonic/gin"
@@ -12,9 +13,9 @@ import (
 	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
 	"github.com/up9inc/mizu/agent/pkg/up9"
 	"github.com/up9inc/mizu/agent/pkg/validation"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 func HealthCheck(c *gin.Context) {
@@ -93,10 +94,6 @@ func GetGeneralStats(c *gin.Context) {
 	c.JSON(http.StatusOK, providers.GetGeneralStats())
 }
 
-func GetRecentTLSLinks(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
-}
-
 func GetCurrentResolvingInformation(c *gin.Context) {
 	c.JSON(http.StatusOK, holder.GetResolver().GetMap())
 }

agent/pkg/dependency/type_names.go

@@ -5,6 +5,7 @@ type DependencyContainerType string
 const (
 	ServiceMapGeneratorDependency = "ServiceMapGeneratorDependency"
 	OasGeneratorDependency        = "OasGeneratorDependency"
+	EntriesInserter               = "EntriesInserter"
 	EntriesProvider               = "EntriesProvider"
 	EntriesSocketStreamer         = "EntriesSocketStreamer"
 	EntryStreamerSocketConnector  = "EntryStreamerSocketConnector"

agent/pkg/elastic/esClient.go

@@ -1,116 +0,0 @@
-package elastic
-
-import (
-	"bytes"
-	"crypto/tls"
-	"encoding/json"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/elastic/go-elasticsearch/v7"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/tap/api"
-)
-
-type client struct {
-	es            *elasticsearch.Client
-	index         string
-	insertedCount int
-}
-
-var instance *client
-var once sync.Once
-
-func GetInstance() *client {
-	once.Do(func() {
-		instance = newClient()
-	})
-	return instance
-}
-
-func (client *client) Configure(config shared.ElasticConfig) {
-	if config.Url == "" || config.User == "" || config.Password == "" {
-		if client.es != nil {
-			client.es = nil
-		}
-		logger.Log.Infof("No elastic configuration was supplied, elastic exporter disabled")
-		return
-	}
-	transport := http.DefaultTransport
-	tlsClientConfig := &tls.Config{InsecureSkipVerify: true}
-	transport.(*http.Transport).TLSClientConfig = tlsClientConfig
-	cfg := elasticsearch.Config{
-		Addresses: []string{config.Url},
-		Username:  config.User,
-		Password:  config.Password,
-		Transport: transport,
-	}
-
-	es, err := elasticsearch.NewClient(cfg)
-	if err != nil {
-		logger.Log.Errorf("Failed to initialize elastic client %v", err)
-	}
-
-	// Have the client instance return a response
-	res, err := es.Info()
-	if err != nil {
-		logger.Log.Errorf("Elastic client.Info() ERROR: %v", err)
-	} else {
-		client.es = es
-		client.index = "mizu_traffic_http_" + time.Now().Format("2006_01_02_15_04")
-		client.insertedCount = 0
-		logger.Log.Infof("Elastic client configured, index: %s, cluster info: %v", client.index, res)
-	}
-	defer res.Body.Close()
-}
-
-func newClient() *client {
-	return &client{
-		es:    nil,
-		index: "",
-	}
-}
-
-type httpEntry struct {
-	Source      *api.TCP               `json:"src"`
-	Destination *api.TCP               `json:"dst"`
-	Outgoing    bool                   `json:"outgoing"`
-	CreatedAt   time.Time              `json:"createdAt"`
-	Request     map[string]interface{} `json:"request"`
-	Response    map[string]interface{} `json:"response"`
-	ElapsedTime int64                  `json:"elapsedTime"`
-}
-
-func (client *client) PushEntry(entry *api.Entry) {
-	if client.es == nil {
-		return
-	}
-
-	if entry.Protocol.Name != "http" {
-		return
-	}
-
-	entryToPush := httpEntry{
-		Source:      entry.Source,
-		Destination: entry.Destination,
-		Outgoing:    entry.Outgoing,
-		CreatedAt:   entry.StartTime,
-		Request:     entry.Request,
-		Response:    entry.Response,
-		ElapsedTime: entry.ElapsedTime,
-	}
-
-	entryJson, err := json.Marshal(entryToPush)
-	if err != nil {
-		logger.Log.Errorf("json.Marshal ERROR: %v", err)
-		return
-	}
-	var buffer bytes.Buffer
-	buffer.WriteString(string(entryJson))
-	res, _ := client.es.Index(client.index, &buffer)
-	if res.StatusCode == 201 {
-		client.insertedCount += 1
-	}
-}

agent/pkg/entries/entries_provider.go

@@ -2,26 +2,27 @@ package entries
 
 import (
 	"encoding/json"
+	"errors"
 	"time"
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/app"
 	"github.com/up9inc/mizu/agent/pkg/har"
 	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 type EntriesProvider interface {
 	GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error)
-	GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId int) (*tapApi.EntryWrapper, error)
+	GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error)
 }
 
 type BasenineEntriesProvider struct{}
 
 func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error) {
-	data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
+	data, _, lastMeta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
 		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
 		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
 	if err != nil {
@@ -48,7 +49,7 @@ func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesReque
 	}
 
 	var metadata *basenine.Metadata
-	err = json.Unmarshal(meta, &metadata)
+	err = json.Unmarshal(lastMeta, &metadata)
 	if err != nil {
 		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
 	}
@@ -56,7 +57,7 @@ func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesReque
 	return dataSlice, metadata, nil
 }
 
-func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId int) (*tapApi.EntryWrapper, error) {
+func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error) {
 	var entry *tapApi.Entry
 	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, entryId, singleEntryRequest.Query)
 	if err != nil {
@@ -64,7 +65,7 @@ func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntr
 	}
 	err = json.Unmarshal(bytes, &entry)
 	if err != nil {
-		return nil, err
+		return nil, errors.New(string(bytes))
 	}
 
 	extension := app.ExtensionsMap[entry.Protocol.Name]

agent/pkg/har/types.go

@@ -2,9 +2,10 @@ package har
 
 import (
 	"encoding/base64"
-	"github.com/up9inc/mizu/shared/logger"
 	"time"
 	"unicode/utf8"
+
+	"github.com/up9inc/mizu/logger"
 )
 
 /*

agent/pkg/har/utils.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 // Keep it because we might want cookies in the future
@@ -124,8 +124,8 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
 
 	postData, _ := request["postData"].(map[string]interface{})
 	mimeType := postData["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
+	if mimeType == nil {
+		mimeType = ""
 	}
 	text := postData["text"]
 	postDataText := ""
@@ -177,8 +177,8 @@ func NewResponse(response map[string]interface{}) (harResponse *Response, err er
 
 	content, _ := response["content"].(map[string]interface{})
 	mimeType := content["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
+	if mimeType == nil {
+		mimeType = ""
 	}
 	encoding := content["encoding"]
 	text := content["text"]

agent/pkg/models/models.go

@@ -9,11 +9,10 @@ import (
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap"
 )
 
 type EntriesRequest struct {
-	LeftOff   int    `form:"leftOff" validate:"required,min=-1"`
+	LeftOff   string `form:"leftOff" validate:"required"`
 	Direction int    `form:"direction" validate:"required,oneof='1' '-1'"`
 	Query     string `form:"query"`
 	Limit     int    `form:"limit" validate:"required,min=1"`
@@ -44,11 +43,6 @@ type WebSocketTappedEntryMessage struct {
 	Data *tapApi.OutputChannelItem
 }
 
-type WebsocketOutboundLinkMessage struct {
-	*shared.WebSocketMessageMetadata
-	Data *tap.OutboundLink
-}
-
 type AuthStatus struct {
 	Email string `json:"email"`
 	Model string `json:"model"`

agent/pkg/oas/feeder_test.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/url"
@@ -15,7 +16,7 @@ import (
 
 	"github.com/up9inc/mizu/agent/pkg/har"
 
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 func getFiles(baseDir string) (result []string, err error) {
@@ -124,13 +125,13 @@ func feedFromHAR(file string, isSync bool, gen *defaultOasGenerator) (uint, erro
 	cnt := uint(0)
 	for _, entry := range harDoc.Log.Entries {
 		cnt += 1
-		feedEntry(&entry, "", file, gen, cnt)
+		feedEntry(&entry, "", file, gen, fmt.Sprintf("%024d", cnt))
 	}
 
 	return cnt, nil
 }
 
-func feedEntry(entry *har.Entry, source string, file string, gen *defaultOasGenerator, cnt uint) {
+func feedEntry(entry *har.Entry, source string, file string, gen *defaultOasGenerator, cnt string) {
 	entry.Comment = file
 	if entry.Response.Status == 302 {
 		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
@@ -192,7 +193,7 @@ func feedFromLDJSON(file string, isSync bool, gen *defaultOasGenerator) (uint, e
 				logger.Log.Warningf("Failed decoding entry: %s", line)
 			} else {
 				cnt += 1
-				feedEntry(&entry, source, file, gen, cnt)
+				feedEntry(&entry, source, file, gen, fmt.Sprintf("%024d", cnt))
 			}
 		}
 	}

agent/pkg/oas/oas-samples/carts.json

@@ -0,0 +1,96 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "http://carts",
+    "description": "Mizu observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
+    "version": "1.0"
+  },
+  "servers": [
+    {
+      "url": "http://carts"
+    }
+  ],
+  "paths": {
+    "/carts/{cartId}/items": {
+      "get": {
+        "summary": "/carts/{cartId}/items",
+        "description": "Mizu observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
+        "operationId": "84c9b926-1f73-4ab4-b381-3c124528959f",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": [
+                  {
+                    "id": "60fe98fb86c0fc000869a90c",
+                    "itemId": "3395a43e-2d88-40de-b95f-e00e1502085b",
+                    "quantity": 1,
+                    "unitPrice": 18
+                  }
+                ],
+                "x-sample-entry": "000000000000000000000010"
+              }
+            },
+            "x-sample-entry": "000000000000000000000010"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 3,
+            "failures": 0,
+            "firstSeen": 1627298058.3798368,
+            "lastSeen": 1627298065.2397773,
+            "sumRT": 0.05,
+            "sumDuration": 6.859940528869629
+          }
+        },
+        "x-counters-total": {
+          "entries": 3,
+          "failures": 0,
+          "firstSeen": 1627298058.3798368,
+          "lastSeen": 1627298065.2397773,
+          "sumRT": 0.05,
+          "sumDuration": 6.859940528869629
+        },
+        "x-last-seen-ts": 1627298065.2397773,
+        "x-sample-entry": "000000000000000000000010"
+      },
+      "parameters": [
+        {
+          "name": "cartId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "mHK0P7zTktmV1zv57iWAvCTd43FFMHap"
+            }
+          },
+          "x-sample-entry": "000000000000000000000010"
+        }
+      ]
+    }
+  },
+  "x-counters-per-source": {
+    "some-source": {
+      "entries": 3,
+      "failures": 0,
+      "firstSeen": 1627298058.3798368,
+      "lastSeen": 1627298065.2397773,
+      "sumRT": 0.05,
+      "sumDuration": 6.859940528869629
+    }
+  },
+  "x-counters-total": {
+    "entries": 3,
+    "failures": 0,
+    "firstSeen": 1627298058.3798368,
+    "lastSeen": 1627298065.2397773,
+    "sumRT": 0.05,
+    "sumDuration": 6.859940528869629
+  }
+}

agent/pkg/oas/oas-samples/catalogue.json

@@ -0,0 +1,485 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "Preloaded",
+    "description": "Test file for loading pre-existing OAS",
+    "version": "0.1"
+  },
+  "paths": {
+    "/catalogue": {
+      "get": {
+        "tags": [
+          "catalogue"
+        ],
+        "summary": "/catalogue",
+        "description": "Mizu observed 3 entries (0 failed), at 2.647 hits/s, average response time is 0.008 seconds",
+        "operationId": "dd6c3dbe-6b6b-4ddd-baed-757e237ddb8a",
+        "parameters": [
+          {
+            "name": "page",
+            "in": "query",
+            "required": false,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "1"
+              }
+            },
+            "x-sample-entry": "000000000000000000000002"
+          },
+          {
+            "name": "size",
+            "in": "query",
+            "required": true,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "6"
+              },
+              "example #1": {
+                "value": "3"
+              },
+              "example #2": {
+                "value": "5"
+              }
+            },
+            "x-sample-entry": "000000000000000000000011"
+          },
+          {
+            "name": "tags",
+            "in": "query",
+            "required": false,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": ""
+              },
+              "example #1": {
+                "value": "blue"
+              }
+            },
+            "x-sample-entry": "000000000000000000000007"
+          },
+          {
+            "name": "sort",
+            "in": "query",
+            "required": false,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "id"
+              }
+            },
+            "x-sample-entry": "000000000000000000000007"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": [
+                  {
+                    "count": 1,
+                    "description": "Socks fit for a Messiah. You too can experience walking in water with these special edition beauties. Each hole is lovingly proggled to leave smooth edges. The only sock approved by a higher power.",
+                    "id": "03fef6ac-1896-4ce8-bd69-b798f85c6e0b",
+                    "imageUrl": [
+                      "/catalogue/images/holy_1.jpeg",
+                      "/catalogue/images/holy_2.jpeg"
+                    ],
+                    "name": "Holy",
+                    "price": 99.99,
+                    "tag": [
+                      "action",
+                      "magic"
+                    ]
+                  },
+                  {
+                    "count": 438,
+                    "description": "proident occaecat irure et excepteur labore minim nisi amet irure",
+                    "id": "3395a43e-2d88-40de-b95f-e00e1502085b",
+                    "imageUrl": [
+                      "/catalogue/images/colourful_socks.jpg",
+                      "/catalogue/images/colourful_socks.jpg"
+                    ],
+                    "name": "Colourful",
+                    "price": 18,
+                    "tag": [
+                      "brown",
+                      "blue"
+                    ]
+                  },
+                  {
+                    "count": 820,
+                    "description": "Ready for action. Engineers: be ready to smash that next bug! Be ready, with these super-action-sport-masterpieces. This particular engineer was chased away from the office with a stick.",
+                    "id": "510a0d7e-8e83-4193-b483-e27e09ddc34d",
+                    "imageUrl": [
+                      "/catalogue/images/puma_1.jpeg",
+                      "/catalogue/images/puma_2.jpeg"
+                    ],
+                    "name": "SuperSport XL",
+                    "price": 15,
+                    "tag": [
+                      "sport",
+                      "formal",
+                      "black"
+                    ]
+                  },
+                  {
+                    "count": 738,
+                    "description": "A mature sock, crossed, with an air of nonchalance.",
+                    "id": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38",
+                    "imageUrl": [
+                      "/catalogue/images/cross_1.jpeg",
+                      "/catalogue/images/cross_2.jpeg"
+                    ],
+                    "name": "Crossed",
+                    "price": 17.32,
+                    "tag": [
+                      "blue",
+                      "action",
+                      "red",
+                      "formal"
+                    ]
+                  },
+                  {
+                    "count": 808,
+                    "description": "enim officia aliqua excepteur esse deserunt quis aliquip nostrud anim",
+                    "id": "819e1fbf-8b7e-4f6d-811f-693534916a8b",
+                    "imageUrl": [
+                      "/catalogue/images/WAT.jpg",
+                      "/catalogue/images/WAT2.jpg"
+                    ],
+                    "name": "Figueroa",
+                    "price": 14,
+                    "tag": [
+                      "green",
+                      "formal",
+                      "blue"
+                    ]
+                  },
+                  {
+                    "count": 175,
+                    "description": "consequat amet cupidatat minim laborum tempor elit ex consequat in",
+                    "id": "837ab141-399e-4c1f-9abc-bace40296bac",
+                    "imageUrl": [
+                      "/catalogue/images/catsocks.jpg",
+                      "/catalogue/images/catsocks2.jpg"
+                    ],
+                    "name": "Cat socks",
+                    "price": 15,
+                    "tag": [
+                      "brown",
+                      "formal",
+                      "green"
+                    ]
+                  }
+                ],
+                "x-sample-entry": "000000000000000000000011"
+              }
+            },
+            "x-sample-entry": "000000000000000000000011"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 3,
+            "failures": 0,
+            "firstSeen": 1627298057.7849188,
+            "lastSeen": 1627298065.7258668,
+            "sumRT": 0.024999999999999998,
+            "sumDuration": 7.940948009490967
+          }
+        },
+        "x-counters-total": {
+          "entries": 3,
+          "failures": 0,
+          "firstSeen": 1627298057.7849188,
+          "lastSeen": 1627298065.7258668,
+          "sumRT": 0.024999999999999998,
+          "sumDuration": 7.940948009490967
+        },
+        "x-last-seen-ts": 1627298065.7258668,
+        "x-sample-entry": "000000000000000000000011"
+      }
+    },
+    "/catalogue/size": {
+      "get": {
+        "tags": [
+          "catalogue"
+        ],
+        "summary": "/catalogue/size",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.013 seconds",
+        "operationId": "2315e69d-9d66-48cf-b3d3-fec9c30bd28b",
+        "parameters": [
+          {
+            "name": "tags",
+            "in": "query",
+            "required": true,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": ""
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
+          },
+          {
+            "name": "x-some",
+            "in": "header",
+            "required": true,
+            "style": "simple",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "demo val"
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": {
+                  "err": null,
+                  "size": 9
+                },
+                "x-sample-entry": "000000000000000000000001"
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1627298057.7841518,
+            "lastSeen": 1627298057.7841518,
+            "sumRT": 0.013,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1627298057.7841518,
+          "lastSeen": 1627298057.7841518,
+          "sumRT": 0.013,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1627298057.7841518,
+        "x-sample-entry": "000000000000000000000001"
+      }
+    },
+    "/catalogue/{id}": {
+      "get": {
+        "tags": [
+          "catalogue"
+        ],
+        "summary": "/catalogue/{id}",
+        "description": "Mizu observed 4 entries (0 failed), at 1.899 hits/s, average response time is 0.003 seconds",
+        "parameters": [
+          {
+            "name": "non-required-header",
+            "in": "header",
+            "required": false,
+            "style": "simple",
+            "schema": {
+              "type": "string"
+            },
+            "example": "some-uuid-maybe"
+          },
+          {
+            "name": "x-some",
+            "in": "header",
+            "required": false,
+            "style": "simple",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "demoval"
+              }
+            },
+            "x-sample-entry": "000000000000000000000004"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": {
+                  "count": 438,
+                  "description": "proident occaecat irure et excepteur labore minim nisi amet irure",
+                  "id": "3395a43e-2d88-40de-b95f-e00e1502085b",
+                  "imageUrl": [
+                    "/catalogue/images/colourful_socks.jpg",
+                    "/catalogue/images/colourful_socks.jpg"
+                  ],
+                  "name": "Colourful",
+                  "price": 18,
+                  "tag": [
+                    "brown",
+                    "blue"
+                  ]
+                },
+                "x-sample-entry": "000000000000000000000012"
+              }
+            },
+            "x-sample-entry": "000000000000000000000012"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 4,
+            "failures": 0,
+            "firstSeen": 1627298058.1315014,
+            "lastSeen": 1627298065.7293031,
+            "sumRT": 0.013999999999999999,
+            "sumDuration": 7.597801685333252
+          }
+        },
+        "x-counters-total": {
+          "entries": 4,
+          "failures": 0,
+          "firstSeen": 1627298058.1315014,
+          "lastSeen": 1627298065.7293031,
+          "sumRT": 0.013999999999999999,
+          "sumDuration": 7.597801685333252
+        },
+        "x-last-seen-ts": 1627298065.7293031,
+        "x-sample-entry": "000000000000000000000012"
+      },
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "3395a43e-2d88-40de-b95f-e00e1502085b"
+            },
+            "example #1": {
+              "value": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38"
+            }
+          },
+          "example": "some-uuid-maybe",
+          "x-sample-entry": "000000000000000000000012"
+        }
+      ]
+    },
+    "/catalogue/{id}/details": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ]
+    },
+    "/tags": {
+      "get": {
+        "summary": "/tags",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.007 seconds",
+        "operationId": "c4d7d0ed-1a78-4370-a049-efe3abc631a6",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": {
+                  "err": null,
+                  "tags": [
+                    "brown",
+                    "geek",
+                    "formal",
+                    "blue",
+                    "skin",
+                    "red",
+                    "action",
+                    "sport",
+                    "black",
+                    "magic",
+                    "green"
+                  ]
+                },
+                "x-sample-entry": "000000000000000000000003"
+              }
+            },
+            "x-sample-entry": "000000000000000000000003"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1627298057.7841816,
+            "lastSeen": 1627298057.7841816,
+            "sumRT": 0.007,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1627298057.7841816,
+          "lastSeen": 1627298057.7841816,
+          "sumRT": 0.007,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1627298057.7841816,
+        "x-sample-entry": "000000000000000000000003"
+      }
+    }
+  },
+  "x-counters-per-source": {
+    "some-source": {
+      "entries": 9,
+      "failures": 0,
+      "firstSeen": 1627298057.7841518,
+      "lastSeen": 1627298065.7293031,
+      "sumRT": 0.05899999999999999,
+      "sumDuration": 15.538749694824219
+    }
+  },
+  "x-counters-total": {
+    "entries": 9,
+    "failures": 0,
+    "firstSeen": 1627298057.7841518,
+    "lastSeen": 1627298065.7293031,
+    "sumRT": 0.05899999999999999,
+    "sumDuration": 15.538749694824219
+  }
+}

agent/pkg/oas/oas-samples/httpbin.org.json

@@ -0,0 +1,897 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "https://httpbin.org",
+    "description": "Mizu observed 19 entries (0 failed), at 0.106 hits/s, average response time is 0.172 seconds",
+    "version": "1.0"
+  },
+  "servers": [
+    {
+      "url": "https://httpbin.org"
+    }
+  ],
+  "paths": {
+    "/appears-once": {
+      "get": {
+        "summary": "/appears-once",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "2d34623e-fde8-4720-8390-9a7439051755",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000004"
+              }
+            },
+            "x-sample-entry": "000000000000000000000004"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750580.0471218,
+            "lastSeen": 1567750580.0471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750580.0471218,
+          "lastSeen": 1567750580.0471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750580.0471218,
+        "x-sample-entry": "000000000000000000000004"
+      }
+    },
+    "/appears-twice": {
+      "get": {
+        "summary": "/appears-twice",
+        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds",
+        "operationId": "9c5330f3-8062-468b-b5a3-df1ad82b4846",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000006"
+              }
+            },
+            "x-sample-entry": "000000000000000000000006"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750580.7471218,
+            "lastSeen": 1567750581.7471218,
+            "sumRT": 1.26,
+            "sumDuration": 1
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750580.7471218,
+          "lastSeen": 1567750581.7471218,
+          "sumRT": 1.26,
+          "sumDuration": 1
+        },
+        "x-last-seen-ts": 1567750581.7471218,
+        "x-sample-entry": "000000000000000000000006"
+      }
+    },
+    "/body-optional": {
+      "post": {
+        "summary": "/body-optional",
+        "description": "Mizu observed 3 entries (0 failed), at 0.003 hits/s, average response time is 0.001 seconds",
+        "operationId": "34f3d66c-b1f7-4dca-9cab-987fcc8ae472",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000012"
+              }
+            },
+            "x-sample-entry": "000000000000000000000012"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 3,
+            "failures": 0,
+            "firstSeen": 1567750581.7471218,
+            "lastSeen": 1567750581.757122,
+            "sumRT": 0.003,
+            "sumDuration": 0.010000228881835938
+          }
+        },
+        "x-counters-total": {
+          "entries": 3,
+          "failures": 0,
+          "firstSeen": 1567750581.7471218,
+          "lastSeen": 1567750581.757122,
+          "sumRT": 0.003,
+          "sumDuration": 0.010000228881835938
+        },
+        "x-last-seen-ts": 1567750581.757122,
+        "x-sample-entry": "000000000000000000000012",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "application/json": {
+              "example": "{\"key\", \"val\"}",
+              "x-sample-entry": "000000000000000000000011"
+            }
+          },
+          "x-sample-entry": "000000000000000000000012"
+        }
+      }
+    },
+    "/body-required": {
+      "post": {
+        "summary": "/body-required",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "ff6add53-ab1c-4d4e-b590-0835fa318276",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000013"
+              }
+            },
+            "x-sample-entry": "000000000000000000000013"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750581.757122,
+            "lastSeen": 1567750581.757122,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750581.757122,
+          "lastSeen": 1567750581.757122,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750581.757122,
+        "x-sample-entry": "000000000000000000000013",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "": {
+              "example": "body exists",
+              "x-sample-entry": "000000000000000000000013"
+            }
+          },
+          "required": true,
+          "x-sample-entry": "000000000000000000000013"
+        }
+      }
+    },
+    "/form-multipart": {
+      "post": {
+        "summary": "/form-multipart",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "153f0925-9fc7-4e9f-9d33-f1470f25f0f7",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "example": {},
+                "x-sample-entry": "000000000000000000000009"
+              }
+            },
+            "x-sample-entry": "000000000000000000000009"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.7471218,
+            "lastSeen": 1567750582.7471218,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.7471218,
+          "lastSeen": 1567750582.7471218,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.7471218,
+        "x-sample-entry": "000000000000000000000009",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "multipart/form-data": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "file",
+                  "path"
+                ],
+                "properties": {
+                  "file": {
+                    "type": "string",
+                    "contentMediaType": "application/json",
+                    "examples": [
+                      "{\"functions\": 123}"
+                    ]
+                  },
+                  "path": {
+                    "type": "string",
+                    "examples": [
+                      "/content/components"
+                    ]
+                  }
+                }
+              },
+              "example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n",
+              "x-sample-entry": "000000000000000000000009"
+            }
+          },
+          "required": true,
+          "x-sample-entry": "000000000000000000000009"
+        }
+      }
+    },
+    "/form-urlencoded": {
+      "post": {
+        "summary": "/form-urlencoded",
+        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds",
+        "operationId": "c92189f5-5636-46eb-ac71-92b17941a568",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000008"
+              }
+            },
+            "x-sample-entry": "000000000000000000000008"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750580.7471218,
+            "lastSeen": 1567750581.7471218,
+            "sumRT": 0.002,
+            "sumDuration": 1
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750580.7471218,
+          "lastSeen": 1567750581.7471218,
+          "sumRT": 0.002,
+          "sumDuration": 1
+        },
+        "x-last-seen-ts": 1567750581.7471218,
+        "x-sample-entry": "000000000000000000000008",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "application/x-www-form-urlencoded": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "agent-id",
+                  "callback-url",
+                  "token"
+                ],
+                "properties": {
+                  "agent-id": {
+                    "type": "string",
+                    "examples": [
+                      "ade"
+                    ]
+                  },
+                  "callback-url": {
+                    "type": "string",
+                    "examples": [
+                      ""
+                    ]
+                  },
+                  "optional": {
+                    "type": "string",
+                    "examples": [
+                      "another"
+                    ]
+                  },
+                  "token": {
+                    "type": "string",
+                    "examples": [
+                      "sometoken",
+                      "sometoken-second-val"
+                    ]
+                  }
+                }
+              },
+              "example": "agent-id=ade\u0026callback-url=\u0026token=sometoken",
+              "x-sample-entry": "000000000000000000000008"
+            }
+          },
+          "required": true,
+          "x-sample-entry": "000000000000000000000008"
+        }
+      }
+    },
+    "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "85270437-7aae-4a5b-b988-3662092463d0",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000014"
+              }
+            },
+            "x-sample-entry": "000000000000000000000014"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582,
+            "lastSeen": 1567750582,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582,
+          "lastSeen": 1567750582,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582,
+        "x-sample-entry": "000000000000000000000014"
+      },
+      "parameters": [
+        {
+          "name": "prefixgibberishfineId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "234324"
+            }
+          },
+          "x-sample-entry": "000000000000000000000014"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}",
+        "description": "Mizu observed 2 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "da597734-1cf5-4d3b-917b-6b02dacf7b7b",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000018"
+              }
+            },
+            "x-sample-entry": "000000000000000000000018"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750582.000003,
+            "lastSeen": 1567750582.000004,
+            "sumRT": 0.002,
+            "sumDuration": 9.5367431640625e-7
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750582.000003,
+          "lastSeen": 1567750582.000004,
+          "sumRT": 0.002,
+          "sumDuration": 9.5367431640625e-7
+        },
+        "x-last-seen-ts": 1567750582.000004,
+        "x-sample-entry": "000000000000000000000018"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/1": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/1",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "e965a245-9cfc-48ed-94e1-f765eadb3960",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000015"
+              }
+            },
+            "x-sample-entry": "000000000000000000000015"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.000001,
+            "lastSeen": 1567750582.000001,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.000001,
+          "lastSeen": 1567750582.000001,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.000001,
+        "x-sample-entry": "000000000000000000000015"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/static": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/static",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "7af420dc-f8b7-450f-8f6f-18b039aa3cde",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000016"
+              }
+            },
+            "x-sample-entry": "000000000000000000000016"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.000002,
+            "lastSeen": 1567750582.000002,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.000002,
+          "lastSeen": 1567750582.000002,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.000002,
+        "x-sample-entry": "000000000000000000000016"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/{param1}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/{param1}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "02a1771d-2d50-4a8c-8be2-29c7e59b8435",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000019"
+              }
+            },
+            "x-sample-entry": "000000000000000000000019"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.000002,
+            "lastSeen": 1567750582.000002,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.000002,
+          "lastSeen": 1567750582.000002,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.000002,
+        "x-sample-entry": "000000000000000000000019"
+      },
+      "parameters": [
+        {
+          "name": "param1",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "23421"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        },
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/{Id}": {
+      "get": {
+        "summary": "/{Id}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "77ec4910-d47a-46a5-8234-fb80a11034b4",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000003"
+              }
+            },
+            "x-sample-entry": "000000000000000000000003"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750579.7471218,
+            "lastSeen": 1567750579.7471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750579.7471218,
+          "lastSeen": 1567750579.7471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750579.7471218,
+        "x-sample-entry": "000000000000000000000003"
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          },
+          "x-sample-entry": "000000000000000000000003"
+        }
+      ]
+    },
+    "/{Id}/sub1": {
+      "get": {
+        "summary": "/{Id}/sub1",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds",
+        "operationId": "198675eb-9faf-407b-83fa-0483a730bbbe",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "text/html": {
+                "x-sample-entry": "000000000000000000000001"
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750483.864529,
+            "lastSeen": 1567750483.864529,
+            "sumRT": 0.111,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750483.864529,
+          "lastSeen": 1567750483.864529,
+          "sumRT": 0.111,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750483.864529,
+        "x-sample-entry": "000000000000000000000001"
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          },
+          "x-sample-entry": "000000000000000000000003"
+        }
+      ]
+    },
+    "/{Id}/sub2": {
+      "get": {
+        "summary": "/{Id}/sub2",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "31d880f1-152f-4dd6-84a7-463e13b694a5",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000002"
+              }
+            },
+            "x-sample-entry": "000000000000000000000002"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750578.7471218,
+            "lastSeen": 1567750578.7471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750578.7471218,
+          "lastSeen": 1567750578.7471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750578.7471218,
+        "x-sample-entry": "000000000000000000000002"
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          },
+          "x-sample-entry": "000000000000000000000003"
+        }
+      ]
+    }
+  },
+  "x-counters-per-source": {
+    "": {
+      "entries": 19,
+      "failures": 0,
+      "firstSeen": 1567750483.864529,
+      "lastSeen": 1567750582.7471218,
+      "sumRT": 3.273999999999999,
+      "sumDuration": 2.0100011825561523
+    }
+  },
+  "x-counters-total": {
+    "entries": 19,
+    "failures": 0,
+    "firstSeen": 1567750483.864529,
+    "lastSeen": 1567750582.7471218,
+    "sumRT": 3.273999999999999,
+    "sumDuration": 2.0100011825561523
+  }
+}

agent/pkg/oas/oas_generator.go

@@ -1,16 +1,14 @@
 package oas
 
 import (
-	"context"
 	"encoding/json"
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap/api"
 	"net/url"
 	"sync"
 
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/tap/api"
+
+	"github.com/up9inc/mizu/logger"
 )
 
 var (
@@ -18,21 +16,20 @@ var (
 	instance *defaultOasGenerator
 )
 
+type OasGeneratorSink interface {
+	HandleEntry(mizuEntry *api.Entry)
+}
+
 type OasGenerator interface {
-	Start(conn *basenine.Connection)
+	Start()
 	Stop()
 	IsStarted() bool
 	GetServiceSpecs() *sync.Map
-	SetEntriesQuery(query string) bool
 }
 
 type defaultOasGenerator struct {
-	started       bool
-	ctx           context.Context
-	cancel        context.CancelFunc
-	serviceSpecs  *sync.Map
-	dbConn        *basenine.Connection
-	entriesQuery string
+	started      bool
+	serviceSpecs *sync.Map
 }
 
 func GetDefaultOasGeneratorInstance() *defaultOasGenerator {
@@ -43,34 +40,8 @@ func GetDefaultOasGeneratorInstance() *defaultOasGenerator {
 	return instance
 }
 
-func (g *defaultOasGenerator) Start(conn *basenine.Connection) {
-	if g.started {
-		return
-	}
-
-	if g.dbConn == nil {
-		if conn == nil {
-			logger.Log.Infof("Creating new DB connection for OAS generator to address %s:%s", shared.BasenineHost, shared.BaseninePort)
-			newConn, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
-			if err != nil {
-				logger.Log.Error("Error connecting to DB for OAS generator, err: %v", err)
-				return
-			}
-
-			conn = newConn
-		}
-
-		g.dbConn = conn
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	g.cancel = cancel
-	g.ctx = ctx
-	g.serviceSpecs = &sync.Map{}
-
+func (g *defaultOasGenerator) Start() {
 	g.started = true
-
-	go g.runGenerator()
 }
 
 func (g *defaultOasGenerator) Stop() {
@@ -78,74 +49,33 @@ func (g *defaultOasGenerator) Stop() {
 		return
 	}
 
-	if g.dbConn != nil {
-		g.dbConn.Close()
-		g.dbConn = nil
-	}
-
-	g.cancel()
-	g.reset()
-
 	g.started = false
+
+	g.reset()
 }
 
 func (g *defaultOasGenerator) IsStarted() bool {
 	return g.started
 }
 
-func (g *defaultOasGenerator) runGenerator() {
-	// Make []byte channels to receive the data and the meta
-	dataChan := make(chan []byte)
-	metaChan := make(chan []byte)
-
-	logger.Log.Infof("Querying DB for OAS generator with query '%s'", g.entriesQuery)
-	g.dbConn.Query(g.entriesQuery, dataChan, metaChan)
-
-	for {
-		select {
-		case <-g.ctx.Done():
-			logger.Log.Infof("OAS Generator was canceled")
-			close(dataChan)
-			close(metaChan)
-			return
-
-		case metaBytes, ok := <-metaChan:
-			if !ok {
-				logger.Log.Infof("OAS Generator - meta channel closed")
-				break
-			}
-			logger.Log.Debugf("Meta: %s", metaBytes)
-
-		case dataBytes, ok := <-dataChan:
-			if !ok {
-				logger.Log.Infof("OAS Generator - entries channel closed")
-				break
-			}
-
-			logger.Log.Debugf("Data: %s", dataBytes)
-			e := new(api.Entry)
-			err := json.Unmarshal(dataBytes, e)
-			if err != nil {
-				continue
-			}
-			g.handleEntry(e)
-		}
+func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry) {
+	if !g.started {
+		return
 	}
-}
 
-func (g *defaultOasGenerator) handleEntry(mizuEntry *api.Entry) {
 	if mizuEntry.Protocol.Name == "http" {
+		dest := mizuEntry.Destination.Name
+		if dest == "" {
+			logger.Log.Debugf("OAS: Unresolved entry %d", mizuEntry.Id)
+			return
+		}
+
 		entry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
 		if err != nil {
 			logger.Log.Warningf("Failed to turn MizuEntry %d into HAR Entry: %s", mizuEntry.Id, err)
 			return
 		}
 
-		dest := mizuEntry.Destination.Name
-		if dest == "" {
-			dest = mizuEntry.Destination.IP + ":" + mizuEntry.Destination.Port
-		}
-
 		entryWSource := &EntryWithSource{
 			Entry:       *entry,
 			Source:      mizuEntry.Source.Name,
@@ -174,7 +104,7 @@ func (g *defaultOasGenerator) handleHARWithSource(entryWSource *EntryWithSource)
 		return
 	}
 
-	logger.Log.Debugf("Handled entry %d as opId: %s", entryWSource.Id, opId) // TODO: set opId back to entry?
+	logger.Log.Debugf("Handled entry %s as opId: %s", entryWSource.Id, opId) // TODO: set opId back to entry?
 }
 
 func (g *defaultOasGenerator) getGen(dest string, urlStr string) *SpecGen {
@@ -202,18 +132,9 @@ func (g *defaultOasGenerator) GetServiceSpecs() *sync.Map {
 	return g.serviceSpecs
 }
 
-func (g *defaultOasGenerator) SetEntriesQuery(query string) bool {
-	changed := g.entriesQuery != query
-	g.entriesQuery = query
-	return changed
-}
-
 func NewDefaultOasGenerator() *defaultOasGenerator {
 	return &defaultOasGenerator{
 		started:      false,
-		ctx:          nil,
-		cancel:       nil,
-		serviceSpecs: nil,
-		dbConn:       nil,
+		serviceSpecs: &sync.Map{},
 	}
 }

agent/pkg/oas/oas_generator_test.go

@@ -4,10 +4,11 @@ import (
 	"encoding/json"
 	"github.com/up9inc/mizu/agent/pkg/har"
 	"testing"
+	"time"
 )
 
 func TestOASGen(t *testing.T) {
-	gen := new(defaultOasGenerator)
+	gen := GetDefaultOasGeneratorInstance()
 
 	e := new(har.Entry)
 	err := json.Unmarshal([]byte(`{"startedDateTime": "20000101","request": {"url": "https://host/path", "method": "GET"}, "response": {"status": 200}}`), e)
@@ -20,8 +21,7 @@ func TestOASGen(t *testing.T) {
 		Entry:       *e,
 	}
 
-	dummyConn := GetFakeDBConn(`{"startedDateTime": "20000101","request": {"url": "https://host/path", "method": "GET"}, "response": {"status": 200}}`)
-	gen.Start(dummyConn)
+	gen.Start()
 	gen.handleHARWithSource(ews)
 	g, ok := gen.serviceSpecs.Load("some")
 	if !ok {
@@ -38,5 +38,8 @@ func TestOASGen(t *testing.T) {
 	if !gen.IsStarted() {
 		t.Errorf("Should be started")
 	}
+
+	time.Sleep(100 * time.Millisecond)
+
 	gen.Stop()
 }

agent/pkg/oas/specgen.go

@@ -3,10 +3,6 @@ package oas
 import (
 	"encoding/json"
 	"errors"
-	"github.com/chanced/openapi"
-	"github.com/google/uuid"
-	"github.com/nav-inc/datetime"
-	"github.com/up9inc/mizu/shared/logger"
 	"io"
 	"io/ioutil"
 	"mime"
@@ -18,6 +14,11 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/chanced/openapi"
+	"github.com/google/uuid"
+	"github.com/nav-inc/datetime"
+	"github.com/up9inc/mizu/logger"
+
 	"github.com/up9inc/mizu/agent/pkg/har"
 
 	"time"
@@ -32,7 +33,7 @@ type EntryWithSource struct {
 	Source      string
 	Destination string
 	Entry       har.Entry
-	Id          uint
+	Id          string
 }
 
 type reqResp struct { // hello, generics in Go
@@ -67,7 +68,7 @@ func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
 	g.tree = new(Node)
 	for pathStr, pathObj := range oas.Paths.Items {
 		pathSplit := strings.Split(string(pathStr), "/")
-		g.tree.getOrSet(pathSplit, pathObj, 0)
+		g.tree.getOrSet(pathSplit, pathObj, "")
 
 		// clean "last entry timestamp" markers from the past
 		for _, pathAndOp := range g.tree.listOps() {
@@ -341,7 +342,7 @@ func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *Ent
 	return nil
 }
 
-func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, sampleId uint) error {
+func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, sampleId string) error {
 	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
 	urlParsed, err := url.Parse(req.URL)
 	if err != nil {
@@ -401,7 +402,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, s
 	return nil
 }
 
-func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool, sampleId uint) error {
+func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool, sampleId string) error {
 	// TODO: we don't support "default" response
 	respObj, err := getResponseObj(resp, opObj, isSuccess)
 	if err != nil {
@@ -422,7 +423,7 @@ func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool
 	return nil
 }
 
-func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sampleId uint) {
+func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sampleId string) {
 	visited := map[string]*openapi.HeaderObj{}
 	for _, pair := range reqHeaders {
 		if isHeaderIgnored(pair.Name) {
@@ -466,7 +467,7 @@ func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sa
 	}
 }
 
-func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, sampleId uint) (*openapi.MediaType, error) {
+func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, sampleId string) (*openapi.MediaType, error) {
 	content, found := respContent[ctype]
 	if !found {
 		respContent[ctype] = &openapi.MediaType{}

agent/pkg/oas/specgen_test.go

@@ -1,10 +1,8 @@
 package oas
 
 import (
-	"bytes"
 	"encoding/json"
 	"io/ioutil"
-	"net"
 	"os"
 	"regexp"
 	"strings"
@@ -13,22 +11,11 @@ import (
 	"time"
 
 	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/wI2L/jsondiff"
-
-	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/logger"
+	"github.com/wI2L/jsondiff"
 )
 
-func GetFakeDBConn(send string) *basenine.Connection {
-	dummyConn := new(basenine.Connection)
-	dummyConn.Conn = FakeConn{
-		sendBuffer:    bytes.NewBufferString(send),
-		receiveBuffer: bytes.NewBufferString(""),
-	}
-	return dummyConn
-}
-
 // if started via env, write file into subdir
 func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string {
 	content, err := json.MarshalIndent(spec, "", "  ")
@@ -278,17 +265,3 @@ func TestLoadValid3_1(t *testing.T) {
 		t.FailNow()
 	}
 }
-
-type FakeConn struct {
-	sendBuffer    *bytes.Buffer
-	receiveBuffer *bytes.Buffer
-}
-
-func (f FakeConn) Read(p []byte) (int, error)       { return f.sendBuffer.Read(p) }
-func (f FakeConn) Write(p []byte) (int, error)      { return f.receiveBuffer.Write(p) }
-func (FakeConn) Close() error                       { return nil }
-func (FakeConn) LocalAddr() net.Addr                { return nil }
-func (FakeConn) RemoteAddr() net.Addr               { return nil }
-func (FakeConn) SetDeadline(t time.Time) error      { return nil }
-func (FakeConn) SetReadDeadline(t time.Time) error  { return nil }
-func (FakeConn) SetWriteDeadline(t time.Time) error { return nil }

agent/pkg/oas/test_artifacts/params.har.spec.json

@@ -22,10 +22,10 @@
             "content": {
               "application/json": {
                 "example": null,
-                "x-sample-entry": 4
+                "x-sample-entry": "000000000000000000000004"
               }
             },
-            "x-sample-entry": 4
+            "x-sample-entry": "000000000000000000000004"
           }
         },
         "x-counters-per-source": {
@@ -47,7 +47,7 @@
           "sumDuration": 0
         },
         "x-last-seen-ts": 1567750580.04,
-        "x-sample-entry": 4
+        "x-sample-entry": "000000000000000000000004"
       }
     },
     "/appears-twice": {
@@ -61,10 +61,10 @@
             "content": {
               "application/json": {
                 "example": null,
-                "x-sample-entry": 6
+                "x-sample-entry": "000000000000000000000006"
               }
             },
-            "x-sample-entry": 6
+            "x-sample-entry": "000000000000000000000006"
           }
         },
         "x-counters-per-source": {
@@ -86,7 +86,7 @@
           "sumDuration": 1
         },
         "x-last-seen-ts": 1567750581.74,
-        "x-sample-entry": 6
+        "x-sample-entry": "000000000000000000000006"
       }
     },
     "/body-optional": {
@@ -99,10 +99,10 @@
             "description": "Successful call with status 200",
             "content": {
               "": {
-                "x-sample-entry": 12
+                "x-sample-entry": "000000000000000000000012"
               }
             },
-            "x-sample-entry": 12
+            "x-sample-entry": "000000000000000000000012"
           }
         },
         "x-counters-per-source": {
@@ -124,16 +124,16 @@
           "sumDuration": 0.01
         },
         "x-last-seen-ts": 1567750581.75,
-        "x-sample-entry": 12,
+        "x-sample-entry": "000000000000000000000012",
         "requestBody": {
           "description": "Generic request body",
           "content": {
             "application/json": {
               "example": "{\"key\", \"val\"}",
-              "x-sample-entry": 11
+              "x-sample-entry": "000000000000000000000011"
             }
           },
-          "x-sample-entry": 12
+          "x-sample-entry": "000000000000000000000012"
         }
       }
     },
@@ -147,10 +147,10 @@
             "description": "Successful call with status 200",
             "content": {
               "": {
-                "x-sample-entry": 13
+                "x-sample-entry": "000000000000000000000013"
               }
             },
-            "x-sample-entry": 13
+            "x-sample-entry": "000000000000000000000013"
           }
         },
         "x-counters-per-source": {
@@ -172,17 +172,17 @@
           "sumDuration": 0
         },
         "x-last-seen-ts": 1567750581.75,
-        "x-sample-entry": 13,
+        "x-sample-entry": "000000000000000000000013",
         "requestBody": {
           "description": "Generic request body",
           "content": {
             "": {
               "example": "body exists",
-              "x-sample-entry": 13
+              "x-sample-entry": "000000000000000000000013"
             }
           },
           "required": true,
-          "x-sample-entry": 13
+          "x-sample-entry": "000000000000000000000013"
         }
       }
     },
@@ -197,10 +197,10 @@
             "content": {
               "": {
                 "example": {},
-                "x-sample-entry": 9
+                "x-sample-entry": "000000000000000000000009"
               }
             },
-            "x-sample-entry": 9
+            "x-sample-entry": "000000000000000000000009"
           }
         },
         "x-counters-per-source": {
@@ -222,7 +222,7 @@
           "sumDuration": 0
         },
         "x-last-seen-ts": 1567750582.74,
-        "x-sample-entry": 9,
+        "x-sample-entry": "000000000000000000000009",
         "requestBody": {
           "description": "Generic request body",
           "content": {
@@ -250,11 +250,11 @@
                 }
               },
               "example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n",
-              "x-sample-entry": 9
+              "x-sample-entry": "000000000000000000000009"
             }
           },
           "required": true,
-          "x-sample-entry": 9
+          "x-sample-entry": "000000000000000000000009"
         }
       }
     },
@@ -268,10 +268,10 @@
             "description": "Successful call with status 200",
             "content": {
               "": {
-                "x-sample-entry": 8
+                "x-sample-entry": "000000000000000000000008"
               }
             },
-            "x-sample-entry": 8
+            "x-sample-entry": "000000000000000000000008"
           }
         },
         "x-counters-per-source": {
@@ -293,7 +293,7 @@
           "sumDuration": 1
         },
         "x-last-seen-ts": 1567750581.74,
-        "x-sample-entry": 8,
+        "x-sample-entry": "000000000000000000000008",
         "requestBody": {
           "description": "Generic request body",
           "content": {
@@ -334,11 +334,11 @@
                 }
               },
               "example": "agent-id=ade\u0026callback-url=\u0026token=sometoken",
-              "x-sample-entry": 8
+              "x-sample-entry": "000000000000000000000008"
             }
           },
           "required": true,
-          "x-sample-entry": 8
+          "x-sample-entry": "000000000000000000000008"
         }
       }
     },
@@ -355,10 +355,10 @@
             "description": "Successful call with status 200",
             "content": {
               "": {
-                "x-sample-entry": 14
+                "x-sample-entry": "000000000000000000000014"
               }
             },
-            "x-sample-entry": 14
+            "x-sample-entry": "000000000000000000000014"
           }
         },
         "x-counters-per-source": {
@@ -380,7 +380,7 @@
           "sumDuration": 0
         },
         "x-last-seen-ts": 1567750582,
-        "x-sample-entry": 14
+        "x-sample-entry": "000000000000000000000014"
       },
       "parameters": [
         {
@@ -396,7 +396,7 @@
               "value": "234324"
             }
           },
-          "x-sample-entry": 14
+          "x-sample-entry": "000000000000000000000014"
         }
       ]
     },
@@ -413,10 +413,10 @@
             "description": "Successful call with status 200",
             "content": {
               "": {
-                "x-sample-entry": 18
+                "x-sample-entry": "000000000000000000000018"
               }
             },
-            "x-sample-entry": 18
+            "x-sample-entry": "000000000000000000000018"
           }
         },
         "x-counters-per-source": {
@@ -438,7 +438,7 @@
           "sumDuration": 9.53e-7
         },
         "x-last-seen-ts": 1567750582.00,
-        "x-sample-entry": 18
+        "x-sample-entry": "000000000000000000000018"
       },
       "parameters": [
         {
@@ -467,7 +467,7 @@
               "value": "prefix-gibberish-afterwards"
             }
           },
-          "x-sample-entry": 19
+          "x-sample-entry": "000000000000000000000019"
         }
       ]
     },
@@ -484,10 +484,10 @@
             "description": "Successful call with status 200",
             "content": {
               "": {
-                "x-sample-entry": 15
+                "x-sample-entry": "000000000000000000000015"
               }
             },
-            "x-sample-entry": 15
+            "x-sample-entry": "000000000000000000000015"
           }
         },
         "x-counters-per-source": {
@@ -509,7 +509,7 @@
           "sumDuration": 0
         },
         "x-last-seen-ts": 1567750582.00,
-        "x-sample-entry": 15
+        "x-sample-entry": "000000000000000000000015"
       },
       "parameters": [
         {
@@ -538,7 +538,7 @@
               "value": "prefix-gibberish-afterwards"
             }
           },
-          "x-sample-entry": 19
+          "x-sample-entry": "000000000000000000000019"
         }
       ]
     },
@@ -555,10 +555,10 @@
             "description": "Successful call with status 200",
             "content": {
               "": {
-                "x-sample-entry": 16
+                "x-sample-entry": "000000000000000000000016"
               }
             },
-            "x-sample-entry": 16
+            "x-sample-entry": "000000000000000000000016"
           }
         },
         "x-counters-per-source": {
@@ -580,7 +580,7 @@
           "sumDuration": 0
         },
         "x-last-seen-ts": 1567750582.00,
-        "x-sample-entry": 16
+        "x-sample-entry": "000000000000000000000016"
       },
       "parameters": [
         {
@@ -609,7 +609,7 @@
               "value": "prefix-gibberish-afterwards"
             }
           },
-          "x-sample-entry": 19
+          "x-sample-entry": "000000000000000000000019"
         }
       ]
     },
@@ -626,10 +626,10 @@
             "description": "Successful call with status 200",
             "content": {
               "": {
-                "x-sample-entry": 19
+                "x-sample-entry": "000000000000000000000019"
               }
             },
-            "x-sample-entry": 19
+            "x-sample-entry": "000000000000000000000019"
           }
         },
         "x-counters-per-source": {
@@ -651,7 +651,7 @@
           "sumDuration": 0
         },
         "x-last-seen-ts": 1567750582.00,
-        "x-sample-entry": 19
+        "x-sample-entry": "000000000000000000000019"
       },
       "parameters": [
         {
@@ -667,7 +667,7 @@
               "value": "23421"
             }
           },
-          "x-sample-entry": 19
+          "x-sample-entry": "000000000000000000000019"
         },
         {
           "name": "parampatternId",
@@ -695,7 +695,7 @@
               "value": "prefix-gibberish-afterwards"
             }
           },
-          "x-sample-entry": 19
+          "x-sample-entry": "000000000000000000000019"
         }
       ]
     },
@@ -710,10 +710,10 @@
             "content": {
               "application/json": {
                 "example": null,
-                "x-sample-entry": 3
+                "x-sample-entry": "000000000000000000000003"
               }
             },
-            "x-sample-entry": 3
+            "x-sample-entry": "000000000000000000000003"
           }
         },
         "x-counters-per-source": {
@@ -735,7 +735,7 @@
           "sumDuration": 0
         },
         "x-last-seen-ts": 1567750579.74,
-        "x-sample-entry": 3
+        "x-sample-entry": "000000000000000000000003"
       },
       "parameters": [
         {
@@ -754,7 +754,7 @@
               "value": "<UUID4>"
             }
           },
-          "x-sample-entry": 3
+          "x-sample-entry": "000000000000000000000003"
         }
       ]
     },
@@ -768,10 +768,10 @@
             "description": "Successful call with status 200",
             "content": {
               "text/html": {
-                "x-sample-entry": 1
+                "x-sample-entry": "000000000000000000000001"
               }
             },
-            "x-sample-entry": 1
+            "x-sample-entry": "000000000000000000000001"
           }
         },
         "x-counters-per-source": {
@@ -793,7 +793,7 @@
           "sumDuration": 0
         },
         "x-last-seen-ts": 1567750483.86,
-        "x-sample-entry": 1
+        "x-sample-entry": "000000000000000000000001"
       },
       "parameters": [
         {
@@ -812,7 +812,7 @@
               "value": "<UUID4>"
             }
           },
-          "x-sample-entry": 3
+          "x-sample-entry": "000000000000000000000003"
         }
       ]
     },
@@ -827,10 +827,10 @@
             "content": {
               "application/json": {
                 "example": null,
-                "x-sample-entry": 2
+                "x-sample-entry": "000000000000000000000002"
               }
             },
-            "x-sample-entry": 2
+            "x-sample-entry": "000000000000000000000002"
           }
         },
         "x-counters-per-source": {
@@ -852,7 +852,7 @@
           "sumDuration": 0
         },
         "x-last-seen-ts": 1567750578.74,
-        "x-sample-entry": 2
+        "x-sample-entry": "000000000000000000000002"
       },
       "parameters": [
         {
@@ -871,7 +871,7 @@
               "value": "<UUID4>"
             }
           },
-          "x-sample-entry": 3
+          "x-sample-entry": "000000000000000000000003"
         }
       ]
     }

agent/pkg/oas/tree.go

@@ -2,12 +2,13 @@ package oas
 
 import (
 	"encoding/json"
-	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/shared/logger"
 	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
+
+	"github.com/chanced/openapi"
+	"github.com/up9inc/mizu/logger"
 )
 
 type NodePath = []string
@@ -20,7 +21,7 @@ type Node struct {
 	children  []*Node
 }
 
-func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj, sampleId uint) (node *Node) {
+func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj, sampleId string) (node *Node) {
 	if existingPathObj == nil {
 		panic("Invalid function call")
 	}

agent/pkg/oas/tree_test.go

@@ -1,6 +1,7 @@
 package oas
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 
@@ -23,7 +24,7 @@ func TestTree(t *testing.T) {
 	for i, tc := range testCases {
 		split := strings.Split(tc.inp, "/")
 		pathObj := new(openapi.PathObj)
-		node := tree.getOrSet(split, pathObj, uint(i))
+		node := tree.getOrSet(split, pathObj, fmt.Sprintf("%024d", i))
 
 		fillPathParams(node, pathObj)
 

agent/pkg/oas/utils.go

@@ -9,7 +9,7 @@ import (
 	"github.com/up9inc/mizu/agent/pkg/har"
 
 	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 func exampleResolver(ref string) (*openapi.ExampleObj, error) {
@@ -115,7 +115,7 @@ type nvParams struct {
 	GeneralizeName func(name string) string
 }
 
-func handleNameVals(gw nvParams, params **openapi.ParameterList, checkIgnore bool, sampleId uint) {
+func handleNameVals(gw nvParams, params **openapi.ParameterList, checkIgnore bool, sampleId string) {
 	visited := map[string]*openapi.ParameterObj{}
 	for _, pair := range gw.Pairs {
 		if (checkIgnore && gw.IsIgnored(pair.Name)) || pair.Name == "" {
@@ -477,8 +477,8 @@ func intersectSliceWithMap(required []string, names map[string]struct{}) []strin
 	return required
 }
 
-func setSampleID(extensions *openapi.Extensions, id uint) {
-	if id > 0 {
+func setSampleID(extensions *openapi.Extensions, id string) {
+	if id != "" {
 		if *extensions == nil {
 			*extensions = openapi.Extensions{}
 		}

agent/pkg/oas/utils_test.go

@@ -29,7 +29,7 @@ func TestAnyJSON(t *testing.T) {
 		} else if tc.inp == "null" && any != nil {
 			t.Errorf("null has to parse as nil (but got %s)", any)
 		} else {
-			t.Logf("%s => %s", tc.inp, any)
+			t.Logf("%s => %v", tc.inp, any)
 		}
 	}
 }

agent/pkg/providers/status_provider.go

@@ -4,19 +4,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"time"
 
-	"github.com/patrickmn/go-cache"
 	"github.com/up9inc/mizu/agent/pkg/models"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap"
 )
 
-const tlsLinkRetainmentTime = time.Minute * 15
-
 var (
-	authStatus     *models.AuthStatus
-	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
+	authStatus *models.AuthStatus
 )
 
 func GetAuthStatus() (*models.AuthStatus, error) {
@@ -51,16 +45,3 @@ func GetAuthStatus() (*models.AuthStatus, error) {
 
 	return authStatus, nil
 }
-
-func GetAllRecentTLSAddresses() []string {
-	recentTLSLinks := make([]string, 0)
-
-	for _, outboundLinkItem := range RecentTLSLinks.Items() {
-		outboundLink, castOk := outboundLinkItem.Object.(*tap.OutboundLink)
-		if castOk {
-			recentTLSLinks = append(recentTLSLinks, outboundLink.DstIP)
-		}
-	}
-
-	return recentTLSLinks
-}

agent/pkg/providers/tappedPods/tapped_pods_provider.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
 	"github.com/up9inc/mizu/agent/pkg/utils"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 const FilePath = shared.DataDirPath + "tapped-pods.json"

agent/pkg/providers/tappers/tappers_provider.go

@@ -5,8 +5,8 @@ import (
 	"sync"
 
 	"github.com/up9inc/mizu/agent/pkg/utils"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 const FilePath = shared.DataDirPath + "tappers-status.json"

agent/pkg/resolver/go.sum

@@ -219,8 +219,6 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
-github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

agent/pkg/resolver/resolver.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 
 	cmap "github.com/orcaman/concurrent-map"

agent/pkg/routes/status_routes.go

@@ -21,7 +21,5 @@ func StatusRoutes(ginApp *gin.Engine) {
 
 	routeGroup.GET("/general", controllers.GetGeneralStats) // get general stats about entries in DB
 
-	routeGroup.GET("/recentTLSLinks", controllers.GetRecentTLSLinks)
-
 	routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
 }

agent/pkg/rules/rulesHTTP.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/up9inc/mizu/agent/pkg/har"
 
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 
 	"github.com/up9inc/mizu/shared"
 	"github.com/yalp/jsonpath"

agent/pkg/servicemap/servicemap.go

@@ -1,10 +1,11 @@
 package servicemap
 
 import (
-	"github.com/jinzhu/copier"
 	"sync"
 
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/jinzhu/copier"
+
+	"github.com/up9inc/mizu/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 

agent/pkg/up9/main.go

@@ -17,8 +17,8 @@ import (
 	"github.com/up9inc/mizu/agent/pkg/utils"
 
 	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
@@ -211,11 +211,15 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 
 	logger.Log.Infof("Getting entries from the database")
 
+BasenineReconnect:
 	var connection *basenine.Connection
 	var err error
 	connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
 	if err != nil {
-		panic(err)
+		logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
+		connection.Close()
+		time.Sleep(shared.BasenineReconnectInterval * time.Second)
+		goto BasenineReconnect
 	}
 
 	data := make(chan []byte)
@@ -323,7 +327,12 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
 	go handleMetaChannel(&wg, connection, meta)
 	wg.Add(2)
 
-	connection.Query(query, data, meta)
+	if err = connection.Query("latest", query, data, meta); err != nil {
+		logger.Log.Errorf("Query mode call failed: %v", err)
+		connection.Close()
+		time.Sleep(shared.BasenineReconnectInterval * time.Second)
+		goto BasenineReconnect
+	}
 
 	wg.Wait()
 }

agent/pkg/utils/utils.go

@@ -13,8 +13,8 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 var (

cli/Makefile

@@ -36,16 +36,16 @@ build-base: ## Build mizu CLI binary (select platform via GOOS / GOARCH env vari
 	(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
 
 build-all: ## Build for all supported platforms.
-	@echo "Compiling for every OS and Platform"
-	@mkdir -p bin && sed s/_VER_/$(VER)/g README.md.TEMPLATE >  bin/README.md
-	@$(MAKE) build GOOS=linux GOARCH=amd64
-	@$(MAKE) build GOOS=linux GOARCH=arm64
-	@$(MAKE) build GOOS=darwin GOARCH=amd64
-	@$(MAKE) build GOOS=darwin GOARCH=arm64
-	@$(MAKE) build GOOS=windows GOARCH=amd64
-	@mv ./bin/mizu_windows_amd64 ./bin/mizu.exe
-	@echo "---------"
-	@find ./bin -ls
+	echo "Compiling for every OS and Platform" && \
+	mkdir -p bin && sed s/_VER_/$(VER)/g README.md.TEMPLATE >  bin/README.md && \
+	$(MAKE) build GOOS=linux GOARCH=amd64 && \
+	$(MAKE) build GOOS=linux GOARCH=arm64 && \
+	$(MAKE) build GOOS=darwin GOARCH=amd64 && \
+	$(MAKE) build GOOS=darwin GOARCH=arm64 && \
+	$(MAKE) build GOOS=windows GOARCH=amd64 && \
+	mv ./bin/mizu_windows_amd64 ./bin/mizu.exe && \
+	echo "---------" && \
+	find ./bin -ls
 
 clean: ## Clean all build artifacts.
 	go clean

cli/apiserver/provider.go

@@ -4,15 +4,16 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/up9inc/mizu/cli/utils"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"time"
 
+	"github.com/up9inc/mizu/cli/utils"
+
 	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 	core "k8s.io/api/core/v1"
 )
 

cli/auth/authProvider.go

@@ -12,7 +12,7 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 	"golang.org/x/oauth2"
 )
 

cli/cmd/check.go

@@ -5,7 +5,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/telemetry"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 var checkCmd = &cobra.Command{

cli/cmd/check/imagePullInCluster.go

@@ -3,13 +3,14 @@ package check
 import (
 	"context"
 	"fmt"
-	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
-	core "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"regexp"
 	"time"
+
+	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/kubernetes"
+	core "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func ImagePullInCluster(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {

cli/cmd/check/kubernetesApi.go

@@ -2,14 +2,14 @@ package check
 
 import (
 	"fmt"
+
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/semver"
 )
 
-
 func KubernetesApi() (*kubernetes.Provider, *semver.SemVersion, bool) {
 	logger.Log.Infof("\nkubernetes-api\n--------------------")
 

cli/cmd/check/kubernetesPermissions.go

@@ -4,15 +4,16 @@ import (
 	"context"
 	"embed"
 	"fmt"
+	"strings"
+
 	"github.com/up9inc/mizu/cli/bucket"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
 	rbac "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/scheme"
-	"strings"
 )
 
 func TapKubernetesPermissions(ctx context.Context, embedFS embed.FS, kubernetesProvider *kubernetes.Provider) bool {

cli/cmd/check/kubernetesResources.go

@@ -3,10 +3,11 @@ package check
 import (
 	"context"
 	"fmt"
+
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 func KubernetesResources(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {

cli/cmd/check/kubernetesVersion.go

@@ -2,9 +2,10 @@ package check
 
 import (
 	"fmt"
+
 	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/semver"
 )
 

cli/cmd/check/serverConnection.go

@@ -3,12 +3,13 @@ package check
 import (
 	"context"
 	"fmt"
+	"regexp"
+
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
-	"regexp"
 )
 
 func ServerConnection(kubernetesProvider *kubernetes.Provider) bool {

cli/cmd/checkRunner.go

@@ -4,10 +4,11 @@ import (
 	"context"
 	"embed"
 	"fmt"
+
 	"github.com/up9inc/mizu/cli/cmd/check"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 var (

cli/cmd/common.go

@@ -19,8 +19,8 @@ import (
 	"github.com/up9inc/mizu/shared"
 
 	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 func GetApiServerUrl(port uint16) string {

cli/cmd/config.go

@@ -9,7 +9,7 @@ import (
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 var configCmd = &cobra.Command{

cli/cmd/goUtils/funcWrappers.go

@@ -4,7 +4,7 @@ import (
 	"reflect"
 	"runtime/debug"
 
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 func HandleExcWrapper(fn interface{}, params ...interface{}) (result []reflect.Value) {

cli/cmd/installRunner.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/up9inc/mizu/cli/bucket"
 	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 func runMizuInstall() {

cli/cmd/logs.go

@@ -10,7 +10,7 @@ import (
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/telemetry"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 var logsCmd = &cobra.Command{

cli/cmd/root.go

@@ -11,7 +11,7 @@ import (
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/mizu/version"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 var rootCmd = &cobra.Command{

cli/cmd/tap.go

@@ -14,8 +14,8 @@ import (
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 const uploadTrafficMessageToConfirm = `NOTE: running mizu with --%s flag will upload recorded traffic for further analysis and enriched presentation options.`
@@ -123,4 +123,5 @@ func init() {
 	tapCmd.Flags().String(configStructs.ContractFile, defaultTapConfig.ContractFile, "OAS/Swagger file to validate to monitor the contracts")
 	tapCmd.Flags().Bool(configStructs.ServiceMeshName, defaultTapConfig.ServiceMesh, "Record decrypted traffic if the cluster is configured with a service mesh and with mtls")
 	tapCmd.Flags().Bool(configStructs.TlsName, defaultTapConfig.Tls, "Record tls traffic")
+	tapCmd.Flags().Bool(configStructs.ProfilerName, defaultTapConfig.Profiler, "Run pprof server")
 }

cli/cmd/tapRunner.go

@@ -25,9 +25,9 @@ import (
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap/api"
 )
 
@@ -124,7 +124,7 @@ func RunMizuTap() {
 	}
 
 	logger.Log.Infof("Waiting for Mizu Agent to start...")
-	if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel()); err != nil {
+	if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel(), config.Config.Tap.Profiler); err != nil {
 		var statusError *k8serrors.StatusError
 		if errors.As(err, &statusError) && (statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists) {
 			logger.Log.Info("Mizu is already running in this namespace, change the `mizu-resources-namespace` configuration or run `mizu clean` to remove the currently running Mizu instance")
@@ -164,7 +164,6 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 		ServiceMap:             config.Config.ServiceMap,
 		OAS:                    config.Config.OAS,
 		Telemetry:              config.Config.Telemetry,
-		Elastic:                config.Config.Elastic,
 	}
 
 	return &mizuAgentConfig

cli/cmd/version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/telemetry"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"

cli/cmd/view.go

@@ -6,7 +6,7 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/telemetry"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 var viewCmd = &cobra.Command{

cli/cmd/viewRunner.go

@@ -11,8 +11,8 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 func runMizuView() {

cli/config/config.go

@@ -9,8 +9,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"

cli/config/configStruct.go

@@ -41,7 +41,6 @@ type ConfigStruct struct {
 	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
 	ServiceMap             bool                        `yaml:"service-map" default:"true"`
 	OAS                    bool                        `yaml:"oas" default:"true"`
-	Elastic                shared.ElasticConfig        `yaml:"elastic"`
 }
 
 func (config *ConfigStruct) validate() error {

cli/config/configStructs/tapConfig.go

@@ -11,8 +11,7 @@ import (
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 
-	basenine "github.com/up9inc/basenine/server/lib"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/units"
 )
 
@@ -31,6 +30,7 @@ const (
 	ContractFile                  = "contract"
 	ServiceMeshName               = "service-mesh"
 	TlsName                       = "tls"
+	ProfilerName                  = "profiler"
 )
 
 type TapConfig struct {
@@ -55,6 +55,7 @@ type TapConfig struct {
 	TapperResources        shared.Resources `yaml:"tapper-resources"`
 	ServiceMesh            bool             `yaml:"service-mesh" default:"false"`
 	Tls                    bool             `yaml:"tls" default:"false"`
+	Profiler               bool             `yaml:"profiler" default:"false"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {
@@ -79,10 +80,6 @@ func (config *TapConfig) GetInsertionFilter() string {
 			}
 		}
 	}
-	_, err := basenine.Parse(insertionFilter)
-	if err != nil {
-		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Insertion filter syntax error: %v", err))
-	}
 	return insertionFilter
 }
 

cli/go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.3.0
 	github.com/spf13/pflag v1.0.5
-	github.com/up9inc/basenine/server/lib v0.0.0-20220326121918-785f3061c8ce
+	github.com/up9inc/mizu/logger v0.0.0
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
@@ -33,11 +33,8 @@ require (
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
-	github.com/alecthomas/participle/v2 v2.0.0-alpha7 // indirect
 	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
-	github.com/clbanning/mxj/v2 v2.5.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dlclark/regexp2 v1.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
@@ -72,12 +69,12 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
-	github.com/ohler55/ojg v1.12.13 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday v1.6.0 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
+	github.com/up9inc/mizu/tap/dbgctl v0.0.0 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect
 	go.starlark.net v0.0.0-20220203230714-bb14e151c28f // indirect
 	golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab // indirect
@@ -103,6 +100,10 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
+replace github.com/up9inc/mizu/logger v0.0.0 => ../logger
+
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
+
+replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../tap/dbgctl

cli/go.sum

@@ -83,10 +83,6 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/alecthomas/participle/v2 v2.0.0-alpha7 h1:cK4vjj0VSgb3lN1nuKA5F7dw+1s1pWBe5bx7nNCnN+c=
-github.com/alecthomas/participle/v2 v2.0.0-alpha7/go.mod h1:NumScqsC42o9x+dGj8/YqsIfhrIQjFEOFovxotbBirA=
-github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E=
-github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -120,8 +116,6 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
-github.com/clbanning/mxj/v2 v2.5.5 h1:oT81vUeEiQQ/DcHbzSytRngP6Ky9O+L+0Bw0zSJag9E=
-github.com/clbanning/mxj/v2 v2.5.5/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -155,8 +149,6 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS
 github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
-github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -487,8 +479,6 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/ohler55/ojg v1.12.13 h1:FvfVpYzLgMraLcg3rrXiRXaihOP6fnzQNEU9YyZ/AmM=
-github.com/ohler55/ojg v1.12.13/go.mod h1:LBbIVRAgoFbYBXQhRhuEpaJIqq+goSO63/FQ+nyJU88=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -600,8 +590,6 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/up9inc/basenine/server/lib v0.0.0-20220326121918-785f3061c8ce h1:PypqybjmuxftGkX4NmP4JAUyEykZj2r6W4r9lnRZ/kE=
-github.com/up9inc/basenine/server/lib v0.0.0-20220326121918-785f3061c8ce/go.mod h1:ZIkxWiJm65jYQIso9k+OZKhR7gQ1we2jNyE2kQX9IQI=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=

cli/mizu/fsUtils/mizuLogsUtils.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 func GetLogFilePath() string {

cli/mizu/fsUtils/zipUtils.go

@@ -8,7 +8,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 func AddFileToZip(zipWriter *zip.Writer, filename string) error {

cli/mizu/version/versionCheck.go

@@ -13,7 +13,7 @@ import (
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/cli/pkg/version"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 
 	"github.com/google/go-github/v37/github"
 	"github.com/up9inc/mizu/cli/uiUtils"

cli/resources/cleanResources.go

@@ -3,12 +3,13 @@ package resources
 import (
 	"context"
 	"fmt"
+
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/cli/utils"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
@@ -54,7 +55,6 @@ func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, ku
 		}
 	}
 
-
 	if resources, err := kubernetesProvider.ListManagedClusterRoleBindings(ctx); err != nil {
 		resourceDesc := "ClusterRoleBindings"
 		handleDeletionError(err, resourceDesc, &leftoverResources)

cli/resources/createResources.go

@@ -8,13 +8,13 @@ import (
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/cli/uiUtils"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
 	core "k8s.io/api/core/v1"
 )
 
-func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level) (bool, error) {
+func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, profiler bool) (bool, error) {
 	if !isNsRestrictedMode {
 		if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil {
 			return false, err
@@ -50,6 +50,7 @@ func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.
 		Resources:             apiServerResources,
 		ImagePullPolicy:       imagePullPolicy,
 		LogLevel:              logLevel,
+		Profiler:              profiler,
 	}
 
 	if err := createMizuApiServerPod(ctx, kubernetesProvider, opts); err != nil {

cli/telemetry/telemetry.go

@@ -4,14 +4,15 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"os"
+	"time"
+
 	"github.com/denisbrodbeck/machineid"
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/shared/logger"
-	"net/http"
-	"os"
-	"time"
+	"github.com/up9inc/mizu/logger"
 )
 
 const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"

cli/uiUtils/confirmation.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 func AskForConfirmation(s string) bool {

cli/uiUtils/openBrowser.go

@@ -5,7 +5,7 @@ import (
 	"os/exec"
 	"runtime"
 
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 func OpenBrowser(url string) {

cli/utils/waitUtils.go

@@ -2,10 +2,11 @@ package utils
 
 import (
 	"context"
-	"github.com/up9inc/mizu/shared/logger"
 	"os"
 	"os/signal"
 	"syscall"
+
+	"github.com/up9inc/mizu/logger"
 )
 
 func WaitForFinish(ctx context.Context, cancel context.CancelFunc) {

devops/linux-x86_64-musl-go-libpcap/Dockerfile

@@ -0,0 +1,31 @@
+FROM messense/rust-musl-cross:x86_64-musl AS builder-from-arm64v8-to-amd64
+
+ENV CROSS_TRIPLE x86_64-unknown-linux-musl
+ENV CROSS_ROOT /usr/local/musl
+
+ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
+    AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
+    CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
+    CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
+    CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
+    LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld \
+    FC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gfortran
+
+# Install Go
+WORKDIR /
+RUN curl https://go.dev/dl/go1.17.6.linux-arm64.tar.gz -Lo ./go.linux-arm64.tar.gz \
+ && curl https://go.dev/dl/go1.17.6.linux-arm64.tar.gz.asc -Lo ./go.linux-arm64.tar.gz.asc \
+ && curl https://dl.google.com/dl/linux/linux_signing_key.pub  -Lo linux_signing_key.pub \
+ && gpg --import linux_signing_key.pub && gpg --verify ./go.linux-arm64.tar.gz.asc ./go.linux-arm64.tar.gz \
+ && rm -rf /usr/local/go && tar -C /usr/local -xzf go.linux-arm64.tar.gz
+ENV PATH "$PATH:/usr/local/go/bin"
+
+# Compile libpcap from source
+RUN curl https://www.tcpdump.org/release/libpcap-1.10.1.tar.gz -Lo ./libpcap.tar.gz \
+ && curl https://www.tcpdump.org/release/libpcap-1.10.1.tar.gz.sig -Lo ./libpcap.tar.gz.sig \
+ && curl https://www.tcpdump.org/release/signing-key.asc -Lo ./signing-key.asc \
+ && gpg --import signing-key.asc && gpg --verify libpcap.tar.gz.sig libpcap.tar.gz \
+ && tar -xzf libpcap.tar.gz && mv ./libpcap-* ./libpcap
+WORKDIR /libpcap
+RUN ./configure --host=x86_64 && make \
+ && cp /libpcap/libpcap.a /usr/local/musl/lib/gcc/x86_64-unknown-linux-musl/*/

devops/linux-x86_64-musl-go-libpcap/build-push.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+docker build . -t up9inc/linux-x86_64-musl-go-libpcap && docker push up9inc/linux-x86_64-musl-go-libpcap

devops/ui-common-pack.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# exit when any command fails
+set -e
+
+dst_folder=$1
+echo "dst folder: $dst_folder";
+
+cd $dst_folder/../ui-common
+npm i
+npm pack
+mv up9-mizu-common-0.0.0.tgz $dst_folder