Develop -> main (Patch release 31.1)

Merge pull request #1041 from up9inc/develop #patch

.github/workflows/acceptance_tests.yml

@@ -8,9 +8,6 @@ on:
     branches:
       - 'develop'
 
-env:
-  MIZU_CI_IMAGE: mizu/ci:0.0
-
 jobs:
   run-acceptance-tests:
     name: Run acceptance tests
@@ -24,19 +21,6 @@ jobs:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Build
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: false
-          load: true
-          tags: ${{ env.MIZU_CI_IMAGE }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
       - name: Setup acceptance test
         run: ./acceptanceTests/setup.sh
 

.github/workflows/acceptance_tests_on_pr.yml

@@ -2,9 +2,6 @@ name: Acceptance tests on PR
 
 on: push
 
-env:
-  MIZU_CI_IMAGE: mizu/ci:0.0
-
 concurrency:
   group: acceptance-tests-on-pr-${{ github.ref }}
   cancel-in-progress: true
@@ -24,19 +21,6 @@ jobs:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Build
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: false
-          load: true
-          tags: ${{ env.MIZU_CI_IMAGE }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
       - name: Setup acceptance test
         run: ./acceptanceTests/setup.sh
 

.github/workflows/static_code_analysis.yml

@@ -10,7 +10,7 @@ permissions:
   contents: read
 
 jobs:
-  go-lint:
+  golangci:
     name: Go lint
     runs-on: ubuntu-latest
     steps:
@@ -141,61 +141,3 @@ jobs:
         with:
           version: latest
           working-directory: tap/extensions/redis
-
-      - name: Check logger modified files
-        id: logger_modified_files
-        run: devops/check_modified_files.sh logger/
-
-      - name: Go lint - logger
-        uses: golangci/golangci-lint-action@v2
-        if: steps.logger_modified_files.outputs.matched == 'true'
-        with:
-          version: latest
-          working-directory: logger
-
-  es-lint:
-    name: ES lint
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 2
-
-      - uses: actions/setup-node@v2
-        with:
-          node-version: 16
-        
-      - name: Check modified UI files
-        id: ui_modified_files
-        run: devops/check_modified_files.sh ui/
-      
-      - name: ESLint prerequisites ui
-        if: steps.ui_modified_files.outputs.matched == 'true'
-        run: |
-          sudo npm install -g eslint
-          cd ui
-          npm run prestart
-          npm i
-
-      - name: ESLint ui
-        if: steps.ui_modified_files.outputs.matched == 'true'
-        run: |
-          cd ui
-          npm run eslint
-      
-      - name: Check modified ui-common files
-        id: ui_common_modified_files
-        run: devops/check_modified_files.sh ui-common/
-      
-      - name: ESLint prerequisites ui-common
-        if: steps.ui_common_modified_files.outputs.matched == 'true'
-        run: |
-          sudo npm install -g eslint
-          cd ui-common
-          npm i
-
-      - name: ESLint ui-common
-        if: steps.ui_common_modified_files.outputs.matched == 'true'
-        run: |
-          cd ui-common
-          npm run eslint

.gitignore

@@ -52,7 +52,4 @@ tap/extensions/*/expect
 # UI folders to ignore
 **/node_modules/**
 **/dist/**
-*.editorconfig
-
-# Ignore *.log files
-*.log
+*.editorconfig

Dockerfile

@@ -44,18 +44,11 @@ ENV CGO_ENABLED=1 GOOS=linux
 ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap"
 
 
-### Builder image for AArch64 to x86-64 cross-compilation
-FROM up9inc/linux-x86_64-musl-go-libpcap AS builder-from-arm64v8-to-amd64
-ENV CGO_ENABLED=1 GOOS=linux
-ENV GOARCH=amd64 CGO_CFLAGS="-I/libpcap"
-
-
 ### Final builder image where the build happens
 # Possible build strategies:
 # BUILDARCH=amd64 TARGETARCH=amd64
 # BUILDARCH=arm64v8 TARGETARCH=arm64v8
 # BUILDARCH=amd64 TARGETARCH=arm64v8
-# BUILDARCH=arm64v8 TARGETARCH=amd64
 ARG BUILDARCH=amd64
 ARG TARGETARCH=amd64
 FROM builder-from-${BUILDARCH}-to-${TARGETARCH} AS builder
@@ -65,19 +58,18 @@ WORKDIR /app/agent-build
 
 COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
-COPY logger/go.mod logger/go.mod ../logger/
 COPY tap/go.mod tap/go.mod ../tap/
 COPY tap/api/go.mod ../tap/api/
-COPY tap/dbgctl/go.mod ../tap/dbgctl/
 COPY tap/extensions/amqp/go.mod ../tap/extensions/amqp/
 COPY tap/extensions/http/go.mod ../tap/extensions/http/
 COPY tap/extensions/kafka/go.mod ../tap/extensions/kafka/
 COPY tap/extensions/redis/go.mod ../tap/extensions/redis/
 RUN go mod download
+# cheap trick to make the build faster (as long as go.mod did not change)
+RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
 
 # Copy and build agent code
 COPY shared ../shared
-COPY logger ../logger
 COPY tap ../tap
 COPY agent .
 
@@ -95,8 +87,8 @@ RUN go build -ldflags="-extldflags=-static -s -w \
     -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum
-ADD https://github.com/up9inc/basenine/releases/download/v0.8.2/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
-ADD https://github.com/up9inc/basenine/releases/download/v0.8.2/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
+ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
 
 RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
     chmod +x ./basenine_linux_"${GOARCH}" && \

Makefile

@@ -83,7 +83,6 @@ test-lint:  ## Run lint on all modules
 	cd cli && golangci-lint run
 	cd acceptanceTests && golangci-lint run
 	cd tap/api && golangci-lint run
-	cd tap/dbgctl && golangci-lint run
 	cd tap/extensions/ && for D in */; do cd $$D && golangci-lint run && cd ..; done
 
 test-cli:  ## Run cli tests

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -57,8 +57,15 @@ export function rightOnHoverCheck(path, expectedText) {
     cy.get(`#rightSideContainer [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(expectedText));
 }
 
+export function checkThatAllEntriesShown() {
+    cy.get('#entries-length').then(number => {
+        if (number.text() === '1')
+            cy.get('[title="Fetch old records"]').click();
+    });
+}
+
 export function checkFilterByMethod(funcDict) {
-    const {protocol, method, methodQuery, summary, summaryQuery, numberOfRecords} = funcDict;
+    const {protocol, method, methodQuery, summary, summaryQuery} = funcDict;
     const summaryDict = getSummaryDict(summary, summaryQuery);
     const methodDict = getMethodDict(method, methodQuery);
     const protocolDict = getProtocolDict(protocol.name, protocol.text);
@@ -69,53 +76,53 @@ export function checkFilterByMethod(funcDict) {
         cy.get('[type="submit"]').click();
         cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
 
-        waitForFetch(numberOfRecords);
-        pauseStream();
+        cy.get('#entries-length').then(number => {
+            // if the entries list isn't expanded it expands here
+            if (number.text() === '0' || number.text() === '1') // todo change when TRA-4262 is fixed
+                cy.get('[title="Fetch old records"]').click();
 
-        cy.get(`#list [id^=entry]`).then(elements => {
-            const listElmWithIdAttr = Object.values(elements);
-            let doneCheckOnFirst = false;
+            cy.get('#entries-length').should('not.have.text', '0').and('not.have.text', '1').then(() => {
+                cy.get(`#list [id]`).then(elements => {
+                    const listElmWithIdAttr = Object.values(elements);
+                    let doneCheckOnFirst = false;
 
-            cy.get('#entries-length').invoke('text').then(len => {
-                listElmWithIdAttr.forEach(entry => {
-                    if (entry?.id && entry.id.match(RegExp(/entry-(\d{24})$/gm))) {
-                        const entryId = getEntryId(entry.id);
+                    cy.get('#entries-length').invoke('text').then(len => {
+                        resizeIfNeeded(len);
+                        listElmWithIdAttr.forEach(entry => {
+                            if (entry?.id && entry.id.match(RegExp(/entry-(\d{24})$/gm))) {
+                                const entryId = getEntryId(entry.id);
 
-                        leftTextCheck(entryId, methodDict.pathLeft, methodDict.expectedText);
-                        leftTextCheck(entryId, protocolDict.pathLeft, protocolDict.expectedTextLeft);
-                        if (summaryDict)
-                            leftTextCheck(entryId, summaryDict.pathLeft, summaryDict.expectedText);
+                                leftTextCheck(entryId, methodDict.pathLeft, methodDict.expectedText);
+                                leftTextCheck(entryId, protocolDict.pathLeft, protocolDict.expectedTextLeft);
+                                if (summaryDict)
+                                    leftTextCheck(entryId, summaryDict.pathLeft, summaryDict.expectedText);
 
-                        if (!doneCheckOnFirst) {
-                            deepCheck(funcDict, protocolDict, methodDict, entry);
-                            doneCheckOnFirst = true;
-                        }
-                    }
+                                if (!doneCheckOnFirst) {
+                                    deepCheck(funcDict, protocolDict, methodDict, entry);
+                                    doneCheckOnFirst = true;
+                                }
+                            }
+                        });
+                        resizeIfNeeded(len);
+                    });
                 });
             });
         });
     });
 }
 
-export const refreshWaitTimeout = 10000;
-
-export function waitForFetch(gt) {
-    cy.get('#entries-length', {timeout: refreshWaitTimeout}).should((el) => {
-        expect(parseInt(el.text().trim(), 10)).to.be.greaterThan(gt);
-    });
-}
-
-export function pauseStream() {
-    cy.get('#pause-icon').click();
-    cy.get('#pause-icon').should('not.be.visible');
-}
-
-
 export function getEntryId(id) {
     // take the second part from the string (entry-<ID>)
     return id.split('-')[1];
 }
 
+function resizeIfNeeded(entriesLen) {
+    if (entriesLen > maxEntriesInDom){
+        Cypress.config().viewportHeight === Cypress.env('normalMizuHeight') ?
+            resizeToHugeMizu() : resizeToNormalMizu()
+    }
+}
+
 function deepCheck(generalDict, protocolDict, methodDict, entry) {
     const entryId = getEntryId(entry.id);
     const {summary, value} = generalDict;

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -1,4 +1,5 @@
 import {
+    checkThatAllEntriesShown,
     isValueExistsInElement,
     resizeToHugeMizu,
 } from "../testHelpers/TrafficHelper";
@@ -11,14 +12,13 @@ checkEntries();
 
 function checkEntries() {
     it('checking all entries', function () {
-        cy.get('#entries-length').should('not.have.text', '0').then(() => {
-            resizeToHugeMizu();
+        checkThatAllEntriesShown();
+        resizeToHugeMizu();
 
-            cy.get('#list [id^=entry]').each(entryElement => {
-                entryElement.click();
-                cy.get('#tbody-Headers').should('be.visible');
-                isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
-            });
+        cy.get('#list [id^=entry]').each(entryElement => {
+            entryElement.click();
+            cy.get('#tbody-Headers').should('be.visible');
+            isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
         });
     });
 }

acceptanceTests/cypress/integration/tests/Rabbit.js

@@ -5,7 +5,6 @@ it('opening mizu', function () {
 });
 
 const rabbitProtocolDetails = {name: 'AMQP', text: 'Advanced Message Queuing Protocol 0-9-1'};
-const numberOfRecords = 5;
 
 checkFilterByMethod({
     protocol: rabbitProtocolDetails,
@@ -13,7 +12,6 @@ checkFilterByMethod({
     methodQuery: 'request.method == "exchange declare"',
     summary: 'exchange',
     summaryQuery: 'request.exchange == "exchange"',
-    numberOfRecords: numberOfRecords,
     value: null
 });
 
@@ -23,7 +21,6 @@ checkFilterByMethod({
     methodQuery: 'request.method == "queue declare"',
     summary: 'queue',
     summaryQuery: 'request.queue == "queue"',
-    numberOfRecords: numberOfRecords,
     value: null
 });
 
@@ -33,7 +30,6 @@ checkFilterByMethod({
     methodQuery: 'request.method == "queue bind"',
     summary: 'queue',
     summaryQuery: 'request.queue == "queue"',
-    numberOfRecords: numberOfRecords,
     value: null
 });
 
@@ -43,7 +39,6 @@ checkFilterByMethod({
     methodQuery: 'request.method == "basic publish"',
     summary: 'exchange',
     summaryQuery: 'request.exchange == "exchange"',
-    numberOfRecords: numberOfRecords,
     value: {tab: valueTabs.request, regex: /^message$/mg}
 });
 
@@ -53,7 +48,6 @@ checkFilterByMethod({
     methodQuery: 'request.method == "basic consume"',
     summary: 'queue',
     summaryQuery: 'request.queue == "queue"',
-    numberOfRecords: numberOfRecords,
     value: null
 });
 
@@ -63,6 +57,5 @@ checkFilterByMethod({
     methodQuery: 'request.method == "basic deliver"',
     summary: 'exchange',
     summaryQuery: 'request.queue == "exchange"',
-    numberOfRecords: numberOfRecords,
     value: {tab: valueTabs.request, regex: /^message$/mg}
 });

acceptanceTests/cypress/integration/tests/Redis.js

@@ -5,7 +5,6 @@ it('opening mizu', function () {
 });
 
 const redisProtocolDetails = {name: 'redis', text: 'Redis Serialization Protocol'};
-const numberOfRecords = 5;
 
 checkFilterByMethod({
     protocol: redisProtocolDetails,
@@ -13,7 +12,6 @@ checkFilterByMethod({
     methodQuery: 'request.command == "PING"',
     summary: null,
     summaryQuery: '',
-    numberOfRecords: numberOfRecords,
     value: null
 })
 
@@ -23,7 +21,6 @@ checkFilterByMethod({
     methodQuery: 'request.command == "SET"',
     summary: 'key',
     summaryQuery: 'request.key == "key"',
-    numberOfRecords: numberOfRecords,
     value: {tab: valueTabs.request, regex: /^\[value, keepttl]$/mg}
 })
 
@@ -33,7 +30,6 @@ checkFilterByMethod({
     methodQuery: 'request.command == "EXISTS"',
     summary: 'key',
     summaryQuery: 'request.key == "key"',
-    numberOfRecords: numberOfRecords,
     value: {tab: valueTabs.response, regex: /^1$/mg}
 })
 
@@ -43,7 +39,6 @@ checkFilterByMethod({
     methodQuery: 'request.command == "GET"',
     summary: 'key',
     summaryQuery: 'request.key == "key"',
-    numberOfRecords: numberOfRecords,
     value: {tab: valueTabs.response, regex: /^value$/mg}
 })
 
@@ -53,6 +48,5 @@ checkFilterByMethod({
     methodQuery: 'request.command == "DEL"',
     summary: 'key',
     summaryQuery: 'request.key == "key"',
-    numberOfRecords: numberOfRecords,
     value: {tab: valueTabs.response, regex: /^1$|^0$/mg}
 })

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -7,12 +7,12 @@ import {
     resizeToNormalMizu,
     rightOnHoverCheck,
     rightTextCheck,
-    verifyMinimumEntries,
-    refreshWaitTimeout,
-    waitForFetch,
-    pauseStream
+    verifyMinimumEntries
 } from "../testHelpers/TrafficHelper";
 
+const refreshWaitTimeout = 10000;
+
+
 const fullParam = Cypress.env('arrayDict'); // "Name:fooNamespace:barName:foo1Namespace:bar1"
 const podsArray = fullParam.split('Name:').slice(1); // ["fooNamespace:bar", "foo1Namespace:bar1"]
 podsArray.forEach((podStr, index) => {
@@ -65,77 +65,70 @@ it('right side sanity test', function () {
 checkIllegalFilter('invalid filter');
 
 checkFilter({
-    filter: 'http',
+    name: 'http',
     leftSidePath: '> :nth-child(1) > :nth-child(1)',
     leftSideExpectedText: 'HTTP',
     rightSidePath: '[title=HTTP]',
     rightSideExpectedText: 'Hypertext Transfer Protocol -- HTTP/1.1',
-    applyByCtrlEnter: true,
-    numberOfRecords: 20,
+    applyByEnter: true
 });
 
 checkFilter({
-    filter: 'response.status == 200',
+    name: 'response.status == 200',
     leftSidePath: '[title="Status Code"]',
     leftSideExpectedText: '200',
     rightSidePath: '> :nth-child(2) [title="Status Code"]',
     rightSideExpectedText: '200',
-    applyByCtrlEnter: false,
-    numberOfRecords: 20
+    applyByEnter: false
 });
 
 if (Cypress.env('shouldCheckSrcAndDest')) {
     serviceMapCheck();
 
     checkFilter({
-        filter: 'src.name == ""',
+        name: 'src.name == ""',
         leftSidePath: '[title="Source Name"]',
         leftSideExpectedText: '[Unresolved]',
         rightSidePath: '> :nth-child(2) [title="Source Name"]',
         rightSideExpectedText: '[Unresolved]',
-        applyByCtrlEnter: false,
-        numberOfRecords: 20
+        applyByEnter: false
     });
 
     checkFilter({
-        filter: `dst.name == "httpbin.mizu-tests"`,
+        name: `dst.name == "httpbin.mizu-tests"`,
         leftSidePath: '> :nth-child(3) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
         leftSideExpectedText: 'httpbin.mizu-tests',
         rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
         rightSideExpectedText: 'httpbin.mizu-tests',
-        applyByCtrlEnter: false,
-        numberOfRecords: 20
+        applyByEnter: false
     });
 }
 
 checkFilter({
-    filter: 'request.method == "GET"',
+    name: 'request.method == "GET"',
     leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
     leftSideExpectedText: 'GET',
     rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
     rightSideExpectedText: 'GET',
-    applyByCtrlEnter: true,
-    numberOfRecords: 20
+    applyByEnter: true
 });
 
 checkFilter({
-    filter: 'request.path == "/get"',
+    name: 'request.path == "/get"',
     leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
     leftSideExpectedText: '/get',
     rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
     rightSideExpectedText: '/get',
-    applyByCtrlEnter: false,
-    numberOfRecords: 20
+    applyByEnter: false
 });
 
 checkFilter({
-    filter: 'src.ip == "127.0.0.1"',
+    name: 'src.ip == "127.0.0.1"',
     leftSidePath: '[title="Source IP"]',
     leftSideExpectedText: '127.0.0.1',
     rightSidePath: '> :nth-child(2) [title="Source IP"]',
     rightSideExpectedText: '127.0.0.1',
-    applyByCtrlEnter: false,
-    numberOfRecords: 20
+    applyByEnter: false
 });
 
 checkFilterNoResults('request.method == "POST"');
@@ -189,62 +182,67 @@ function checkIllegalFilter(illegalFilterName) {
 
 function checkFilter(filterDetails) {
     const {
-        filter,
+        name,
         leftSidePath,
         rightSidePath,
         rightSideExpectedText,
         leftSideExpectedText,
-        applyByCtrlEnter,
-        numberOfRecords
+        applyByEnter
     } = filterDetails;
 
     const entriesForDeeperCheck = 5;
 
-    it(`checking the filter: ${filter}`, function () {
-        cy.get('.w-tc-editor-text').clear();
-        // applying the filter with alt+enter or with the button
-        cy.get('.w-tc-editor-text').type(`${filter}${applyByCtrlEnter ? '{ctrl+enter}' : ''}`);
-        cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
-        if (!applyByCtrlEnter)
-            cy.get('[type="submit"]').click();
+    it(`checking the filter: ${name}`, function () {
+        cy.get('#total-entries').should('not.have.text', '0').then(number => {
+            const totalEntries = number.text();
 
-        waitForFetch(numberOfRecords);
-        pauseStream();
+            cy.get(`#list [id^=entry]`).last().then(elem => {
+                const element = elem[0];
+                const entryId = getEntryId(element.id);
+                // checks the hover on the last entry (the only one in DOM at the beginning)
+                leftOnHoverCheck(entryId, leftSidePath, name);
 
-        cy.get(`#list [id^=entry]`).last().then(elem => {
-            const element = elem[0];
-            const entryId = getEntryId(element.id);
+                cy.get('.w-tc-editor-text').clear();
+                // applying the filter with alt+enter or with the button
+                cy.get('.w-tc-editor-text').type(`${name}${applyByEnter ? '{alt+enter}' : ''}`);
+                cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
+                if (!applyByEnter)
+                    cy.get('[type="submit"]').click();
 
-            // only one entry in DOM after filtering, checking all checks on it
-            leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
-            leftOnHoverCheck(entryId, leftSidePath, filter);
+                // only one entry in DOM after filtering, checking all checks on it
+                leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
+                leftOnHoverCheck(entryId, leftSidePath, name);
 
-            rightTextCheck(rightSidePath, rightSideExpectedText);
-            rightOnHoverCheck(rightSidePath, filter);
-            checkRightSideResponseBody();
-        });
+                rightTextCheck(rightSidePath, rightSideExpectedText);
+                rightOnHoverCheck(rightSidePath, name);
+                checkRightSideResponseBody();
+            });
 
-        resizeToHugeMizu();
+            cy.get('[title="Fetch old records"]').click();
+            resizeToHugeMizu();
 
-        // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
-        cy.get(`#list [id^=entry]`).each(elem => {
-            const element = elem[0];
-            let entryId = getEntryId(element.id);
-            leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
-        });
+            // waiting for the entries number to load
+            cy.get('#entries-length', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
 
-        // making the other 3 checks on the first X entries (longer time for each check)
-        deeperCheck(leftSidePath, rightSidePath, filter, rightSideExpectedText, entriesForDeeperCheck);
+            // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
+            cy.get(`#list [id^=entry]`).each(elem => {
+                const element = elem[0];
+                let entryId = getEntryId(element.id);
+                leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
+            });
 
-        // reloading then waiting for the entries number to load
-        resizeToNormalMizu();
-        cy.reload();
-        waitForFetch(numberOfRecords);
-        pauseStream();
+            // making the other 3 checks on the first X entries (longer time for each check)
+            deeperCheck(leftSidePath, rightSidePath, name, leftSideExpectedText, rightSideExpectedText, entriesForDeeperCheck);
+
+            // reloading then waiting for the entries number to load
+            resizeToNormalMizu();
+            cy.reload();
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+        })
     });
 }
 
-function deeperCheck(leftSidePath, rightSidePath, filterName, rightSideExpectedText, entriesNumToCheck) {
+function deeperCheck(leftSidePath, rightSidePath, filterName, leftSideExpectedText, rightSideExpectedText, entriesNumToCheck) {
     cy.get(`#list [id^=entry]`).each((element, index) => {
         if (index < entriesNumToCheck) {
             const entryId = getEntryId(element[0].id);

acceptanceTests/go.mod

@@ -29,7 +29,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/up9inc/mizu/logger v0.0.0 // indirect
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
@@ -49,10 +48,6 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace github.com/up9inc/mizu/logger v0.0.0 => ../logger
-
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
-
-replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../tap/dbgctl

acceptanceTests/setup.sh

@@ -1,10 +1,7 @@
 #!/bin/bash
-set -e
 
 PREFIX=$HOME/local/bin
 VERSION=v1.22.0
-TUNNEL_LOG="tunnel.log"
-PROXY_LOG="proxy.log"
 
 echo "Attempting to install minikube and assorted tools to $PREFIX"
 
@@ -14,7 +11,7 @@ if ! [ -x "$(command -v kubectl)" ]; then
   chmod +x kubectl
   mv kubectl "$PREFIX"
 else
-  echo "kubectl is already installed"
+  echo "kubetcl is already installed"
 fi
 
 if ! [ -x "$(command -v minikube)" ]; then
@@ -30,57 +27,44 @@ echo "Starting minikube..."
 minikube start
 
 echo "Creating mizu tests namespaces"
-kubectl create namespace mizu-tests --dry-run=client -o yaml | kubectl apply -f -
-kubectl create namespace mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
+kubectl create namespace mizu-tests
+kubectl create namespace mizu-tests2
 
 echo "Creating httpbin deployments"
-kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
-kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests
+kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests
 
-kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2
 
 echo "Creating redis deployment"
-kubectl create deployment redis --image=redis -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl create deployment redis --image=redis -n mizu-tests
 
 echo "Creating rabbitmq deployment"
-kubectl create deployment rabbitmq --image=rabbitmq -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl create deployment rabbitmq --image=rabbitmq -n mizu-tests
 
 echo "Creating httpbin services"
-kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
-kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
+kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests
 
-kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2
 
 echo "Creating redis service"
-kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests
 
 echo "Creating rabbitmq service"
-kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests
 
-# TODO: need to understand how to fail if address already in use
 echo "Starting proxy"
-rm -f ${PROXY_LOG}
-kubectl proxy --port=8080 > ${PROXY_LOG} &
-PID1=$!
-echo "kubectl proxy process id is ${PID1} and log of proxy in ${PROXY_LOG}"
+kubectl proxy --port=8080 &
 
-if [[ -z "${CI}" ]]; then
-  echo "Setting env var of mizu ci image"
-  export MIZU_CI_IMAGE="mizu/ci:0.0"
-  echo "Build agent image"
-  docker build -t "${MIZU_CI_IMAGE}" .
-else
-  echo "not building docker image in CI because it is created as separate step"
-fi
+echo "Setting minikube docker env"
+eval $(minikube docker-env)
 
-minikube image load "${MIZU_CI_IMAGE}"
+echo "Build agent image"
+docker build -t mizu/ci:0.0 .
 
 echo "Build cli"
 cd cli && make build GIT_BRANCH=ci SUFFIX=ci
 
-# TODO: need to understand how to fail if password is asked (sudo)
 echo "Starting tunnel"
-rm -f ${TUNNEL_LOG}
-minikube tunnel > ${TUNNEL_LOG} &
-PID2=$!
-echo "Minikube tunnel process id is ${PID2} and log of tunnel in ${TUNNEL_LOG}"
+minikube tunnel &

acceptanceTests/testsUtils.go

@@ -212,10 +212,9 @@ func DeleteKubeFile(kubeContext string, namespace string, filename string) error
 }
 
 func getDefaultCommandArgs() []string {
-	agentImageValue := os.Getenv("MIZU_CI_IMAGE")
 	setFlag := "--set"
 	telemetry := "telemetry=false"
-	agentImage := fmt.Sprintf("agent-image=%s", agentImageValue)
+	agentImage := "agent-image=mizu/ci:0.0"
 	imagePullPolicy := "image-pull-policy=IfNotPresent"
 	headless := "headless=true"
 

agent/go.mod

@@ -6,8 +6,8 @@ require (
 	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
 	github.com/chanced/openapi v0.0.8
 	github.com/djherbis/atime v1.1.0
+	github.com/elastic/go-elasticsearch/v7 v7.17.0
 	github.com/getkin/kin-openapi v0.89.0
-	github.com/gin-contrib/pprof v1.3.0
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.7.7
 	github.com/go-playground/locales v0.14.0
@@ -19,13 +19,12 @@ require (
 	github.com/nav-inc/datetime v0.1.3
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v1.0.0
+    github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/stretchr/testify v1.7.0
-	github.com/up9inc/basenine/client/go v0.0.0-20220509204026-c37adfc587f4
-	github.com/up9inc/mizu/logger v0.0.0
+	github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
-	github.com/up9inc/mizu/tap/dbgctl v0.0.0
 	github.com/up9inc/mizu/tap/extensions/amqp v0.0.0
 	github.com/up9inc/mizu/tap/extensions/http v0.0.0
 	github.com/up9inc/mizu/tap/extensions/kafka v0.0.0
@@ -62,7 +61,6 @@ require (
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.2.2 // indirect
-	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.6 // indirect
 	github.com/go-openapi/swag v0.21.1 // indirect
@@ -78,7 +76,6 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -88,7 +85,6 @@ require (
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/mertyildiran/gqlparser/v2 v2.4.6 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
@@ -104,22 +100,17 @@ require (
 	github.com/russross/blackfriday v1.6.0 // indirect
 	github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
 	github.com/segmentio/kafka-go v0.4.27 // indirect
-	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/spf13/cobra v1.3.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/struCoder/pidusage v0.2.1 // indirect
 	github.com/tidwall/gjson v1.14.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/sjson v1.2.4 // indirect
-	github.com/tklauser/go-sysconf v0.3.10 // indirect
-	github.com/tklauser/numcpus v0.4.0 // indirect
 	github.com/ugorji/go/codec v1.2.6 // indirect
 	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect
-	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.starlark.net v0.0.0-20220203230714-bb14e151c28f // indirect
-	golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b // indirect
+	golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab // indirect
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
@@ -144,8 +135,6 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace github.com/up9inc/mizu/logger v0.0.0 => ../logger
-
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 
 replace github.com/up9inc/mizu/tap v0.0.0 => ../tap
@@ -159,5 +148,3 @@ replace github.com/up9inc/mizu/tap/extensions/http v0.0.0 => ../tap/extensions/h
 replace github.com/up9inc/mizu/tap/extensions/kafka v0.0.0 => ../tap/extensions/kafka
 
 replace github.com/up9inc/mizu/tap/extensions/redis v0.0.0 => ../tap/extensions/redis
-
-replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../tap/dbgctl

agent/go.sum

@@ -83,7 +83,6 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -166,6 +165,8 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/elastic/go-elasticsearch/v7 v7.17.0 h1:0fcSh4qeC/i1+7QU1KXpmq2iUAdMk4l0/vmbtW1+KJM=
+github.com/elastic/go-elasticsearch/v7 v7.17.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -213,13 +214,10 @@ github.com/getkin/kin-openapi v0.89.0 h1:p4nagHchUKGn85z/f+pse4aSh50nIBOYjOhMIku
 github.com/getkin/kin-openapi v0.89.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/gin-contrib/pprof v1.3.0 h1:G9eK6HnbkSqDZBYbzG4wrjCsA4e+cvYAHUZw6W+W9K0=
-github.com/gin-contrib/pprof v1.3.0/go.mod h1:waMjT1H9b179t3CxuG1cV3DHpga6ybizwfBaM5OXaB0=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-contrib/static v0.0.1 h1:JVxuvHPuUfkoul12N7dtQw7KRn/pSMq7Ue1Va9Swm1U=
 github.com/gin-contrib/static v0.0.1/go.mod h1:CSxeF+wep05e0kCOsqWdAWbSszmc31zTIbD8TvWl7Hs=
-github.com/gin-gonic/gin v1.6.2/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs=
 github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=
@@ -241,8 +239,6 @@ github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
 github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro=
-github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
-github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -408,7 +404,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
@@ -498,8 +493,6 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/mertyildiran/gqlparser/v2 v2.4.6 h1:enAq4F5PYgW/rYExPNzYt7IYrrZnzrfqdywMA1QdFtM=
-github.com/mertyildiran/gqlparser/v2 v2.4.6/go.mod h1:XZId58F+XqRSmoLrdsOLgqA918oNvBzuOORruJWBjDo=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
@@ -561,6 +554,8 @@ github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HD
 github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
@@ -623,8 +618,6 @@ github.com/segmentio/kafka-go v0.4.27 h1:sIhEozeL/TLN2mZ5dkG462vcGEWYKS+u31sXPjK
 github.com/segmentio/kafka-go v0.4.27/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
-github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
-github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -666,8 +659,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/struCoder/pidusage v0.2.1 h1:dFiEgUDkubeIj0XA1NpQ6+8LQmKrLi7NiIQl86E6BoY=
-github.com/struCoder/pidusage v0.2.1/go.mod h1:bewtP2KUA1TBUyza5+/PCpSQ6sc/H6jJbIKAzqW86BA=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.12.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
@@ -681,10 +672,6 @@ github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso
 github.com/tidwall/sjson v1.2.3/go.mod h1:5WdjKx3AQMvCJ4RG6/2UYT7dLrGvJUV1x4jdTAyGvZs=
 github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc=
 github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
-github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
-github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
-github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
-github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
@@ -694,8 +681,8 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
 github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
-github.com/up9inc/basenine/client/go v0.0.0-20220509204026-c37adfc587f4 h1:nNOrU1HVH0fnaG7GNhxCc8kNPVL035Iix7ihUF6lZT8=
-github.com/up9inc/basenine/client/go v0.0.0-20220509204026-c37adfc587f4/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607 h1:UqxUSkOYOmsLZWQtMSk02ttnhdRwBRLOLt2aDiS9tEk=
+github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=
@@ -715,8 +702,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
-github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
@@ -768,9 +753,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab h1:lnZ4LoV0UMdibeCUfIB2a4uFwRu491WX/VB2reB8xNc=
 golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b h1:Qwe1rC8PSniVfAFPFJeyUkB+zcysC3RgJBAGk7eqBEU=
-golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -909,7 +893,6 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -978,7 +961,6 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220207234003-57398862261d h1:Bm7BNOQt2Qv7ZqysjeLjgCBanX+88Z/OtdvsrEv1Djc=
 golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

agent/main.go

@@ -14,10 +14,10 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/gin-contrib/pprof"
 	"github.com/gin-contrib/static"
 	"github.com/gin-gonic/gin"
 	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/elastic"
 	"github.com/up9inc/mizu/agent/pkg/entries"
 	"github.com/up9inc/mizu/agent/pkg/middlewares"
 	"github.com/up9inc/mizu/agent/pkg/models"
@@ -33,11 +33,10 @@ import (
 
 	"github.com/gorilla/websocket"
 	"github.com/op/go-logging"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap"
 	tapApi "github.com/up9inc/mizu/tap/api"
-	"github.com/up9inc/mizu/tap/dbgctl"
 )
 
 var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API")
@@ -47,7 +46,6 @@ var apiServerAddress = flag.String("api-server-address", "", "Address of mizu AP
 var namespace = flag.String("namespace", "", "Resolve IPs if they belong to resources in this namespace (default is all)")
 var harsReaderMode = flag.Bool("hars-read", false, "Run in hars-read mode")
 var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
-var profiler = flag.Bool("profiler", false, "Run pprof server")
 
 const (
 	socketConnectionRetries    = 30
@@ -64,7 +62,7 @@ func main() {
 	app.LoadExtensions()
 
 	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode {
-		panic("One of the flags --tap, --api-server, --standalone or --hars-read must be provided")
+		panic("One of the flags --tap, --api or --standalone or --hars-read must be provided")
 	}
 
 	if *standaloneMode {
@@ -72,14 +70,7 @@ func main() {
 	} else if *tapperMode {
 		runInTapperMode()
 	} else if *apiServerMode {
-		app := runInApiServerMode(*namespace)
-
-		if *profiler {
-			pprof.Register(app)
-		}
-
-		utils.StartServer(app)
-
+		utils.StartServer(runInApiServerMode(*namespace))
 	} else if *harsReaderMode {
 		runInHarReaderMode()
 	}
@@ -162,9 +153,7 @@ func runInTapperMode() {
 	}
 
 	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-	tapOpts := &tap.TapOpts{
-		HostMode:     hostMode,
-	}
+	tapOpts := &tap.TapOpts{HostMode: hostMode}
 
 	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
 
@@ -216,6 +205,7 @@ func enableExpFeatureIfNeeded() {
 		serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap)
 		serviceMapGenerator.Enable()
 	}
+	elastic.GetInstance().Configure(config.Config.Elastic)
 }
 
 func getSyncEntriesConfig() *shared.SyncEntriesConfig {
@@ -293,10 +283,6 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
 			continue
 		}
 
-		if dbgctl.MizuTapperDisableSending {
-			continue
-		}
-
 		// NOTE: This is where the `*tapApi.OutputChannelItem` leaves the code
 		// and goes into the intermediate WebSocket.
 		err = connection.WriteMessage(websocket.TextMessage, marshaledData)
@@ -387,7 +373,6 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
 func initializeDependencies() {
 	dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
 	dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} { return oas.GetDefaultOasGeneratorInstance() })
-	dependency.RegisterGenerator(dependency.EntriesInserter, func() interface{} { return api.GetBasenineEntryInserterInstance() })
 	dependency.RegisterGenerator(dependency.EntriesProvider, func() interface{} { return &entries.BasenineEntriesProvider{} })
 	dependency.RegisterGenerator(dependency.EntriesSocketStreamer, func() interface{} { return &api.BasenineEntryStreamer{} })
 	dependency.RegisterGenerator(dependency.EntryStreamerSocketConnector, func() interface{} { return &api.DefaultEntryStreamerSocketConnector{} })

agent/pkg/api/entry_streamer_socket_connector.go

@@ -5,19 +5,20 @@ import (
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 type EntryStreamerSocketConnector interface {
-	SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error
-	SendMetadata(socketId int, metadata *basenine.Metadata) error
-	SendToastError(socketId int, err error) error
+	SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams)
+	SendMetadata(socketId int, metadata *basenine.Metadata)
+	SendToastError(socketId int, err error)
 	CleanupSocket(socketId int)
 }
 
 type DefaultEntryStreamerSocketConnector struct{}
 
-func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error {
+func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) {
 	var message []byte
 	if params.EnableFullEntries {
 		message, _ = models.CreateFullEntryWebSocketMessage(entry)
@@ -28,32 +29,26 @@ func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tap
 	}
 
 	if err := SendToSocket(socketId, message); err != nil {
-		return err
+		logger.Log.Error(err)
 	}
-
-	return nil
 }
 
-func (e *DefaultEntryStreamerSocketConnector) SendMetadata(socketId int, metadata *basenine.Metadata) error {
+func (e *DefaultEntryStreamerSocketConnector) SendMetadata(socketId int, metadata *basenine.Metadata) {
 	metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
 	if err := SendToSocket(socketId, metadataBytes); err != nil {
-		return err
+		logger.Log.Error(err)
 	}
-
-	return nil
 }
 
-func (e *DefaultEntryStreamerSocketConnector) SendToastError(socketId int, err error) error {
+func (e *DefaultEntryStreamerSocketConnector) SendToastError(socketId int, err error) {
 	toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
 		Type:      "error",
 		AutoClose: 5000,
 		Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
 	})
 	if err := SendToSocket(socketId, toastBytes); err != nil {
-		return err
+		logger.Log.Error(err)
 	}
-
-	return nil
 }
 
 func (e *DefaultEntryStreamerSocketConnector) CleanupSocket(socketId int) {

agent/pkg/api/main.go

@@ -14,6 +14,7 @@ import (
 	"github.com/up9inc/mizu/agent/pkg/models"
 
 	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/elastic"
 	"github.com/up9inc/mizu/agent/pkg/har"
 	"github.com/up9inc/mizu/agent/pkg/holder"
 	"github.com/up9inc/mizu/agent/pkg/providers"
@@ -23,8 +24,11 @@ import (
 	"github.com/up9inc/mizu/agent/pkg/resolver"
 	"github.com/up9inc/mizu/agent/pkg/utils"
 
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
+
+	basenine "github.com/up9inc/basenine/client/go"
 )
 
 var k8sResolver *resolver.Resolver
@@ -99,6 +103,20 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		panic("Channel of captured messages is nil")
 	}
 
+BasenineReconnect:
+	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+	if err != nil {
+		logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
+		time.Sleep(shared.BasenineReconnectInterval * time.Second)
+		goto BasenineReconnect
+	}
+	if err = connection.InsertMode(); err != nil {
+		logger.Log.Errorf("Insert mode call failed: %v", err)
+		connection.Close()
+		time.Sleep(shared.BasenineReconnectInterval * time.Second)
+		goto BasenineReconnect
+	}
+
 	disableOASValidation := false
 	ctx := context.Background()
 	doc, contractContent, router, err := loadOAS(ctx)
@@ -110,11 +128,11 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 	for item := range outputItems {
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation, namespace := resolveIP(item.ConnectionInfo)
-
+		
 		if namespace == "" && item.Namespace != tapApi.UNKNOWN_NAMESPACE {
 			namespace = item.Namespace
 		}
-
+		
 		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation, namespace)
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
@@ -145,13 +163,17 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 
 		providers.EntryAdded(len(data))
 
-		entryInserter := dependency.GetInstance(dependency.EntriesInserter).(EntryInserter)
-		if err := entryInserter.Insert(mizuEntry); err != nil {
-			logger.Log.Errorf("Error inserting entry, err: %v", err)
+		if err = connection.SendText(string(data)); err != nil {
+			logger.Log.Errorf("An error occured while inserting a new record to database: %v", err)
+			connection.Close()
+			time.Sleep(shared.BasenineReconnectInterval * time.Second)
+			goto BasenineReconnect
 		}
 
 		serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMapSink)
 		serviceMapGenerator.NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
+
+		elastic.GetInstance().PushEntry(mizuEntry)
 	}
 }
 

agent/pkg/api/socket_data_inserter.go

@@ -1,71 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-	"fmt"
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/logger"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap/api"
-	"sync"
-	"time"
-)
-
-type EntryInserter interface {
-	Insert(entry *api.Entry) error
-}
-
-type BasenineEntryInserter struct {
-	connection *basenine.Connection
-}
-
-var instance *BasenineEntryInserter
-var once sync.Once
-
-func GetBasenineEntryInserterInstance() *BasenineEntryInserter {
-	once.Do(func() {
-		instance = &BasenineEntryInserter{}
-	})
-
-	return instance
-}
-
-func (e *BasenineEntryInserter) Insert(entry *api.Entry) error {
-	if e.connection == nil {
-		e.connection = initializeConnection()
-	}
-
-	data, err := json.Marshal(entry)
-	if err != nil {
-		return fmt.Errorf("error marshling entry, err: %v", err)
-	}
-
-	if err := e.connection.SendText(string(data)); err != nil {
-		e.connection.Close()
-		e.connection = nil
-
-		return fmt.Errorf("error sending text to database, err: %v", err)
-	}
-
-	return nil
-}
-
-func initializeConnection() *basenine.Connection{
-	for {
-		connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
-		if err != nil {
-			logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
-			time.Sleep(shared.BasenineReconnectInterval * time.Second)
-			continue
-		}
-
-		if err = connection.InsertMode(); err != nil {
-			logger.Log.Errorf("Insert mode call failed: %v", err)
-			connection.Close()
-			time.Sleep(shared.BasenineReconnectInterval * time.Second)
-			continue
-		}
-
-		return connection
-	}
-}

agent/pkg/api/socket_data_streamer.go

@@ -3,12 +3,11 @@ package api
 import (
 	"context"
 	"encoding/json"
-	"time"
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/dependency"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
@@ -34,18 +33,9 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 	meta := make(chan []byte)
 
 	query := params.Query
-	if err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query); err != nil {
-		if err := entryStreamerSocketConnector.SendToastError(socketId, err); err != nil {
-			return err
-		}
-
-		entryStreamerSocketConnector.CleanupSocket(socketId)
-		return err
-	}
-
-	leftOff, err := e.fetch(socketId, params, entryStreamerSocketConnector)
+	err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
 	if err != nil {
-		logger.Log.Errorf("Fetch error: %v", err)
+		entryStreamerSocketConnector.SendToastError(socketId, err)
 	}
 
 	handleDataChannel := func(c *basenine.Connection, data chan []byte) {
@@ -57,15 +47,13 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 			}
 
 			var entry *tapApi.Entry
-			if err = json.Unmarshal(bytes, &entry); err != nil {
-				logger.Log.Debugf("Error unmarshalling entry: %v", err)
+			err = json.Unmarshal(bytes, &entry)
+			if err != nil {
+				logger.Log.Debugf("Error unmarshalling entry: %v", err.Error())
 				continue
 			}
 
-			if err := entryStreamerSocketConnector.SendEntry(socketId, entry, params); err != nil {
-				logger.Log.Errorf("Error sending entry to socket, err: %v", err)
-				return
-			}
+			entryStreamerSocketConnector.SendEntry(socketId, entry, params)
 		}
 	}
 
@@ -78,22 +66,20 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 			}
 
 			var metadata *basenine.Metadata
-			if err = json.Unmarshal(bytes, &metadata); err != nil {
-				logger.Log.Debugf("Error unmarshalling metadata: %v", err)
+			err = json.Unmarshal(bytes, &metadata)
+			if err != nil {
+				logger.Log.Debugf("Error unmarshalling metadata: %v", err.Error())
 				continue
 			}
 
-			if err := entryStreamerSocketConnector.SendMetadata(socketId, metadata); err != nil {
-				logger.Log.Errorf("Error sending metadata to socket, err: %v", err)
-				return
-			}
+			entryStreamerSocketConnector.SendMetadata(socketId, metadata)
 		}
 	}
 
 	go handleDataChannel(connection, data)
 	go handleMetaChannel(connection, meta)
 
-	if err = connection.Query(leftOff, query, data, meta); err != nil {
+	if err = connection.Query(query, data, meta); err != nil {
 		logger.Log.Errorf("Query mode call failed: %v", err)
 		entryStreamerSocketConnector.CleanupSocket(socketId)
 		return err
@@ -108,64 +94,3 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
 
 	return nil
 }
-
-// Reverses a []byte slice.
-func (e *BasenineEntryStreamer) fetch(socketId int, params *WebSocketParams, connector EntryStreamerSocketConnector) (leftOff string, err error) {
-	if params.Fetch <= 0 {
-		leftOff = params.LeftOff
-		return
-	}
-
-	var data [][]byte
-	var firstMeta []byte
-	var lastMeta []byte
-	data, firstMeta, lastMeta, err = basenine.Fetch(
-		shared.BasenineHost,
-		shared.BaseninePort,
-		params.LeftOff,
-		-1,
-		params.Query,
-		params.Fetch,
-		time.Duration(params.TimeoutMs)*time.Millisecond,
-	)
-	if err != nil {
-		return
-	}
-
-	var firstMetadata *basenine.Metadata
-	if err = json.Unmarshal(firstMeta, &firstMetadata); err != nil {
-		return
-	}
-
-	leftOff = firstMetadata.LeftOff
-
-	var lastMetadata *basenine.Metadata
-	if err = json.Unmarshal(lastMeta, &lastMetadata); err != nil {
-		return
-	}
-
-	if err = connector.SendMetadata(socketId, lastMetadata); err != nil {
-		return
-	}
-
-	data = e.reverseBytesSlice(data)
-	for _, row := range data {
-		var entry *tapApi.Entry
-		if err = json.Unmarshal(row, &entry); err != nil {
-			break
-		}
-
-		if err = connector.SendEntry(socketId, entry, params); err != nil {
-			return
-		}
-	}
-	return
-}
-
-// Reverses a []byte slice.
-func (e *BasenineEntryStreamer) reverseBytesSlice(arr [][]byte) (newArr [][]byte) {
-	for i := len(arr) - 1; i >= 0; i-- {
-		newArr = append(newArr, arr[i])
-	}
-	return newArr
-}

agent/pkg/api/socket_routes.go

@@ -10,7 +10,7 @@ import (
 	"github.com/gorilla/websocket"
 	"github.com/up9inc/mizu/agent/pkg/models"
 	"github.com/up9inc/mizu/agent/pkg/utils"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
@@ -34,11 +34,8 @@ type SocketConnection struct {
 }
 
 type WebSocketParams struct {
-	LeftOff           string `json:"leftOff"`
 	Query             string `json:"query"`
 	EnableFullEntries bool   `json:"enableFullEntries"`
-	Fetch             int    `json:"fetch"`
-	TimeoutMs         int    `json:"timeoutMs"`
 }
 
 var (

agent/pkg/api/socket_server_handlers.go

@@ -14,8 +14,8 @@ import (
 
 	tapApi "github.com/up9inc/mizu/tap/api"
 
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 type BrowserClient struct {

agent/pkg/api/utils.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 
 	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func BroadcastTappedPodsStatus() {

agent/pkg/app/main.go

@@ -10,8 +10,7 @@ import (
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/api"
 	"github.com/up9inc/mizu/agent/pkg/utils"
-	"github.com/up9inc/mizu/logger"
-	"github.com/up9inc/mizu/tap/dbgctl"
+	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 	amqpExt "github.com/up9inc/mizu/tap/extensions/amqp"
 	httpExt "github.com/up9inc/mizu/tap/extensions/http"
@@ -25,38 +24,36 @@ var (
 )
 
 func LoadExtensions() {
-	Extensions = make([]*tapApi.Extension, 0)
+	Extensions = make([]*tapApi.Extension, 4)
 	ExtensionsMap = make(map[string]*tapApi.Extension)
 
+	extensionAmqp := &tapApi.Extension{}
+	dissectorAmqp := amqpExt.NewDissector()
+	dissectorAmqp.Register(extensionAmqp)
+	extensionAmqp.Dissector = dissectorAmqp
+	Extensions[0] = extensionAmqp
+	ExtensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
+
 	extensionHttp := &tapApi.Extension{}
 	dissectorHttp := httpExt.NewDissector()
 	dissectorHttp.Register(extensionHttp)
 	extensionHttp.Dissector = dissectorHttp
-	Extensions = append(Extensions, extensionHttp)
+	Extensions[1] = extensionHttp
 	ExtensionsMap[extensionHttp.Protocol.Name] = extensionHttp
 
-	if !dbgctl.MizuTapperDisableNonHttpExtensions {
-		extensionAmqp := &tapApi.Extension{}
-		dissectorAmqp := amqpExt.NewDissector()
-		dissectorAmqp.Register(extensionAmqp)
-		extensionAmqp.Dissector = dissectorAmqp
-		Extensions = append(Extensions, extensionAmqp)
-		ExtensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
+	extensionKafka := &tapApi.Extension{}
+	dissectorKafka := kafkaExt.NewDissector()
+	dissectorKafka.Register(extensionKafka)
+	extensionKafka.Dissector = dissectorKafka
+	Extensions[2] = extensionKafka
+	ExtensionsMap[extensionKafka.Protocol.Name] = extensionKafka
 
-		extensionKafka := &tapApi.Extension{}
-		dissectorKafka := kafkaExt.NewDissector()
-		dissectorKafka.Register(extensionKafka)
-		extensionKafka.Dissector = dissectorKafka
-		Extensions = append(Extensions, extensionKafka)
-		ExtensionsMap[extensionKafka.Protocol.Name] = extensionKafka
-
-		extensionRedis := &tapApi.Extension{}
-		dissectorRedis := redisExt.NewDissector()
-		dissectorRedis.Register(extensionRedis)
-		extensionRedis.Dissector = dissectorRedis
-		Extensions = append(Extensions, extensionRedis)
-		ExtensionsMap[extensionRedis.Protocol.Name] = extensionRedis
-	}
+	extensionRedis := &tapApi.Extension{}
+	dissectorRedis := redisExt.NewDissector()
+	dissectorRedis.Register(extensionRedis)
+	extensionRedis.Dissector = dissectorRedis
+	Extensions[3] = extensionRedis
+	ExtensionsMap[extensionRedis.Protocol.Name] = extensionRedis
 
 	sort.Slice(Extensions, func(i, j int) bool {
 		return Extensions[i].Protocol.Priority < Extensions[j].Protocol.Priority

agent/pkg/controllers/entries_controller.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func HandleEntriesError(c *gin.Context, err error) bool {

agent/pkg/controllers/oas_controller.go

@@ -7,7 +7,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/up9inc/mizu/agent/pkg/dependency"
 	"github.com/up9inc/mizu/agent/pkg/oas"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func GetOASServers(c *gin.Context) {

agent/pkg/controllers/status_controller.go

@@ -13,9 +13,9 @@ import (
 	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
 	"github.com/up9inc/mizu/agent/pkg/up9"
 	"github.com/up9inc/mizu/agent/pkg/validation"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func HealthCheck(c *gin.Context) {

agent/pkg/dependency/type_names.go

@@ -5,7 +5,6 @@ type DependencyContainerType string
 const (
 	ServiceMapGeneratorDependency = "ServiceMapGeneratorDependency"
 	OasGeneratorDependency        = "OasGeneratorDependency"
-	EntriesInserter               = "EntriesInserter"
 	EntriesProvider               = "EntriesProvider"
 	EntriesSocketStreamer         = "EntriesSocketStreamer"
 	EntryStreamerSocketConnector  = "EntryStreamerSocketConnector"

agent/pkg/elastic/esClient.go

@@ -0,0 +1,116 @@
+package elastic
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/tap/api"
+)
+
+type client struct {
+	es            *elasticsearch.Client
+	index         string
+	insertedCount int
+}
+
+var instance *client
+var once sync.Once
+
+func GetInstance() *client {
+	once.Do(func() {
+		instance = newClient()
+	})
+	return instance
+}
+
+func (client *client) Configure(config shared.ElasticConfig) {
+	if config.Url == "" || config.User == "" || config.Password == "" {
+		if client.es != nil {
+			client.es = nil
+		}
+		logger.Log.Infof("No elastic configuration was supplied, elastic exporter disabled")
+		return
+	}
+	transport := http.DefaultTransport
+	tlsClientConfig := &tls.Config{InsecureSkipVerify: true}
+	transport.(*http.Transport).TLSClientConfig = tlsClientConfig
+	cfg := elasticsearch.Config{
+		Addresses: []string{config.Url},
+		Username:  config.User,
+		Password:  config.Password,
+		Transport: transport,
+	}
+
+	es, err := elasticsearch.NewClient(cfg)
+	if err != nil {
+		logger.Log.Errorf("Failed to initialize elastic client %v", err)
+	}
+
+	// Have the client instance return a response
+	res, err := es.Info()
+	if err != nil {
+		logger.Log.Errorf("Elastic client.Info() ERROR: %v", err)
+	} else {
+		client.es = es
+		client.index = "mizu_traffic_http_" + time.Now().Format("2006_01_02_15_04")
+		client.insertedCount = 0
+		logger.Log.Infof("Elastic client configured, index: %s, cluster info: %v", client.index, res)
+	}
+	defer res.Body.Close()
+}
+
+func newClient() *client {
+	return &client{
+		es:    nil,
+		index: "",
+	}
+}
+
+type httpEntry struct {
+	Source      *api.TCP               `json:"src"`
+	Destination *api.TCP               `json:"dst"`
+	Outgoing    bool                   `json:"outgoing"`
+	CreatedAt   time.Time              `json:"createdAt"`
+	Request     map[string]interface{} `json:"request"`
+	Response    map[string]interface{} `json:"response"`
+	ElapsedTime int64                  `json:"elapsedTime"`
+}
+
+func (client *client) PushEntry(entry *api.Entry) {
+	if client.es == nil {
+		return
+	}
+
+	if entry.Protocol.Name != "http" {
+		return
+	}
+
+	entryToPush := httpEntry{
+		Source:      entry.Source,
+		Destination: entry.Destination,
+		Outgoing:    entry.Outgoing,
+		CreatedAt:   entry.StartTime,
+		Request:     entry.Request,
+		Response:    entry.Response,
+		ElapsedTime: entry.ElapsedTime,
+	}
+
+	entryJson, err := json.Marshal(entryToPush)
+	if err != nil {
+		logger.Log.Errorf("json.Marshal ERROR: %v", err)
+		return
+	}
+	var buffer bytes.Buffer
+	buffer.WriteString(string(entryJson))
+	res, _ := client.es.Index(client.index, &buffer)
+	if res.StatusCode == 201 {
+		client.insertedCount += 1
+	}
+}

agent/pkg/entries/entries_provider.go

@@ -9,8 +9,8 @@ import (
 	"github.com/up9inc/mizu/agent/pkg/app"
 	"github.com/up9inc/mizu/agent/pkg/har"
 	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
@@ -22,7 +22,7 @@ type EntriesProvider interface {
 type BasenineEntriesProvider struct{}
 
 func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error) {
-	data, _, lastMeta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
+	data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
 		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
 		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
 	if err != nil {
@@ -49,7 +49,7 @@ func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesReque
 	}
 
 	var metadata *basenine.Metadata
-	err = json.Unmarshal(lastMeta, &metadata)
+	err = json.Unmarshal(meta, &metadata)
 	if err != nil {
 		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
 	}

agent/pkg/har/types.go

@@ -2,10 +2,9 @@ package har
 
 import (
 	"encoding/base64"
+	"github.com/up9inc/mizu/shared/logger"
 	"time"
 	"unicode/utf8"
-
-	"github.com/up9inc/mizu/logger"
 )
 
 /*

agent/pkg/har/utils.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 // Keep it because we might want cookies in the future
@@ -124,8 +124,8 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
 
 	postData, _ := request["postData"].(map[string]interface{})
 	mimeType := postData["mimeType"]
-	if mimeType == nil {
-		mimeType = ""
+	if mimeType == nil || len(mimeType.(string)) == 0 {
+		mimeType = "text/html"
 	}
 	text := postData["text"]
 	postDataText := ""
@@ -177,8 +177,8 @@ func NewResponse(response map[string]interface{}) (harResponse *Response, err er
 
 	content, _ := response["content"].(map[string]interface{})
 	mimeType := content["mimeType"]
-	if mimeType == nil {
-		mimeType = ""
+	if mimeType == nil || len(mimeType.(string)) == 0 {
+		mimeType = "text/html"
 	}
 	encoding := content["encoding"]
 	text := content["text"]

agent/pkg/oas/feeder_test.go

@@ -16,7 +16,7 @@ import (
 
 	"github.com/up9inc/mizu/agent/pkg/har"
 
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func getFiles(baseDir string) (result []string, err error) {

agent/pkg/oas/oas_generator.go

@@ -11,7 +11,7 @@ import (
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap/api"
 
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var (
@@ -104,7 +104,7 @@ func (g *defaultOasGenerator) runGenerator() {
 	g.dbMutex.Lock()
 	defer g.dbMutex.Unlock()
 	logger.Log.Infof("Querying DB for OAS generator with query '%s'", g.entriesQuery)
-	if err := g.dbConn.Query("latest", g.entriesQuery, dataChan, metaChan); err != nil {
+	if err := g.dbConn.Query(g.entriesQuery, dataChan, metaChan); err != nil {
 		logger.Log.Errorf("Query mode call failed: %v", err)
 	}
 

agent/pkg/oas/specgen.go

@@ -17,7 +17,7 @@ import (
 	"github.com/chanced/openapi"
 	"github.com/google/uuid"
 	"github.com/nav-inc/datetime"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 
 	"github.com/up9inc/mizu/agent/pkg/har"
 

agent/pkg/oas/specgen_test.go

@@ -13,7 +13,7 @@ import (
 	"time"
 
 	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 	"github.com/wI2L/jsondiff"
 
 	basenine "github.com/up9inc/basenine/client/go"

agent/pkg/oas/tree.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 
 	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 type NodePath = []string

agent/pkg/oas/utils.go

@@ -9,7 +9,7 @@ import (
 	"github.com/up9inc/mizu/agent/pkg/har"
 
 	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func exampleResolver(ref string) (*openapi.ExampleObj, error) {

agent/pkg/providers/tappedPods/tapped_pods_provider.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
 	"github.com/up9inc/mizu/agent/pkg/utils"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 const FilePath = shared.DataDirPath + "tapped-pods.json"

agent/pkg/providers/tappers/tappers_provider.go

@@ -5,8 +5,8 @@ import (
 	"sync"
 
 	"github.com/up9inc/mizu/agent/pkg/utils"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 const FilePath = shared.DataDirPath + "tappers-status.json"

agent/pkg/resolver/go.sum

@@ -219,6 +219,8 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

agent/pkg/resolver/resolver.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 
 	cmap "github.com/orcaman/concurrent-map"

agent/pkg/rules/rulesHTTP.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/up9inc/mizu/agent/pkg/har"
 
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 
 	"github.com/up9inc/mizu/shared"
 	"github.com/yalp/jsonpath"

agent/pkg/servicemap/servicemap.go

@@ -1,11 +1,10 @@
 package servicemap
 
 import (
+	"github.com/jinzhu/copier"
 	"sync"
 
-	"github.com/jinzhu/copier"
-
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 

agent/pkg/up9/main.go

@@ -17,8 +17,8 @@ import (
 	"github.com/up9inc/mizu/agent/pkg/utils"
 
 	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
@@ -327,7 +327,7 @@ BasenineReconnect:
 	go handleMetaChannel(&wg, connection, meta)
 	wg.Add(2)
 
-	if err = connection.Query("latest", query, data, meta); err != nil {
+	if err = connection.Query(query, data, meta); err != nil {
 		logger.Log.Errorf("Query mode call failed: %v", err)
 		connection.Close()
 		time.Sleep(shared.BasenineReconnectInterval * time.Second)

agent/pkg/utils/utils.go

@@ -13,8 +13,8 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var (

cli/apiserver/provider.go

@@ -4,16 +4,15 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"github.com/up9inc/mizu/cli/utils"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"time"
 
-	"github.com/up9inc/mizu/cli/utils"
-
 	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 	core "k8s.io/api/core/v1"
 )
 

cli/auth/authProvider.go

@@ -12,7 +12,7 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 	"golang.org/x/oauth2"
 )
 

cli/cmd/check.go

@@ -5,7 +5,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/telemetry"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var checkCmd = &cobra.Command{

cli/cmd/check/imagePullInCluster.go

@@ -3,14 +3,13 @@ package check
 import (
 	"context"
 	"fmt"
-	"regexp"
-	"time"
-
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
 	core "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"regexp"
+	"time"
 )
 
 func ImagePullInCluster(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {

cli/cmd/check/kubernetesApi.go

@@ -2,14 +2,14 @@ package check
 
 import (
 	"fmt"
-
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/semver"
 )
 
+
 func KubernetesApi() (*kubernetes.Provider, *semver.SemVersion, bool) {
 	logger.Log.Infof("\nkubernetes-api\n--------------------")
 

cli/cmd/check/kubernetesPermissions.go

@@ -4,16 +4,15 @@ import (
 	"context"
 	"embed"
 	"fmt"
-	"strings"
-
 	"github.com/up9inc/mizu/cli/bucket"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
 	rbac "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/scheme"
+	"strings"
 )
 
 func TapKubernetesPermissions(ctx context.Context, embedFS embed.FS, kubernetesProvider *kubernetes.Provider) bool {

cli/cmd/check/kubernetesResources.go

@@ -3,11 +3,10 @@ package check
 import (
 	"context"
 	"fmt"
-
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func KubernetesResources(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {

cli/cmd/check/kubernetesVersion.go

@@ -2,10 +2,9 @@ package check
 
 import (
 	"fmt"
-
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/semver"
 )
 

cli/cmd/check/serverConnection.go

@@ -3,13 +3,12 @@ package check
 import (
 	"context"
 	"fmt"
-	"regexp"
-
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
+	"regexp"
 )
 
 func ServerConnection(kubernetesProvider *kubernetes.Provider) bool {

cli/cmd/checkRunner.go

@@ -4,11 +4,10 @@ import (
 	"context"
 	"embed"
 	"fmt"
-
 	"github.com/up9inc/mizu/cli/cmd/check"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var (

cli/cmd/common.go

@@ -19,8 +19,8 @@ import (
 	"github.com/up9inc/mizu/shared"
 
 	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func GetApiServerUrl(port uint16) string {

cli/cmd/config.go

@@ -9,7 +9,7 @@ import (
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/telemetry"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var configCmd = &cobra.Command{

cli/cmd/goUtils/funcWrappers.go

@@ -4,7 +4,7 @@ import (
 	"reflect"
 	"runtime/debug"
 
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func HandleExcWrapper(fn interface{}, params ...interface{}) (result []reflect.Value) {

cli/cmd/installRunner.go

@@ -2,10 +2,9 @@ package cmd
 
 import (
 	"fmt"
-
 	"github.com/up9inc/mizu/cli/bucket"
 	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func runMizuInstall() {

cli/cmd/logs.go

@@ -10,7 +10,7 @@ import (
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/telemetry"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var logsCmd = &cobra.Command{

cli/cmd/root.go

@@ -11,7 +11,7 @@ import (
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/mizu/version"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var rootCmd = &cobra.Command{

cli/cmd/tap.go

@@ -14,8 +14,8 @@ import (
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 const uploadTrafficMessageToConfirm = `NOTE: running mizu with --%s flag will upload recorded traffic for further analysis and enriched presentation options.`
@@ -123,5 +123,4 @@ func init() {
 	tapCmd.Flags().String(configStructs.ContractFile, defaultTapConfig.ContractFile, "OAS/Swagger file to validate to monitor the contracts")
 	tapCmd.Flags().Bool(configStructs.ServiceMeshName, defaultTapConfig.ServiceMesh, "Record decrypted traffic if the cluster is configured with a service mesh and with mtls")
 	tapCmd.Flags().Bool(configStructs.TlsName, defaultTapConfig.Tls, "Record tls traffic")
-	tapCmd.Flags().Bool(configStructs.ProfilerName, defaultTapConfig.Profiler, "Run pprof server")
 }

cli/cmd/tapRunner.go

@@ -25,9 +25,9 @@ import (
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap/api"
 )
 
@@ -124,7 +124,7 @@ func RunMizuTap() {
 	}
 
 	logger.Log.Infof("Waiting for Mizu Agent to start...")
-	if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel(), config.Config.Tap.Profiler); err != nil {
+	if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel()); err != nil {
 		var statusError *k8serrors.StatusError
 		if errors.As(err, &statusError) && (statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists) {
 			logger.Log.Info("Mizu is already running in this namespace, change the `mizu-resources-namespace` configuration or run `mizu clean` to remove the currently running Mizu instance")
@@ -164,6 +164,7 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 		ServiceMap:             config.Config.ServiceMap,
 		OAS:                    config.Config.OAS,
 		Telemetry:              config.Config.Telemetry,
+		Elastic:                config.Config.Elastic,
 	}
 
 	return &mizuAgentConfig

cli/cmd/version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/telemetry"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"

cli/cmd/view.go

@@ -6,7 +6,7 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/telemetry"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var viewCmd = &cobra.Command{

cli/cmd/viewRunner.go

@@ -11,8 +11,8 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func runMizuView() {

cli/config/config.go

@@ -9,8 +9,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"

cli/config/configStruct.go

@@ -41,6 +41,7 @@ type ConfigStruct struct {
 	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
 	ServiceMap             bool                        `yaml:"service-map" default:"true"`
 	OAS                    bool                        `yaml:"oas" default:"true"`
+	Elastic                shared.ElasticConfig        `yaml:"elastic"`
 }
 
 func (config *ConfigStruct) validate() error {

cli/config/configStructs/tapConfig.go

@@ -11,7 +11,7 @@ import (
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/units"
 )
 
@@ -30,7 +30,6 @@ const (
 	ContractFile                  = "contract"
 	ServiceMeshName               = "service-mesh"
 	TlsName                       = "tls"
-	ProfilerName                  = "profiler"
 )
 
 type TapConfig struct {
@@ -55,7 +54,6 @@ type TapConfig struct {
 	TapperResources        shared.Resources `yaml:"tapper-resources"`
 	ServiceMesh            bool             `yaml:"service-mesh" default:"false"`
 	Tls                    bool             `yaml:"tls" default:"false"`
-	Profiler               bool             `yaml:"profiler" default:"false"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {

cli/go.mod

@@ -11,7 +11,6 @@ require (
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.3.0
 	github.com/spf13/pflag v1.0.5
-	github.com/up9inc/mizu/logger v0.0.0
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
@@ -74,7 +73,6 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday v1.6.0 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
-	github.com/up9inc/mizu/tap/dbgctl v0.0.0 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect
 	go.starlark.net v0.0.0-20220203230714-bb14e151c28f // indirect
 	golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab // indirect
@@ -100,10 +98,6 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace github.com/up9inc/mizu/logger v0.0.0 => ../logger
-
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
-
-replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../tap/dbgctl

cli/mizu/fsUtils/mizuLogsUtils.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func GetLogFilePath() string {

cli/mizu/fsUtils/zipUtils.go

@@ -8,7 +8,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func AddFileToZip(zipWriter *zip.Writer, filename string) error {

cli/mizu/version/versionCheck.go

@@ -13,7 +13,7 @@ import (
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/cli/pkg/version"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 
 	"github.com/google/go-github/v37/github"
 	"github.com/up9inc/mizu/cli/uiUtils"

cli/resources/cleanResources.go

@@ -3,13 +3,12 @@ package resources
 import (
 	"context"
 	"fmt"
-
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/cli/utils"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
@@ -55,6 +54,7 @@ func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, ku
 		}
 	}
 
+
 	if resources, err := kubernetesProvider.ListManagedClusterRoleBindings(ctx); err != nil {
 		resourceDesc := "ClusterRoleBindings"
 		handleDeletionError(err, resourceDesc, &leftoverResources)

cli/resources/createResources.go

@@ -8,13 +8,13 @@ import (
 	"github.com/up9inc/mizu/cli/errormessage"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/kubernetes"
+	"github.com/up9inc/mizu/shared/logger"
 	core "k8s.io/api/core/v1"
 )
 
-func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, profiler bool) (bool, error) {
+func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level) (bool, error) {
 	if !isNsRestrictedMode {
 		if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil {
 			return false, err
@@ -50,7 +50,6 @@ func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.
 		Resources:             apiServerResources,
 		ImagePullPolicy:       imagePullPolicy,
 		LogLevel:              logLevel,
-		Profiler:              profiler,
 	}
 
 	if err := createMizuApiServerPod(ctx, kubernetesProvider, opts); err != nil {

cli/telemetry/telemetry.go

@@ -4,15 +4,14 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"net/http"
-	"os"
-	"time"
-
 	"github.com/denisbrodbeck/machineid"
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/mizu"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
+	"net/http"
+	"os"
+	"time"
 )
 
 const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"

cli/uiUtils/confirmation.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func AskForConfirmation(s string) bool {

cli/uiUtils/openBrowser.go

@@ -5,7 +5,7 @@ import (
 	"os/exec"
 	"runtime"
 
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func OpenBrowser(url string) {

cli/utils/waitUtils.go

@@ -2,11 +2,10 @@ package utils
 
 import (
 	"context"
+	"github.com/up9inc/mizu/shared/logger"
 	"os"
 	"os/signal"
 	"syscall"
-
-	"github.com/up9inc/mizu/logger"
 )
 
 func WaitForFinish(ctx context.Context, cancel context.CancelFunc) {

devops/linux-x86_64-musl-go-libpcap/Dockerfile

@@ -1,31 +0,0 @@
-FROM messense/rust-musl-cross:x86_64-musl AS builder-from-arm64v8-to-amd64
-
-ENV CROSS_TRIPLE x86_64-unknown-linux-musl
-ENV CROSS_ROOT /usr/local/musl
-
-ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
-    AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
-    CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
-    CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
-    CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
-    LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld \
-    FC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gfortran
-
-# Install Go
-WORKDIR /
-RUN curl https://go.dev/dl/go1.17.6.linux-arm64.tar.gz -Lo ./go.linux-arm64.tar.gz \
- && curl https://go.dev/dl/go1.17.6.linux-arm64.tar.gz.asc -Lo ./go.linux-arm64.tar.gz.asc \
- && curl https://dl.google.com/dl/linux/linux_signing_key.pub  -Lo linux_signing_key.pub \
- && gpg --import linux_signing_key.pub && gpg --verify ./go.linux-arm64.tar.gz.asc ./go.linux-arm64.tar.gz \
- && rm -rf /usr/local/go && tar -C /usr/local -xzf go.linux-arm64.tar.gz
-ENV PATH "$PATH:/usr/local/go/bin"
-
-# Compile libpcap from source
-RUN curl https://www.tcpdump.org/release/libpcap-1.10.1.tar.gz -Lo ./libpcap.tar.gz \
- && curl https://www.tcpdump.org/release/libpcap-1.10.1.tar.gz.sig -Lo ./libpcap.tar.gz.sig \
- && curl https://www.tcpdump.org/release/signing-key.asc -Lo ./signing-key.asc \
- && gpg --import signing-key.asc && gpg --verify libpcap.tar.gz.sig libpcap.tar.gz \
- && tar -xzf libpcap.tar.gz && mv ./libpcap-* ./libpcap
-WORKDIR /libpcap
-RUN ./configure --host=x86_64 && make \
- && cp /libpcap/libpcap.a /usr/local/musl/lib/gcc/x86_64-unknown-linux-musl/*/

devops/linux-x86_64-musl-go-libpcap/build-push.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-set -e
-
-docker build . -t up9inc/linux-x86_64-musl-go-libpcap && docker push up9inc/linux-x86_64-musl-go-libpcap

devops/ui-common-pack.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# exit when any command fails
-set -e
-
-dst_folder=$1
-echo "dst folder: $dst_folder";
-
-cd $dst_folder/../ui-common
-npm i
-npm pack
-mv up9-mizu-common-0.0.0.tgz $dst_folder

logger/go.mod

@@ -1,5 +0,0 @@
-module github.com/up9inc/mizu/logger
-
-go 1.17
-
-require github.com/op/go-logging v0.0.0-20160315200505-970db520ece7

logger/go.sum

@@ -1,2 +0,0 @@
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=

performance_analysis/README.md

@@ -1,107 +0,0 @@
-
-# Performance analysis
-
-This directory contains tools for analyzing tapper performance.
-
-# Periodic tapper logs
-
-In tapper logs there are some periodic lines that shows its internal state and consumed resources.
-
-Internal state example (formatted and commented):
-```
-stats - {
-	"processedBytes":468940592, // how many bytes we read from pcap
-	"packetsCount":174883, // how many packets we read from pcap
-	"tcpPacketsCount":174883, // how many tcp packets we read from pcap
-	"reassembledTcpPayloadsCount":66893, // how many chunks sent to tcp stream
-	"matchedPairs":24821, // how many request response pairs found
-	"droppedTcpStreams":2 // how many tcp streams remained stale and dropped
-}
-```
-
-Consumed resources example (formatted and commented):
-```
-mem: 24441240, // golang heap size
-goroutines: 29, // how many goroutines
-cpu: 91.208791, // how much cpu the tapper process consume (in percentage per core)
-cores: 16,  // how many cores there are on the machine
-rss: 87052288 // how many bytes held by the tapper process
-```
-
-# Plot tapper logs
-
-In order to plot a tapper log or many logs into a graph, use the `plot_from_tapper_logs.py` util.
-
-It gets a list of tapper logs as a parameter, and output an image with a nice graph.
-
-The log file names should be named in this format `XX_DESCRIPTION.log` when XX is the number between determining the color of the output graph and description is the name of the series. It allows for easy comparison between various modes.
-
-Example run:
-```
-cd $MIZU_HOME/performance_analysis
-virtualenv venv
-source venv/bin/activate
-pip install -r requirements.txt
-python plot_from_tapper_logs.py 00_tapper.log
-```
-
-# Tapper Modes
-
-Every packet seen by the tapper is processed in a pipeline that contains various stages. 
-* Pcap - Read the packet from libpcap
-* Assembler - Assemble the packet into a TcpStream
-* TcpStream - Hold stream information and TcpReaders
-* Dissectors - Read from TcpReader and recognize the packet content and protocol.
-* Emit - Marshal the request response pair into a Json
-* Send - Send the Json to Api Server
-
-Tapper can be run with various debug modes:
-* No Pcap - Start the tapper process, but don't read from any packets from pcap
-* No Assembler - Read packets from pcap, but don't assemble them
-* No TcpStream - Assemble the packets, but don't create TcpStream for them
-* No Dissectors - Create a TcpStream for the packets, but don't dissect their content
-* No Emit - Dissect the TcpStream, but don't emit the matched request response pair 
-* No Send - Emit the request response pair, but don't send them to the Api Server.
-* Regular mode
-
-![Tapper Modes](https://github.com/up9inc/mizu/blob/debug/profile-tapper-benchmark/performance_analysis/tapper-modes.png)
-
-# Run benchmark with various tapper modes
-
-## Prerequisite
-
-In order to run the benchmark you probably want:
-1. An up and running Api Server
-2. An up and running Basenine
-3. An up and running UI (optional)
-4. An up and running test server, like nginx, that can return a known payload at a known endpoint.
-5. Set MIZU_HOME environment variable to points to mizu directory
-6. Install the `hey` tool
-
-## Running the benchmark
-
-In order to run a benchmark use the `run_tapper_benchmark.sh` script.
-
-Example run:
-```
-cd $MIZU_HOME/performance_analysis
-source venv/bin/activate # Assuming you already run plot_from_tapper_logs.py 
-./run_tapper_benchmark.sh
-```
-
-Running it without params use the default values, use the following environment variables for customization:
-```
-export=MIZU_BENCHMARK_OUTPUT_DIR=/path/to/dir # Set the output directory for tapper logs and graph
-export=MIZU_BENCHMARK_CLIENT_PERIOD=1m # How long each test run
-export=MIZU_BENCHMARK_URL=http://server:port/path # The URL to use for the benchmarking process (the test server endpoint)
-export=MIZU_BENCHMARK_RUN_COUNT=3 # How many times each tapper mode should run
-export=MIZU_BENCHMARK_QPS=250 # How many queries per second the each client should send to the test server
-export=MIZU_BENCHMARK_CLIENTS_COUNT=5 # How many clients should run in parallel during the benchmark
-```
-
-# Example output graph
-
-An example output graph from a 15 min run with 15K payload and 1000 QPS looks like
-
-![Example Graph](https://github.com/up9inc/mizu/blob/debug/profile-tapper-benchmark/performance_analysis/example-graph.png)
-

performance_analysis/plot_from_tapper_logs.py

@@ -1,182 +0,0 @@
-import matplotlib.pyplot as plt
-import numpy as np
-import pandas as pd
-import pathlib
-import re
-import sys
-import typing
-
-COLORMAP = plt.get_cmap('turbo')
-
-# Extract cpu and rss samples from log files and plot them
-# Input: List of log files
-#
-# example:
-#   python plot_from_tapper_logs.py 01_no_pcap_01.log 99_normal_00.log
-#
-# The script assumes that the log file names start with a number (pattern '\d+')
-# and groups based on this number. Files that start will the same number will be plotted with the same color.
-# Change group_pattern to an empty string to disable this, or change to a regex of your liking.
-
-
-def get_sample(name: str, line: str, default_value: float):
-    pattern = name + r': ?(\d+(\.\d+)?)'
-    maybe_sample = re.findall(pattern, line)
-    if len(maybe_sample) == 0:
-        return default_value
-
-    sample = float(maybe_sample[0][0])
-    return sample
-
-
-def append_sample(name: str, line: str, samples: typing.List[float]):
-    sample = get_sample(name, line, -1)
-
-    if sample == -1:
-        return
-
-    samples.append(sample)
-
-
-def extract_samples(f: typing.IO) -> typing.Tuple[pd.Series, pd.Series, pd.Series, pd.Series, pd.Series, pd.Series, pd.Series, pd.Series]:
-    cpu_samples = []
-    rss_samples = []
-    count_samples = []
-    matched_samples = []
-    live_samples = []
-    processed_samples = []
-    heap_samples = []
-    goroutines_samples = []
-    for line in f:
-        append_sample('cpu', line, cpu_samples)
-        append_sample('rss', line, rss_samples)
-        ignored_packets_count = get_sample('"ignoredPacketsCount"', line, -1)
-        packets_count = get_sample('"packetsCount"', line, -1)
-        if ignored_packets_count != -1 and packets_count != -1:
-            count_samples.append(packets_count - ignored_packets_count)
-        append_sample('"matchedPairs"', line, matched_samples)
-        append_sample('"liveTcpStreams"', line, live_samples)
-        append_sample('"processedBytes"', line, processed_samples)
-        append_sample('mem', line, heap_samples)
-        append_sample('goroutines', line, goroutines_samples)
-
-    cpu_samples = pd.Series(cpu_samples)
-    rss_samples = pd.Series(rss_samples)
-    count_samples = pd.Series(count_samples)
-    matched_samples = pd.Series(matched_samples)
-    live_samples = pd.Series(live_samples)
-    processed_samples = pd.Series(processed_samples)
-    heap_samples = pd.Series(heap_samples)
-    goroutines_samples = pd.Series(goroutines_samples)
-
-    return cpu_samples, rss_samples, count_samples, matched_samples, live_samples, processed_samples, heap_samples, goroutines_samples
-
-
-def plot(ax, df: pd.DataFrame, title: str, xlabel: str, ylabel: str, group_pattern: typing.Optional[str]):
-    if group_pattern:
-        color = get_group_color(df.columns, group_pattern)
-        df.plot(color=color, ax=ax)
-    else:
-        df.plot(cmap=COLORMAP, ax=ax)
-
-    ax.ticklabel_format(style='plain')
-    plt.title(title)
-    plt.legend()
-    plt.xlabel(xlabel)
-    plt.ylabel(ylabel)
-
-
-def get_group_color(names, pattern):
-    props = [int(re.findall(pattern, pathlib.Path(name).name)[0]) for name in names]
-    key = dict(zip(sorted(list(set(props))), range(len(set(props)))))
-    n_colors = len(key)
-    color_options = plt.get_cmap('jet')(np.linspace(0, 1, n_colors))
-    groups = [key[prop] for prop in props]
-    color = color_options[groups]  # type: ignore
-    return color
-
-
-if __name__ == '__main__':
-    filenames = sys.argv[1:]
-
-    cpu_samples_all_files = []
-    rss_samples_all_files = []
-    count_samples_all_files = []
-    matched_samples_all_files = []
-    live_samples_all_files = []
-    processed_samples_all_files = []
-    heap_samples_all_files = []
-    goroutines_samples_all_files = []
-
-    for ii, filename in enumerate(filenames):
-        print("Analyzing {}".format(filename))
-        with open(filename, 'r') as f:
-            cpu_samples, rss_samples, count_samples, matched_samples, live_samples, processed_samples, heap_samples, goroutines_samples = extract_samples(f)
-
-        cpu_samples.name = pathlib.Path(filename).name
-        rss_samples.name = pathlib.Path(filename).name
-        count_samples.name = pathlib.Path(filename).name
-        matched_samples.name = pathlib.Path(filename).name
-        live_samples.name = pathlib.Path(filename).name
-        processed_samples.name = pathlib.Path(filename).name
-        heap_samples.name = pathlib.Path(filename).name
-        goroutines_samples.name = pathlib.Path(filename).name
-
-        cpu_samples_all_files.append(cpu_samples)
-        rss_samples_all_files.append(rss_samples)
-        count_samples_all_files.append(count_samples)
-        matched_samples_all_files.append(matched_samples)
-        live_samples_all_files.append(live_samples)
-        processed_samples_all_files.append(processed_samples)
-        heap_samples_all_files.append(heap_samples)
-        goroutines_samples_all_files.append(goroutines_samples)
-
-    cpu_samples_df = pd.concat(cpu_samples_all_files, axis=1)
-    rss_samples_df = pd.concat(rss_samples_all_files, axis=1)
-    count_samples_df = pd.concat(count_samples_all_files, axis=1)
-    matched_samples_df = pd.concat(matched_samples_all_files, axis=1)
-    live_samples_df = pd.concat(live_samples_all_files, axis=1)
-    processed_samples_df = pd.concat(processed_samples_all_files, axis=1)
-    heap_samples_df = pd.concat(heap_samples_all_files, axis=1)
-    goroutines_samples_df = pd.concat(goroutines_samples_all_files, axis=1)
-
-    group_pattern = r'^\d+'
-
-    cpu_plot = plt.subplot(8, 2, 1)
-    plot(cpu_plot, cpu_samples_df, 'cpu', '', 'cpu (%)', group_pattern)
-    cpu_plot.legend().remove()
-
-    mem_plot = plt.subplot(8, 2, 2)
-    plot(mem_plot, (rss_samples_df / 1024 / 1024), 'rss', '', 'mem (mega)', group_pattern)
-    mem_plot.legend(loc='center left', bbox_to_anchor=(1, 0.5))
-
-    packets_plot = plt.subplot(8, 2, 3)
-    plot(packets_plot, count_samples_df, 'packetsCount', '', 'packetsCount', group_pattern)
-    packets_plot.legend().remove()
-
-    matched_plot = plt.subplot(8, 2, 4)
-    plot(matched_plot, matched_samples_df, 'matchedCount', '', 'matchedCount', group_pattern)
-    matched_plot.legend().remove()
-
-    live_plot = plt.subplot(8, 2, 5)
-    plot(live_plot, live_samples_df, 'liveStreamsCount', '', 'liveStreamsCount', group_pattern)
-    live_plot.legend().remove()
-
-    processed_plot = plt.subplot(8, 2, 6)
-    plot(processed_plot, (processed_samples_df / 1024 / 1024), 'processedBytes', '', 'bytes (mega)', group_pattern)
-    processed_plot.legend().remove()
-
-    heap_plot = plt.subplot(8, 2, 7)
-    plot(heap_plot, (heap_samples_df / 1024 / 1024), 'heap', '', 'heap (mega)', group_pattern)
-    heap_plot.legend().remove()
-
-    goroutines_plot = plt.subplot(8, 2, 8)
-    plot(goroutines_plot, goroutines_samples_df, 'goroutines', '', 'goroutines', group_pattern)
-    goroutines_plot.legend().remove()
-
-    fig = plt.gcf()
-    fig.set_size_inches(20, 18)
-
-    print('Saving graph to graph.png')
-    plt.savefig('graph.png', bbox_inches='tight')
-    

performance_analysis/requirements.txt

@@ -1,2 +0,0 @@
-matplotlib
-pandas

performance_analysis/run_tapper_benchmark.sh

@@ -1,99 +0,0 @@
-#!/bin/bash
-
-[ -z "$MIZU_HOME" ] && { echo "MIZU_HOME is missing"; exit 1; }
-[ -z "$MIZU_BENCHMARK_OUTPUT_DIR" ] && export MIZU_BENCHMARK_OUTPUT_DIR="/tmp/mizu-benchmark-results-$(date +%d-%m-%H-%M)"
-[ -z "$MIZU_BENCHMARK_CLIENT_PERIOD" ] && export MIZU_BENCHMARK_CLIENT_PERIOD="1m"
-[ -z "$MIZU_BENCHMARK_URL" ] && export MIZU_BENCHMARK_URL="http://localhost:8081/data/b.1000.json"
-[ -z "$MIZU_BENCHMARK_RUN_COUNT" ] && export MIZU_BENCHMARK_RUN_COUNT="3"
-[ -z "$MIZU_BENCHMARK_QPS" ] && export MIZU_BENCHMARK_QPS="500"
-[ -z "$MIZU_BENCHMARK_CLIENTS_COUNT" ] && export MIZU_BENCHMARK_CLIENTS_COUNT="5"
-
-function log() {
-	local message=$@
-	printf "[%s] %s\n" "$(date "+%d-%m %H:%M:%S")" "$message"
-}
-
-function run_single_bench() {
-	local mode_num=$1
-	local mode_str=$2
-
-	log "Starting ${mode_num}_${mode_str} (runs: $MIZU_BENCHMARK_RUN_COUNT) (period: $MIZU_BENCHMARK_CLIENT_PERIOD)"
-
-	for ((i=0;i<"$MIZU_BENCHMARK_RUN_COUNT";i++)); do
-		log "  $i: Running tapper"
-		rm -f tapper.log
-		tapper_args=("--tap" "--api-server-address" "ws://localhost:8899/wsTapper" "-stats" "10")
-		if [[ $(uname) == "Darwin" ]]
-		then
-			tapper_args+=("-i" "lo0" "-"decoder "Loopback")
-		else
-			tapper_args+=("-i" "lo")
-		fi
-		nohup ./agent/build/mizuagent ${tapper_args[@]} > tapper.log 2>&1 &
-
-		log "  $i: Running client (hey)"
-		hey -z $MIZU_BENCHMARK_CLIENT_PERIOD -c $MIZU_BENCHMARK_CLIENTS_COUNT -q $MIZU_BENCHMARK_QPS $MIZU_BENCHMARK_URL > /dev/null || return 1
-
-		log "  $i: Killing tapper"
-		kill -9 $(ps -ef | grep agent/build/mizuagent | grep tap | grep -v grep | awk '{ print $2 }') > /dev/null 2>&1
-
-		local output_file=$MIZU_BENCHMARK_OUTPUT_DIR/${mode_num}_${mode_str}_${i}.log
-		log "  $i: Moving output to $output_file"
-		mv tapper.log $output_file || return 1
-	done
-}
-
-function generate_bench_graph() {
-	cd performance_analysis/ || return 1
-	source venv/bin/activate
-	python plot_from_tapper_logs.py $MIZU_BENCHMARK_OUTPUT_DIR/*.log || return 1
-	mv graph.png $MIZU_BENCHMARK_OUTPUT_DIR || return 1
-}
-
-mkdir -p $MIZU_BENCHMARK_OUTPUT_DIR
-rm -f $MIZU_BENCHMARK_OUTPUT_DIR/*
-log "Writing output to $MIZU_BENCHMARK_OUTPUT_DIR"
-
-cd $MIZU_HOME || exit 1
-
-export HOST_MODE=0
-export MIZU_DEBUG_DISABLE_PCAP=false
-export MIZU_DEBUG_DISABLE_TCP_REASSEMBLY=false
-export MIZU_DEBUG_DISABLE_TCP_STREAM=false
-export MIZU_DEBUG_DISABLE_NON_HTTP_EXTENSSION=false
-export MIZU_DEBUG_DISABLE_DISSECTORS=false
-export MIZU_DEBUG_DISABLE_EMITTING=false
-export MIZU_DEBUG_DISABLE_SENDING=false
-
-export MIZU_DEBUG_DISABLE_PCAP=true
-run_single_bench "01" "no_pcap" || exit 1
-export MIZU_DEBUG_DISABLE_PCAP=false
-
-export MIZU_DEBUG_DISABLE_TCP_REASSEMBLY=true
-run_single_bench "02" "no_assembler" || exit 1
-export MIZU_DEBUG_DISABLE_TCP_REASSEMBLY=false
-
-export MIZU_DEBUG_DISABLE_TCP_STREAM=true
-run_single_bench "03" "no_tcp_stream" || exit 1
-export MIZU_DEBUG_DISABLE_TCP_STREAM=false
-
-export MIZU_DEBUG_DISABLE_NON_HTTP_EXTENSSION=true
-run_single_bench "04" "only_http" || exit 1
-export MIZU_DEBUG_DISABLE_NON_HTTP_EXTENSSION=false
-
-export MIZU_DEBUG_DISABLE_DISSECTORS=true
-run_single_bench "05" "no_dissectors" || exit 1
-export MIZU_DEBUG_DISABLE_DISSECTORS=false
-
-export MIZU_DEBUG_DISABLE_EMITTING=true
-run_single_bench "06" "no_emit" || exit 1
-export MIZU_DEBUG_DISABLE_EMITTING=false
-
-export MIZU_DEBUG_DISABLE_SENDING=true
-run_single_bench "07" "no_send" || exit 1
-export MIZU_DEBUG_DISABLE_SENDING=false
-
-run_single_bench "08" "normal" || exit 1
-
-generate_bench_graph || exit 1
-log "Output written to to $MIZU_BENCHMARK_OUTPUT_DIR"

shared/consts.go

@@ -10,6 +10,7 @@ const (
 	ValidationRulesFileName    = "validation-rules.yaml"
 	ContractFileName           = "contract-oas.yaml"
 	ConfigFileName             = "mizu-config.json"
+	GoGCEnvVar                 = "GOGC"
 	DefaultApiServerPort       = 8899
 	LogLevelEnvVar             = "LOG_LEVEL"
 	MizuAgentImageRepo         = "docker.io/up9inc/mizu"

shared/go.mod

@@ -6,7 +6,6 @@ require (
 	github.com/docker/go-units v0.4.0
 	github.com/golang-jwt/jwt/v4 v4.2.0
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
-	github.com/up9inc/mizu/logger v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.23.3
@@ -67,7 +66,6 @@ require (
 	github.com/spf13/cobra v1.3.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
-	github.com/up9inc/mizu/tap/dbgctl v0.0.0 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect
 	go.starlark.net v0.0.0-20220203230714-bb14e151c28f // indirect
 	golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab // indirect
@@ -93,8 +91,4 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace github.com/up9inc/mizu/logger v0.0.0 => ../logger
-
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
-
-replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../tap/dbgctl

shared/kubernetes/mizuTapperSyncer.go

@@ -7,9 +7,9 @@ import (
 	"time"
 
 	"github.com/op/go-logging"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
+	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap/api"
 	core "k8s.io/api/core/v1"
 )

shared/kubernetes/provider.go

@@ -12,8 +12,8 @@ import (
 	"regexp"
 
 	"github.com/op/go-logging"
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/semver"
 	"github.com/up9inc/mizu/tap/api"
 	auth "k8s.io/api/authorization/v1"
@@ -181,7 +181,6 @@ type ApiServerOptions struct {
 	Resources             shared.Resources
 	ImagePullPolicy       core.PullPolicy
 	LogLevel              logging.Level
-	Profiler              bool
 }
 
 func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, mountVolumeClaim bool, volumeClaimName string, createAuthContainer bool) (*core.Pod, error) {
@@ -213,15 +212,7 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 		return nil, fmt.Errorf("invalid memory request for %s container", opts.PodName)
 	}
 
-	command := []string{
-		"./mizuagent",
-		"--api-server",
-	}
-
-	if opts.Profiler {
-		command = append(command, "--profiler")
-	}
-
+	command := []string{"./mizuagent", "--api-server"}
 	if opts.IsNamespaceRestricted {
 		command = append(command, "--namespace", opts.Namespace)
 	}
@@ -392,16 +383,6 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
 			Volumes:                       volumes,
 			DNSPolicy:                     core.DNSClusterFirstWithHostNet,
 			TerminationGracePeriodSeconds: new(int64),
-			Tolerations: []core.Toleration{
-				{
-					Operator: core.TolerationOpExists,
-					Effect: core.TaintEffectNoExecute,
-				},
-				{
-					Operator: core.TolerationOpExists,
-					Effect: core.TaintEffectNoSchedule,
-				},
-			},
 		},
 	}
 
@@ -787,6 +768,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	agentContainer.WithEnv(
 		applyconfcore.EnvVar().WithName(shared.LogLevelEnvVar).WithValue(logLevel.String()),
 		applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"),
+		applyconfcore.EnvVar().WithName(shared.GoGCEnvVar).WithValue("12800"),
 		applyconfcore.EnvVar().WithName(shared.MizuFilteringOptionsEnvVar).WithValue(string(mizuApiFilteringOptionsJsonStr)),
 	)
 	agentContainer.WithEnv(

shared/kubernetes/proxy.go

@@ -16,7 +16,7 @@ import (
 	"k8s.io/client-go/tools/portforward"
 	"k8s.io/client-go/transport/spdy"
 
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 	"k8s.io/kubectl/pkg/proxy"
 )
 

shared/kubernetes/watch.go

@@ -7,8 +7,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared/debounce"
+	"github.com/up9inc/mizu/shared/logger"
 
 	"k8s.io/apimachinery/pkg/watch"
 )

shared/models.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 
 	"github.com/op/go-logging"
-	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared/logger"
 
 	"gopkg.in/yaml.v3"
 	v1 "k8s.io/api/core/v1"
@@ -45,6 +45,13 @@ type MizuAgentConfig struct {
 	ServiceMap             bool          `json:"serviceMap"`
 	OAS                    bool          `json:"oas"`
 	Telemetry              bool          `json:"telemetry"`
+	Elastic                ElasticConfig `json:"elastic"`
+}
+
+type ElasticConfig struct {
+	User     string `yaml:"user,omitempty" default:"" readonly:""`
+	Password string `yaml:"password,omitempty" default:"" readonly:""`
+	Url      string `yaml:"url,omitempty" default:"" readonly:""`
 }
 
 type WebSocketMessageMetadata struct {