Acceptance tests module rename (#862)

* Changing mizu ent helm name

.github/workflows/release.yml

@@ -58,6 +58,7 @@ jobs:
             up9inc/mizu
           tags: |
             type=raw,${{ steps.versioning.outputs.version }}
+            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
           flavor: |
             latest=auto
             prefix=
@@ -143,6 +144,7 @@ jobs:
             ${{ steps.base_image_step.outputs.image }}
           tags: |
             type=raw,${{ steps.versioning.outputs.version }}
+            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
           flavor: |
             latest=auto
             prefix=
@@ -205,6 +207,7 @@ jobs:
             up9inc/mizu
           tags: |
             type=raw,${{ steps.versioning.outputs.version }}
+            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
 
       - name: Login to Docker Hub
         uses: docker/login-action@v1

Dockerfile

@@ -78,8 +78,8 @@ RUN go build -ldflags="-extldflags=-static -s -w \
     -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum
-ADD https://github.com/up9inc/basenine/releases/download/v0.4.14/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
-ADD https://github.com/up9inc/basenine/releases/download/v0.4.14/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.4.17/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
+ADD https://github.com/up9inc/basenine/releases/download/v0.4.17/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
 RUN shasum -a 256 -c basenine_linux_${GOARCH}.sha256
 RUN chmod +x ./basenine_linux_${GOARCH}
 RUN mv ./basenine_linux_${GOARCH} ./basenine

Makefile

@@ -103,6 +103,9 @@ test-shared:  ## Run shared tests
 
 test-extensions:  ## Run extensions tests
 	@echo "running http tests"; cd tap/extensions/http && $(MAKE) test
+	@echo "running redis tests"; cd tap/extensions/redis && $(MAKE) test
+	@echo "running kafka tests"; cd tap/extensions/kafka && $(MAKE) test
+	@echo "running amqp tests"; cd tap/extensions/amqp && $(MAKE) test
 
 acceptance-test:  ## Run acceptance tests
 	@echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test

README.md

@@ -8,10 +8,10 @@
         <img alt="GitHub Latest Release" src="https://img.shields.io/github/v/release/up9inc/mizu?logo=GitHub&style=flat-square">
     </a>
     <a href="https://hub.docker.com/r/up9inc/mizu">
-      <img alt="Docker pulls" src="https://img.shields.io/docker/pulls/up9inc/mizu?color=%23099cec">
+      <img alt="Docker pulls" src="https://img.shields.io/docker/pulls/up9inc/mizu?color=%23099cec&logo=Docker&style=flat-square">
     </a>
     <a href="https://hub.docker.com/r/up9inc/mizu">
-      <img alt="Image size" src="https://img.shields.io/docker/image-size/up9inc/mizu/latest">
+      <img alt="Image size" src="https://img.shields.io/docker/image-size/up9inc/mizu/latest?logo=Docker&style=flat-square">
     </a>
     <a href="https://join.slack.com/t/up9/shared_invite/zt-tfjnduli-QzlR8VV4Z1w3YnPIAJfhlQ">
       <img alt="Slack" src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social">

acceptanceTests/cypress.json

@@ -14,7 +14,9 @@
     "tests/RegexMasking.js",
     "tests/IgnoredUserAgents.js",
     "tests/UiTest.js",
-    "tests/Redis.js"
+    "tests/Redis.js",
+    "tests/Rabbit.js",
+    "tests/serviceMapFunction.js"
   ],
 
   "env": {
@@ -22,9 +24,11 @@
     "redactHeaderContent": "User-Header[REDACTED]",
     "redactBodyContent": "{ \"User\": \"[REDACTED]\" }",
     "regexMaskingBodyContent": "[REDACTED]",
-    "minimumEntries": 25,
     "greenFilterColor": "rgb(210, 250, 210)",
     "redFilterColor": "rgb(250, 214, 220)",
-    "bodyJsonClass": ".hljs"
+    "bodyJsonClass": ".hljs",
+    "mizuWidth": 1920,
+    "normalMizuHeight": 1080,
+    "hugeMizuHeight": 3500
   }
 }

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -1,3 +1,11 @@
+export const valueTabs = {
+    response: 'RESPONSE',
+    request: 'REQUEST',
+    none: null
+}
+
+const maxEntriesInDom = 13;
+
 export function isValueExistsInElement(shouldInclude, content, domPathToContainer){
     it(`should ${shouldInclude ? '' : 'not'} include '${content}'`, function () {
         cy.get(domPathToContainer).then(htmlText => {
@@ -9,22 +17,24 @@ export function isValueExistsInElement(shouldInclude, content, domPathToContaine
 }
 
 export function resizeToHugeMizu() {
-    cy.viewport(1920, 3500);
+    cy.viewport(Cypress.env('mizuWidth'), Cypress.env('hugeMizuHeight'));
 }
 
 export function resizeToNormalMizu() {
-    cy.viewport(1920, 1080);
+    cy.viewport(Cypress.env('mizuWidth'), Cypress.env('normalMizuHeight'));
 }
 
 export function verifyMinimumEntries() {
-    const minimumEntries = Cypress.env('minimumEntries');
-    it(`Making sure that mizu shows at least ${minimumEntries} entries`, async function () {
+    const entriesSent = Cypress.env('entriesCount');
+    const minimumEntries = Math.round((0.75 * entriesSent));
+
+    it(`Making sure that mizu shows at least ${minimumEntries} entries`, function () {
         cy.get('#total-entries').then(number => {
             const getNum = () => {
-                const numOfEntries = number.text();
-                return parseInt(numOfEntries);
+                return parseInt(number.text());
             };
-            cy.wrap({there: getNum}).invoke('there').should('be.gte', minimumEntries);
+
+            cy.wrap({num: getNum}).invoke('num').should('be.gt', minimumEntries);
         });
     });
 }
@@ -53,3 +63,125 @@ export function checkThatAllEntriesShown() {
             cy.get('[title="Fetch old records"]').click();
     });
 }
+
+export function checkFilterByMethod(funcDict) {
+    const {protocol, method, summary, hugeMizu} = funcDict;
+    const summaryDict = getSummeryDict(summary);
+    const methodDict = getMethodDict(method);
+    const protocolDict = getProtocolDict(protocol.name, protocol.text);
+
+    it(`Testing the method: ${method}`, function () {
+        // applying filter
+        cy.get('.w-tc-editor-text').clear().type(`method == "${method}"`);
+        cy.get('[type="submit"]').click();
+        cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
+
+        cy.get('#entries-length').then(number => {
+            // if the entries list isn't expanded it expands here
+            if (number.text() === '0' || number.text() === '1') // todo change when TRA-4262 is fixed
+                cy.get('[title="Fetch old records"]').click();
+
+            cy.get('#entries-length').should('not.have.text', '0').and('not.have.text', '1').then(() => {
+                cy.get(`#list [id]`).then(elements => {
+                    const listElmWithIdAttr = Object.values(elements);
+                    let doneCheckOnFirst = false;
+
+                    cy.get('#entries-length').invoke('text').then(len => {
+                        resizeIfNeeded(len);
+                        listElmWithIdAttr.forEach(entry => {
+                            if (entry?.id && entry.id.match(RegExp(/entry-(\d{2}|\d{1})$/gm))) {
+                                const entryNum = getEntryNumById(entry.id);
+
+                                leftTextCheck(entryNum, methodDict.pathLeft, methodDict.expectedText);
+                                leftTextCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedTextLeft);
+                                if (summaryDict)
+                                    leftTextCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedText);
+
+                                if (!doneCheckOnFirst) {
+                                    deepCheck(funcDict, protocolDict, methodDict, entry);
+                                    doneCheckOnFirst = true;
+                                }
+                            }
+                        });
+                        resizeIfNeeded(len);
+                    });
+                });
+            });
+        });
+    });
+}
+
+function resizeIfNeeded(entriesLen) {
+    if (entriesLen > maxEntriesInDom){
+        Cypress.config().viewportHeight === Cypress.env('normalMizuHeight') ?
+            resizeToHugeMizu() : resizeToNormalMizu()
+    }
+}
+
+function deepCheck(generalDict, protocolDict, methodDict, entry) {
+    const entryNum = getEntryNumById(entry.id);
+    const {summary, value} = generalDict;
+    const summaryDict = getSummeryDict(summary);
+
+    leftOnHoverCheck(entryNum, methodDict.pathLeft, methodDict.expectedOnHover);
+    leftOnHoverCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedOnHover);
+    if (summaryDict)
+        leftOnHoverCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedOnHover);
+
+    cy.get(`#${entry.id}`).click();
+
+    rightTextCheck(methodDict.pathRight, methodDict.expectedText);
+    rightTextCheck(protocolDict.pathRight, protocolDict.expectedTextRight);
+    if (summaryDict)
+        rightTextCheck(summaryDict.pathRight, summaryDict.expectedText);
+
+    rightOnHoverCheck(methodDict.pathRight, methodDict.expectedOnHover);
+    rightOnHoverCheck(protocolDict.pathRight, protocolDict.expectedOnHover);
+    if (summaryDict)
+        rightOnHoverCheck(summaryDict.pathRight, summaryDict.expectedOnHover);
+
+    if (value) {
+        if (value.tab === valueTabs.response)
+            cy.contains('Response').click();
+        cy.get(Cypress.env('bodyJsonClass')).then(text => {
+            expect(text.text()).to.match(value.regex)
+        });
+    }
+}
+
+function getSummeryDict(summary) {
+    if (summary) {
+        return {
+            pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+            pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+            expectedText: summary,
+            expectedOnHover: `summary == "${summary}"`
+        };
+    }
+    else {
+        return null;
+    }
+}
+
+function getMethodDict(method) {
+    return {
+        pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+        pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+        expectedText: method,
+        expectedOnHover: `method == "${method}"`
+    };
+}
+
+function getProtocolDict(protocol, protocolText) {
+    return {
+        pathLeft: '> :nth-child(1) > :nth-child(1)',
+        pathRight: '> :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(1)',
+        expectedTextLeft: protocol.toUpperCase(),
+        expectedTextRight: protocolText,
+        expectedOnHover: protocol.toLowerCase()
+    };
+}
+
+function getEntryNumById (id) {
+    return parseInt(id.split('-')[1]);
+}

acceptanceTests/cypress/integration/tests/GuiPort.js

@@ -1,5 +1,4 @@
 import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
-import {verifyMinimumEntries} from "../testHelpers/TrafficHelper";
 
 it('check', function () {
     const podName = Cypress.env('name'), namespace = Cypress.env('namespace');
@@ -9,8 +8,6 @@ it('check', function () {
     cy.visit(`http://localhost:${port}`);
     cy.wait('@statusTap').its('response.statusCode').should('match', /^2\d{2}/);
 
-    verifyMinimumEntries();
-
     cy.get('.podsCount').trigger('mouseover');
     findLineAndCheck(getExpectedDetailsDict(podName, namespace));
 });

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -2,15 +2,12 @@ import {
     checkThatAllEntriesShown,
     isValueExistsInElement,
     resizeToHugeMizu,
-    verifyMinimumEntries
 } from "../testHelpers/TrafficHelper";
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
 });
 
-verifyMinimumEntries();
-
 checkEntries();
 
 function checkEntries() {

acceptanceTests/cypress/integration/tests/NoRedact.js

@@ -1,10 +1,8 @@
-import {isValueExistsInElement, verifyMinimumEntries} from '../testHelpers/TrafficHelper';
+import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
 });
 
-verifyMinimumEntries();
-
 isValueExistsInElement(false, Cypress.env('redactHeaderContent'), '#tbody-Headers');
 isValueExistsInElement(false, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/integration/tests/Rabbit.js

@@ -0,0 +1,49 @@
+import {checkFilterByMethod, valueTabs,} from "../testHelpers/TrafficHelper";
+
+it('opening mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+const rabbitProtocolDetails = {name: 'AMQP', text: 'Advanced Message Queuing Protocol 0-9-1'};
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method: 'exchange declare',
+    summary: 'exchange',
+    value: null
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method: 'queue declare',
+    summary: 'queue',
+    value: null
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method:  'queue bind',
+    summary: 'queue',
+    value: null
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method: 'basic publish',
+    summary: 'exchange',
+    value: {tab: valueTabs.request, regex: /^message$/mg}
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method:  'basic consume',
+    summary: 'queue',
+    value: null
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method:  'basic deliver',
+    summary: 'exchange',
+    value: {tab: valueTabs.request, regex: /^message$/mg}
+});

acceptanceTests/cypress/integration/tests/Redact.js

@@ -1,10 +1,8 @@
-import {isValueExistsInElement, verifyMinimumEntries} from '../testHelpers/TrafficHelper';
+import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
 });
 
-verifyMinimumEntries();
-
 isValueExistsInElement(true, Cypress.env('redactHeaderContent'), '#tbody-Headers');
 isValueExistsInElement(true, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/integration/tests/Redis.js

@@ -1,155 +1,42 @@
-import {
-    leftOnHoverCheck,
-    leftTextCheck,
-    rightOnHoverCheck,
-    rightTextCheck,
-} from "../testHelpers/TrafficHelper";
-
-const valueTabs = {
-    response: 'RESPONSE',
-    request: 'REQUEST',
-    none: null
-}
+import {checkFilterByMethod, valueTabs,} from "../testHelpers/TrafficHelper";
 
 it('opening mizu', function () {
     cy.visit(Cypress.env('testUrl'));
 });
 
-checkRedisFilterByMethod({
+const redisProtocolDetails = {name: 'redis', text: 'Redis Serialization Protocol'};
+
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
     method: 'PING',
-    shouldCheckSummary: false,
-    valueTab: valueTabs.none
-});
+    summary: null,
+    value: null
+})
 
-checkRedisFilterByMethod({
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
     method: 'SET',
-    shouldCheckSummary: true,
-    valueTab: valueTabs.request,
-    valueRegex: /^\[value, keepttl]$/mg
-});
+    summary: 'key',
+    value: {tab: valueTabs.request, regex: /^\[value, keepttl]$/mg}
+})
 
-checkRedisFilterByMethod({
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
     method: 'EXISTS',
-    shouldCheckSummary: true,
-    valueTab: valueTabs.response,
-    valueRegex: /^1$/mg
-});
+    summary: 'key',
+    value: {tab: valueTabs.response, regex: /^1$/mg}
+})
 
-checkRedisFilterByMethod({
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
     method: 'GET',
-    shouldCheckSummary: true,
-    valueTab: valueTabs.response,
-    valueRegex: /^value$/mg
-});
+    summary: 'key',
+    value: {tab: valueTabs.response, regex: /^value$/mg}
+})
 
-checkRedisFilterByMethod({
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
     method: 'DEL',
-    shouldCheckSummary: true,
-    valueTab: valueTabs.response,
-    valueRegex: /^1$|^0$/mg
-});
-
-function checkRedisFilterByMethod(funcDict) {
-    const {method, shouldCheckSummary} = funcDict
-    const summaryDict = getSummeryDict();
-    const methodDict = getMethodDict(method);
-    const protocolDict = getProtocolDict();
-
-    it(`Testing the method: ${method}`, function () {
-        // applying filter
-        cy.get('.w-tc-editor-text').clear().type(`method == "${method}"`);
-        cy.get('[type="submit"]').click();
-        cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
-
-        cy.get('#entries-length').then(number => {
-            // if the entries list isn't expanded it expands here
-            if (number.text() === '0' || number.text() === '1') // todo change when TRA-4262 is fixed
-                cy.get('[title="Fetch old records"]').click();
-
-            cy.get('#entries-length').should('not.have.text', '0').and('not.have.text', '1').then(() => {
-                cy.get(`#list [id]`).then(elements => {
-                   const listElmWithIdAttr = Object.values(elements);
-                   let doneCheckOnFirst = false;
-
-                   listElmWithIdAttr.forEach(entry => {
-                       if (entry?.id && entry.id.match(RegExp(/entry-(\d{2}|\d{1})$/gm))) {
-                           const entryNum = getEntryNumById(entry.id);
-
-                           leftTextCheck(entryNum, methodDict.pathLeft, methodDict.expectedText);
-                           leftTextCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedTextLeft);
-                           if (shouldCheckSummary)
-                               leftTextCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedText);
-
-                           if (!doneCheckOnFirst) {
-                               deepCheck(funcDict, protocolDict, methodDict, summaryDict, entry);
-                               doneCheckOnFirst = true;
-                           }
-                       }
-                   });
-                });
-            });
-        });
-    });
-}
-
-function deepCheck(generalDict, protocolDict, methodDict, summaryDict, entry) {
-    const entryNum = getEntryNumById(entry.id);
-    const {shouldCheckSummary, valueTab, valueRegex} = generalDict;
-
-    leftOnHoverCheck(entryNum, methodDict.pathLeft, methodDict.expectedOnHover);
-    leftOnHoverCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedOnHover);
-    if (shouldCheckSummary)
-        leftOnHoverCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedOnHover);
-
-    cy.get(`#${entry.id}`).click();
-
-    rightTextCheck(methodDict.pathRight, methodDict.expectedText);
-    rightTextCheck(protocolDict.pathRight, protocolDict.expectedTextRight);
-    if (shouldCheckSummary)
-        rightTextCheck(summaryDict.pathRight, summaryDict.expectedText);
-
-    rightOnHoverCheck(methodDict.pathRight, methodDict.expectedOnHover);
-    rightOnHoverCheck(protocolDict.pathRight, protocolDict.expectedOnHover);
-    if (shouldCheckSummary)
-        rightOnHoverCheck(summaryDict.pathRight, summaryDict.expectedOnHover);
-
-    if (valueTab) {
-        if (valueTab === valueTabs.response)
-            cy.contains('Response').click();
-        cy.get(Cypress.env('bodyJsonClass')).then(text => {
-            expect(text.text()).to.match(valueRegex)
-        });
-    }
-}
-
-function getSummeryDict() {
-    return {
-        pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
-        pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
-        expectedText: 'key',
-        expectedOnHover: `summary == "key"`
-    };
-}
-
-function getMethodDict(method) {
-    return {
-        pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
-        pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
-        expectedText: method,
-        expectedOnHover: `method == "${method}"`
-    };
-}
-
-function getProtocolDict() {
-    return {
-        pathLeft: '> :nth-child(1) > :nth-child(1)',
-        pathRight: '> :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(1)',
-        expectedTextLeft: 'REDIS',
-        expectedTextRight: 'Redis Serialization Protocol',
-        expectedOnHover: `redis`
-    };
-}
-
-function getEntryNumById (id) {
-    return parseInt(id.split('-')[1]);
-}
+    summary: 'key',
+    value: {tab: valueTabs.response, regex: /^1$|^0$/mg}
+})

acceptanceTests/cypress/integration/tests/RegexMasking.js

@@ -1,9 +1,7 @@
-import {isValueExistsInElement, verifyMinimumEntries} from "../testHelpers/TrafficHelper";
+import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
 });
 
-verifyMinimumEntries();
-
 isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -64,6 +64,8 @@ it('right side sanity test', function () {
     });
 });
 
+serviceMapCheck();
+
 checkIllegalFilter('invalid filter');
 
 checkFilter({
@@ -188,7 +190,7 @@ function checkFilter(filterDetails){
     const entriesForDeeperCheck = 5;
 
     it(`checking the filter: ${name}`, function () {
-        cy.get('#total-entries').then(number => {
+        cy.get('#total-entries').should('not.have.text', '0').then(number => {
             const totalEntries = number.text();
 
             // checks the hover on the last entry (the only one in DOM at the beginning)
@@ -320,3 +322,42 @@ function checkOnlyLineNumberes(jsonItems, decodedText) {
     cy.get(`${Cypress.env('bodyJsonClass')} >`).should('have.length', 1).and('have.text', decodedText);
     cy.get(`${Cypress.env('bodyJsonClass')} > >`).should('have.length', jsonItems)
 }
+
+function serviceMapCheck() {
+    it('service map test', function () {
+        cy.intercept(`${Cypress.env('testUrl')}/servicemap/get`).as('serviceMapRequest');
+        cy.get('#total-entries').should('not.have.text', '0').then(() => {
+            cy.get('#total-entries').invoke('text').then(entriesNum => {
+                cy.get('[alt="service-map"]').click();
+                cy.wait('@serviceMapRequest').then(({response}) => {
+                    const body = response.body;
+                    const nodeParams = {
+                        destination: 'httpbin.mizu-tests',
+                        source: '127.0.0.1'
+                    };
+                    serviceMapAPICheck(body, parseInt(entriesNum), nodeParams);
+                    cy.reload();
+                });
+            });
+        });
+    });
+}
+
+function serviceMapAPICheck(body, entriesNum, nodeParams) {
+    const {nodes, edges} = body;
+
+    expect(nodes.length).to.equal(Object.keys(nodeParams).length, `Expected nodes count`);
+
+    expect(edges.some(edge => edge.source.name === nodeParams.source)).to.be.true;
+    expect(edges.some(edge => edge.destination.name === nodeParams.destination)).to.be.true;
+
+    let count = 0;
+    edges.forEach(edge => {
+        count += edge.count;
+        if (edge.destination.name === nodeParams.destination) {
+            expect(edge.source.name).to.equal(nodeParams.source);
+        }
+    });
+
+    expect(count).to.equal(entriesNum);
+}

acceptanceTests/extensions_test.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"fmt"
 	"github.com/go-redis/redis/v8"
+	amqp "github.com/rabbitmq/amqp091-go"
 	"os/exec"
 	"testing"
+	"time"
 )
 
 func TestRedis(t *testing.T) {
@@ -99,3 +101,128 @@ func TestRedis(t *testing.T) {
 
 	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/Redis.js\"")
 }
+
+func TestAmqp(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := cleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	ctx := context.Background()
+
+	rabbitmqExternalIp, err := getServiceExternalIp(ctx, defaultNamespaceName, "rabbitmq")
+	if err != nil {
+		t.Errorf("failed to get RabbitMQ external ip, err: %v", err)
+		return
+	}
+
+	conn, err := amqp.Dial(fmt.Sprintf("amqp://guest:guest@%v:5672/", rabbitmqExternalIp))
+	if err != nil {
+		t.Errorf("failed to connect to RabbitMQ, err: %v", err)
+		return
+	}
+	defer conn.Close()
+
+	// Temporary fix for missing amqp entries
+	time.Sleep(10 * time.Second)
+
+	for i := 0; i < defaultEntriesCount/5; i++ {
+		ch, err := conn.Channel()
+		if err != nil {
+			t.Errorf("failed to open a channel, err: %v", err)
+			return
+		}
+
+		exchangeName := "exchange"
+		err = ch.ExchangeDeclare(exchangeName, "direct", true, false, false, false, nil)
+		if err != nil {
+			t.Errorf("failed to declare an exchange, err: %v", err)
+			return
+		}
+
+		q, err := ch.QueueDeclare("queue", true, false, false, false, nil)
+		if err != nil {
+			t.Errorf("failed to declare a queue, err: %v", err)
+			return
+		}
+
+		routingKey := "routing_key"
+		err = ch.QueueBind(q.Name, routingKey, exchangeName, false, nil)
+		if err != nil {
+			t.Errorf("failed to bind the queue, err: %v", err)
+			return
+		}
+
+		err = ch.Publish(exchangeName, routingKey, false, false,
+			amqp.Publishing{
+				DeliveryMode: amqp.Persistent,
+				ContentType:  "text/plain",
+				Body:         []byte("message"),
+			})
+		if err != nil {
+			t.Errorf("failed to publish a message, err: %v", err)
+			return
+		}
+
+		msgChan, err := ch.Consume(q.Name, "Consumer", true, false, false, false, nil)
+		if err != nil {
+			t.Errorf("failed to create a consumer, err: %v", err)
+			return
+		}
+
+		select {
+		case <-msgChan:
+			break
+		case <-time.After(3 * time.Second):
+			t.Errorf("failed to consume a message on time")
+			return
+		}
+
+		err = ch.ExchangeDelete(exchangeName, false, false)
+		if err != nil {
+			t.Errorf("failed to delete the exchange, err: %v", err)
+			return
+		}
+
+		_, err = ch.QueueDelete(q.Name, false, false, false)
+		if err != nil {
+			t.Errorf("failed to delete the queue, err: %v", err)
+			return
+		}
+
+		ch.Close()
+	}
+
+	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/Rabbit.js\"")
+}

acceptanceTests/go.mod

@@ -1,9 +1,10 @@
-module github.com/up9inc/mizu/tests
+module github.com/up9inc/mizu/acceptanceTests
 
 go 1.17
 
 require (
 	github.com/go-redis/redis/v8 v8.11.4
+	github.com/rabbitmq/amqp091-go v1.3.0
 	github.com/up9inc/mizu/shared v0.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/apimachinery v0.23.3

acceptanceTests/go.sum

@@ -427,6 +427,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rabbitmq/amqp091-go v1.3.0 h1:A/QuHiNw7LMCJsxx9iZn5lrIz6OrhIn7Dfk5/1YatWM=
+github.com/rabbitmq/amqp091-go v1.3.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=

acceptanceTests/setup.sh

@@ -39,6 +39,9 @@ kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2
 echo "Creating redis deployment"
 kubectl create deployment redis --image=redis -n mizu-tests
 
+echo "Creating rabbitmq deployment"
+kubectl create deployment rabbitmq --image=rabbitmq -n mizu-tests
+
 echo "Creating httpbin services"
 kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
 kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests
@@ -48,6 +51,9 @@ kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2
 echo "Creating redis service"
 kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests
 
+echo "Creating rabbitmq service"
+kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests
+
 echo "Starting proxy"
 kubectl proxy --port=8080 &
 

acceptanceTests/tap_test.go

@@ -62,7 +62,7 @@ func TestTap(t *testing.T) {
 				}
 			}
 
-			runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/UiTest.js\"")
+			runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/UiTest.js\" --env entriesCount=%d", entriesCount))
 		})
 	}
 }

acceptanceTests/testsUtils.go

@@ -150,9 +150,10 @@ func runCypressTests(t *testing.T, cypressRunCmd string) {
 	t.Logf("running command: %v", cypressCmd.String())
 	out, err := cypressCmd.Output()
 	if err != nil {
-		t.Errorf("%s", out)
+		t.Errorf("error running cypress, error: %v", err)
 		return
 	}
+
 	t.Logf("%s", out)
 }
 

agent/go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/ory/kratos-client-go v0.8.2-alpha.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/stretchr/testify v1.7.0
-	github.com/up9inc/basenine/client/go v0.0.0-20220125035757-926e42208705
+	github.com/up9inc/basenine/client/go v0.0.0-20220220204122-0ef8cb24fab1
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
@@ -54,6 +54,7 @@ require (
 	github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 // indirect
 	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
 	github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect
+	github.com/cilium/ebpf v0.8.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect

agent/go.sum

@@ -137,6 +137,8 @@ github.com/chanced/openapi v0.0.7/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5Xyq
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/cilium/ebpf v0.8.0 h1:2V6KSg3FRADVU2BMIRemZ0hV+9OM+aAHhZDjQyjJTAs=
+github.com/cilium/ebpf v0.8.0/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -210,8 +212,9 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
+github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss=
+github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
@@ -850,8 +853,8 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
 github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
-github.com/up9inc/basenine/client/go v0.0.0-20220125035757-926e42208705 h1:5LLhzv0cjb/F+dU0z3j8teVGjQInMYAocTyAZohKUwY=
-github.com/up9inc/basenine/client/go v0.0.0-20220125035757-926e42208705/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220220204122-0ef8cb24fab1 h1:0XN8s3HtwUBr9hbWRAFulFMsu1f2cabfJbwpz/sOoLA=
+github.com/up9inc/basenine/client/go v0.0.0-20220220204122-0ef8cb24fab1/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
@@ -1158,6 +1161,7 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

agent/main.go

@@ -56,11 +56,9 @@ const (
 
 func main() {
 	logLevel := determineLogLevel()
-	logger.InitLoggerStderrOnly(logLevel)
+	logger.InitLoggerStd(logLevel)
 	flag.Parse()
-	if err := config.LoadConfig(); err != nil {
-		logger.Log.Fatalf("Error loading config file %v", err)
-	}
+
 	app.LoadExtensions()
 
 	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode {
@@ -139,7 +137,10 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engin
 }
 
 func runInApiServerMode(namespace string) *gin.Engine {
-	app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort)
+	if err := config.LoadConfig(); err != nil {
+		logger.Log.Fatalf("Error loading config file %v", err)
+	}
+	app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel)
 	startTime = time.Now().UnixNano() / int64(time.Millisecond)
 	api.StartResolving(namespace)
 
@@ -215,7 +216,7 @@ func enableExpFeatureIfNeeded() {
 		oas.GetOasGeneratorInstance().Start()
 	}
 	if config.Config.ServiceMap {
-		servicemap.GetInstance().SetConfig(config.Config)
+		servicemap.GetInstance().Enable()
 	}
 	elastic.GetInstance().Configure(config.Config.Elastic)
 }

agent/pkg/api/main.go

@@ -118,8 +118,8 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 
 	for item := range outputItems {
 		extension := extensionsMap[item.Protocol.Name]
-		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
-		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
+		resolvedSource, resolvedDestionation, namespace := resolveIP(item.ConnectionInfo)
+		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation, namespace)
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
 				var httpPair tapApi.HTTPRequestResponsePair
@@ -140,7 +140,17 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 				mizuEntry.Rules = rules
 			}
 
-			entryWSource := oas.EntryWithSource{Entry: *harEntry, Source: mizuEntry.Source.Name, Id: mizuEntry.Id}
+			entryWSource := oas.EntryWithSource{
+				Entry:       *harEntry,
+				Source:      mizuEntry.Source.Name,
+				Destination: mizuEntry.Destination.Name,
+				Id:          mizuEntry.Id,
+			}
+
+			if entryWSource.Destination == "" {
+				entryWSource.Destination = mizuEntry.Destination.IP + ":" + mizuEntry.Destination.Port
+			}
+
 			oas.GetOasGeneratorInstance().PushEntry(&entryWSource)
 		}
 
@@ -158,26 +168,32 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 	}
 }
 
-func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, resolvedDestination string) {
+func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, resolvedDestination string, namespace string) {
 	if k8sResolver != nil {
 		unresolvedSource := connectionInfo.ClientIP
-		resolvedSource = k8sResolver.Resolve(unresolvedSource)
-		if resolvedSource == "" {
+		resolvedSourceObject := k8sResolver.Resolve(unresolvedSource)
+		if resolvedSourceObject == nil {
 			logger.Log.Debugf("Cannot find resolved name to source: %s", unresolvedSource)
 			if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
 				return
 			}
+		} else {
+			resolvedSource = resolvedSourceObject.FullAddress
 		}
+
 		unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
-		resolvedDestination = k8sResolver.Resolve(unresolvedDestination)
-		if resolvedDestination == "" {
+		resolvedDestinationObject := k8sResolver.Resolve(unresolvedDestination)
+		if resolvedDestinationObject == nil {
 			logger.Log.Debugf("Cannot find resolved name to dest: %s", unresolvedDestination)
 			if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
 				return
 			}
+		} else {
+			resolvedDestination = resolvedDestinationObject.FullAddress
+			namespace = resolvedDestinationObject.Namespace
 		}
 	}
-	return resolvedSource, resolvedDestination
+	return resolvedSource, resolvedDestination, namespace
 }
 
 func CheckIsServiceIP(address string) bool {

agent/pkg/api/socket_routes.go

@@ -30,6 +30,11 @@ type SocketConnection struct {
 	isTapper      bool
 }
 
+type WebSocketParams struct {
+	Query             string `json:"query"`
+	EnableFullEntries bool   `json:"enableFullEntries"`
+}
+
 var (
 	websocketUpgrader = websocket.Upgrader{
 		ReadBufferSize:  1024,
@@ -110,6 +115,8 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 		logger.Log.Error(err)
 	}
 
+	var params WebSocketParams
+
 	for {
 		_, msg, err := ws.ReadMessage()
 		if err != nil {
@@ -123,7 +130,11 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 		}
 
 		if !isTapper && !isQuerySet {
-			query := string(msg)
+			if err := json.Unmarshal(msg, &params); err != nil {
+				logger.Log.Errorf("Error: %v", socketId, err)
+			}
+
+			query := params.Query
 			err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
 			if err != nil {
 				toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
@@ -150,10 +161,15 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 					var entry *tapApi.Entry
 					err = json.Unmarshal(bytes, &entry)
 
-					base := tapApi.Summarize(entry)
+					var message []byte
+					if params.EnableFullEntries {
+						message, _ = models.CreateFullEntryWebSocketMessage(entry)
+					} else {
+						base := tapApi.Summarize(entry)
+						message, _ = models.CreateBaseEntryWebSocketMessage(base)
+					}
 
-					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
-					if err := SendToSocket(socketId, baseEntryBytes); err != nil {
+					if err := SendToSocket(socketId, message); err != nil {
 						logger.Log.Error(err)
 					}
 				}

agent/pkg/api/socket_server_handlers.go

@@ -104,9 +104,9 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 }
 
 func handleTLSLink(outboundLinkMessage models.WebsocketOutboundLinkMessage) {
-	resolvedName := k8sResolver.Resolve(outboundLinkMessage.Data.DstIP)
-	if resolvedName != "" {
-		outboundLinkMessage.Data.DstIP = resolvedName
+	resolvedNameObject := k8sResolver.Resolve(outboundLinkMessage.Data.DstIP)
+	if resolvedNameObject != nil {
+		outboundLinkMessage.Data.DstIP = resolvedNameObject.FullAddress
 	} else if outboundLinkMessage.Data.SuggestedResolvedName != "" {
 		outboundLinkMessage.Data.DstIP = outboundLinkMessage.Data.SuggestedResolvedName
 	}

agent/pkg/app/main.go

@@ -9,7 +9,6 @@ import (
 	"github.com/op/go-logging"
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/agent/pkg/api"
-	"github.com/up9inc/mizu/agent/pkg/config"
 	"github.com/up9inc/mizu/agent/pkg/controllers"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
@@ -60,27 +59,21 @@ func LoadExtensions() {
 		return Extensions[i].Protocol.Priority < Extensions[j].Protocol.Priority
 	})
 
-	for _, extension := range Extensions {
-		logger.Log.Infof("Extension Properties: %+v", extension)
-	}
-
 	controllers.InitExtensionsMap(ExtensionsMap)
 }
 
-func ConfigureBasenineServer(host string, port string) {
+func ConfigureBasenineServer(host string, port string, dbSize int64, logLevel logging.Level) {
 	if !wait.New(
 		wait.WithProto("tcp"),
 		wait.WithWait(200*time.Millisecond),
 		wait.WithBreak(50*time.Millisecond),
 		wait.WithDeadline(5*time.Second),
-		wait.WithDebug(config.Config.LogLevel == logging.DEBUG),
+		wait.WithDebug(logLevel == logging.DEBUG),
 	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
 		logger.Log.Panicf("Basenine is not available!")
 	}
 
-	// Limit the database size to default 200MB
-	err := basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
-	if err != nil {
+	if err := basenine.Limit(host, port, dbSize); err != nil {
 		logger.Log.Panicf("Error while limiting database size: %v", err)
 	}
 
@@ -88,8 +81,7 @@ func ConfigureBasenineServer(host string, port string) {
 	for _, extension := range Extensions {
 		macros := extension.Dissector.Macros()
 		for macro, expanded := range macros {
-			err = basenine.Macro(host, port, macro, expanded)
-			if err != nil {
+			if err := basenine.Macro(host, port, macro, expanded); err != nil {
 				logger.Log.Panicf("Error while adding a macro: %v", err)
 			}
 		}

agent/pkg/controllers/config_controller.go

@@ -57,7 +57,7 @@ func PostTapConfig(c *gin.Context) {
 
 	ctx, cancel := context.WithCancel(context.Background())
 
-	if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{}, tapApi.TrafficFilteringOptions{}, false); err != nil {
+	if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{}, tapApi.TrafficFilteringOptions{}, false, false); err != nil {
 		c.JSON(http.StatusInternalServerError, err)
 		cancel()
 		return
@@ -100,7 +100,7 @@ func GetTapConfig(c *gin.Context) {
 	c.JSON(http.StatusOK, tapConfigToReturn)
 }
 
-func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, serviceMesh bool) (*kubernetes.MizuTapperSyncer, error) {
+func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, serviceMesh bool, tls bool) (*kubernetes.MizuTapperSyncer, error) {
 	tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
 		TargetNamespaces:         targetNamespaces,
 		PodFilterRegex:           podFilterRegex,
@@ -113,6 +113,7 @@ func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, t
 		MizuApiFilteringOptions:  mizuApiFilteringOptions,
 		MizuServiceAccountExists: true, //assume service account exists since install mode will not function without it anyway
 		ServiceMesh:              serviceMesh,
+		Tls:                      tls,
 	}, time.Now())
 
 	if err != nil {

agent/pkg/controllers/service_map_controller_test.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
-	"github.com/up9inc/mizu/shared"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
@@ -59,9 +58,7 @@ type ServiceMapControllerSuite struct {
 
 func (s *ServiceMapControllerSuite) SetupTest() {
 	s.c = NewServiceMapController()
-	s.c.service.SetConfig(&shared.MizuAgentConfig{
-		ServiceMap: true,
-	})
+	s.c.service.Enable()
 	s.c.service.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
 
 	s.w = httptest.NewRecorder()
@@ -102,13 +99,13 @@ func (s *ServiceMapControllerSuite) TestGet() {
 	// response nodes
 	aNode := servicemap.ServiceMapNode{
 		Id:    1,
-		Name:  TCPEntryA.IP,
+		Name:  TCPEntryA.Name,
 		Entry: TCPEntryA,
 		Count: 1,
 	}
 	bNode := servicemap.ServiceMapNode{
 		Id:    2,
-		Name:  TCPEntryB.IP,
+		Name:  TCPEntryB.Name,
 		Entry: TCPEntryB,
 		Count: 1,
 	}

agent/pkg/middlewares/cors.go

@@ -7,7 +7,7 @@ func CORSMiddleware() gin.HandlerFunc {
 		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
 		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
 		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With, x-session-token")
-		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")
+		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE")
 
 		if c.Request.Method == "OPTIONS" {
 			c.AbortWithStatus(204)

agent/pkg/models/models.go

@@ -42,6 +42,11 @@ type WebSocketEntryMessage struct {
 	Data *tapApi.BaseEntry `json:"data,omitempty"`
 }
 
+type WebSocketFullEntryMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *tapApi.Entry `json:"data,omitempty"`
+}
+
 type WebSocketTappedEntryMessage struct {
 	*shared.WebSocketMessageMetadata
 	Data *tapApi.OutputChannelItem
@@ -88,6 +93,16 @@ func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) {
 	return json.Marshal(message)
 }
 
+func CreateFullEntryWebSocketMessage(entry *tapApi.Entry) ([]byte, error) {
+	message := &WebSocketFullEntryMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeFullEntry,
+		},
+		Data: entry,
+	}
+	return json.Marshal(message)
+}
+
 func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte, error) {
 	message := &WebSocketTappedEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{

agent/pkg/oas/feeder_test.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"io"
 	"io/ioutil"
+	"net/url"
 	"os"
 	"path/filepath"
 	"sort"
@@ -139,7 +140,12 @@ func feedEntry(entry *har.Entry, source string, isSync bool, file string) {
 		logger.Log.Debugf("Interesting: %s", entry.Request.URL)
 	}
 
-	ews := EntryWithSource{Entry: *entry, Source: source, Id: uint(0)}
+	u, err := url.Parse(entry.Request.URL)
+	if err != nil {
+		logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
+	}
+
+	ews := EntryWithSource{Entry: *entry, Source: source, Destination: u.Host, Id: uint(0)}
 	if isSync {
 		GetOasGeneratorInstance().entriesChan <- ews // blocking variant, right?
 	} else {

agent/pkg/oas/oas_generator.go

@@ -54,11 +54,11 @@ func (g *oasGenerator) runGeneretor() {
 				logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
 			}
 
-			val, found := g.ServiceSpecs.Load(u.Host)
+			val, found := g.ServiceSpecs.Load(entryWithSource.Destination)
 			var gen *SpecGen
 			if !found {
-				gen = NewGen(u.Scheme + "://" + u.Host)
-				g.ServiceSpecs.Store(u.Host, gen)
+				gen = NewGen(u.Scheme + "://" + entryWithSource.Destination)
+				g.ServiceSpecs.Store(entryWithSource.Destination, gen)
 			} else {
 				gen = val.(*SpecGen)
 			}
@@ -105,9 +105,10 @@ func newOasGenerator() *oasGenerator {
 }
 
 type EntryWithSource struct {
-	Source string
-	Entry  har.Entry
-	Id     uint
+	Source      string
+	Destination string
+	Entry       har.Entry
+	Id          uint
 }
 
 type oasGenerator struct {

agent/pkg/oas/specgen_test.go

@@ -41,7 +41,7 @@ func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string {
 }
 
 func TestEntries(t *testing.T) {
-	logger.InitLoggerStderrOnly(logging.INFO)
+	logger.InitLoggerStd(logging.INFO)
 	files, err := getFiles("./test_artifacts/")
 	if err != nil {
 		t.Log(err)

agent/pkg/resolver/resolver.go

@@ -30,6 +30,11 @@ type Resolver struct {
 	namespace    string
 }
 
+type ResolvedObjectInfo struct {
+	FullAddress string
+	Namespace   string
+}
+
 func (resolver *Resolver) Start(ctx context.Context) {
 	if !resolver.isStarted {
 		resolver.isStarted = true
@@ -40,12 +45,12 @@ func (resolver *Resolver) Start(ctx context.Context) {
 	}
 }
 
-func (resolver *Resolver) Resolve(name string) string {
+func (resolver *Resolver) Resolve(name string) *ResolvedObjectInfo {
 	resolvedName, isFound := resolver.nameMap.Get(name)
 	if !isFound {
-		return ""
+		return nil
 	}
-	return resolvedName.(string)
+	return resolvedName.(*ResolvedObjectInfo)
 }
 
 func (resolver *Resolver) GetMap() cmap.ConcurrentMap {
@@ -71,7 +76,7 @@ func (resolver *Resolver) watchPods(ctx context.Context) error {
 			}
 			if event.Type == watch.Deleted {
 				pod := event.Object.(*corev1.Pod)
-				resolver.saveResolvedName(pod.Status.PodIP, "", event.Type)
+				resolver.saveResolvedName(pod.Status.PodIP, "", pod.Namespace, event.Type)
 			}
 		case <-ctx.Done():
 			watcher.Stop()
@@ -106,10 +111,10 @@ func (resolver *Resolver) watchEndpoints(ctx context.Context) error {
 					}
 					if subset.Addresses != nil {
 						for _, address := range subset.Addresses {
-							resolver.saveResolvedName(address.IP, serviceHostname, event.Type)
+							resolver.saveResolvedName(address.IP, serviceHostname, endpoint.Namespace, event.Type)
 							for _, port := range ports {
 								ipWithPort := fmt.Sprintf("%s:%d", address.IP, port)
-								resolver.saveResolvedName(ipWithPort, serviceHostname, event.Type)
+								resolver.saveResolvedName(ipWithPort, serviceHostname, endpoint.Namespace, event.Type)
 							}
 						}
 					}
@@ -139,19 +144,19 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
 			service := event.Object.(*corev1.Service)
 			serviceHostname := fmt.Sprintf("%s.%s", service.Name, service.Namespace)
 			if service.Spec.ClusterIP != "" && service.Spec.ClusterIP != kubClientNullString {
-				resolver.saveResolvedName(service.Spec.ClusterIP, serviceHostname, event.Type)
+				resolver.saveResolvedName(service.Spec.ClusterIP, serviceHostname, service.Namespace, event.Type)
 				if service.Spec.Ports != nil {
 					for _, port := range service.Spec.Ports {
 						if port.Port > 0 {
-							resolver.saveResolvedName(fmt.Sprintf("%s:%d", service.Spec.ClusterIP, port.Port), serviceHostname, event.Type)
+							resolver.saveResolvedName(fmt.Sprintf("%s:%d", service.Spec.ClusterIP, port.Port), serviceHostname, service.Namespace, event.Type)
 						}
 					}
 				}
-				resolver.saveServiceIP(service.Spec.ClusterIP, serviceHostname, event.Type)
+				resolver.saveServiceIP(service.Spec.ClusterIP, serviceHostname, service.Namespace, event.Type)
 			}
 			if service.Status.LoadBalancer.Ingress != nil {
 				for _, ingress := range service.Status.LoadBalancer.Ingress {
-					resolver.saveResolvedName(ingress.IP, serviceHostname, event.Type)
+					resolver.saveResolvedName(ingress.IP, serviceHostname, service.Namespace, event.Type)
 				}
 			}
 		case <-ctx.Done():
@@ -161,21 +166,22 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
 	}
 }
 
-func (resolver *Resolver) saveResolvedName(key string, resolved string, eventType watch.EventType) {
+func (resolver *Resolver) saveResolvedName(key string, resolved string, namespace string, eventType watch.EventType) {
 	if eventType == watch.Deleted {
 		resolver.nameMap.Remove(key)
 		logger.Log.Infof("setting %s=nil", key)
 	} else {
-		resolver.nameMap.Set(key, resolved)
+
+		resolver.nameMap.Set(key, &ResolvedObjectInfo{FullAddress: resolved, Namespace: namespace})
 		logger.Log.Infof("setting %s=%s", key, resolved)
 	}
 }
 
-func (resolver *Resolver) saveServiceIP(key string, resolved string, eventType watch.EventType) {
+func (resolver *Resolver) saveServiceIP(key string, resolved string, namespace string, eventType watch.EventType) {
 	if eventType == watch.Deleted {
 		resolver.serviceMap.Remove(key)
 	} else {
-		resolver.serviceMap.Set(key, resolved)
+		resolver.nameMap.Set(key, &ResolvedObjectInfo{FullAddress: resolved, Namespace: namespace})
 	}
 }
 

agent/pkg/servicemap/servicemap.go

@@ -3,7 +3,6 @@ package servicemap
 import (
 	"sync"
 
-	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
@@ -26,13 +25,13 @@ func GetInstance() ServiceMap {
 }
 
 type serviceMap struct {
-	config           *shared.MizuAgentConfig
+	enabled          bool
 	graph            *graph
 	entriesProcessed int
 }
 
 type ServiceMap interface {
-	SetConfig(config *shared.MizuAgentConfig)
+	Enable()
 	IsEnabled() bool
 	NewTCPEntry(source *tapApi.TCP, destination *tapApi.TCP, protocol *tapApi.Protocol)
 	GetStatus() ServiceMapStatus
@@ -46,7 +45,7 @@ type ServiceMap interface {
 
 func newServiceMap() *serviceMap {
 	return &serviceMap{
-		config:           nil,
+		enabled:          false,
 		entriesProcessed: 0,
 		graph:            newDirectedGraph(),
 	}
@@ -156,15 +155,12 @@ func (s *serviceMap) addEdge(u, v *entryData, p *tapApi.Protocol) {
 	s.entriesProcessed++
 }
 
-func (s *serviceMap) SetConfig(config *shared.MizuAgentConfig) {
-	s.config = config
+func (s *serviceMap) Enable() {
+	s.enabled = true
 }
 
 func (s *serviceMap) IsEnabled() bool {
-	if s.config != nil && s.config.ServiceMap {
-		return true
-	}
-	return false
+	return s.enabled
 }
 
 func (s *serviceMap) NewTCPEntry(src *tapApi.TCP, dst *tapApi.TCP, p *tapApi.Protocol) {
@@ -172,20 +168,33 @@ func (s *serviceMap) NewTCPEntry(src *tapApi.TCP, dst *tapApi.TCP, p *tapApi.Pro
 		return
 	}
 
-	srcEntry := &entryData{
-		key:   key(src.IP),
-		entry: src,
-	}
-	if len(srcEntry.entry.Name) == 0 {
+	var srcEntry *entryData
+	var dstEntry *entryData
+
+	if len(src.Name) == 0 {
+		srcEntry = &entryData{
+			key:   key(src.IP),
+			entry: src,
+		}
 		srcEntry.entry.Name = UnresolvedNodeName
+	} else {
+		srcEntry = &entryData{
+			key:   key(src.Name),
+			entry: src,
+		}
 	}
 
-	dstEntry := &entryData{
-		key:   key(dst.IP),
-		entry: dst,
-	}
-	if len(dstEntry.entry.Name) == 0 {
+	if len(dst.Name) == 0 {
+		dstEntry = &entryData{
+			key:   key(dst.IP),
+			entry: dst,
+		}
 		dstEntry.entry.Name = UnresolvedNodeName
+	} else {
+		dstEntry = &entryData{
+			key:   key(dst.Name),
+			entry: dst,
+		}
 	}
 
 	s.addEdge(srcEntry, dstEntry, p)

agent/pkg/servicemap/servicemap_test.go

@@ -6,7 +6,6 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/suite"
-	"github.com/up9inc/mizu/shared"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
@@ -96,9 +95,7 @@ func (s *ServiceMapDisabledSuite) SetupTest() {
 
 func (s *ServiceMapEnabledSuite) SetupTest() {
 	s.instance = GetInstance()
-	s.instance.SetConfig(&shared.MizuAgentConfig{
-		ServiceMap: true,
-	})
+	s.instance.Enable()
 }
 
 func (s *ServiceMapDisabledSuite) TestServiceMapInstance() {
@@ -268,9 +265,14 @@ func (s *ServiceMapEnabledSuite) TestServiceMap() {
 		assert.LessOrEqual(node.Id, expectedNodeCount)
 
 		// entry
-		// node.Name is the key of the node, key = entry.IP
+		// node.Name is the key of the node, key = entry.Name by default
 		// entry.Name is the name of the service and could be unresolved
-		assert.Equal(node.Name, node.Entry.IP)
+		// when entry.Name is unresolved, key = entry.IP
+		if node.Entry.Name == UnresolvedNodeName {
+			assert.Equal(node.Name, node.Entry.IP)
+		} else {
+			assert.Equal(node.Name, node.Entry.Name)
+		}
 		assert.Equal(Port, node.Entry.Port)
 		assert.Equal(entryName, node.Entry.Name)
 
@@ -320,16 +322,24 @@ func (s *ServiceMapEnabledSuite) TestServiceMap() {
 	cdEdge := -1
 	acEdge := -1
 	var validateEdge = func(edge ServiceMapEdge, sourceEntryName string, destEntryName string, protocolName string, protocolCount int) {
-		// source
+		// source node
 		assert.Contains(nodeIds, edge.Source.Id)
 		assert.LessOrEqual(edge.Source.Id, expectedNodeCount)
-		assert.Equal(edge.Source.Name, edge.Source.Entry.IP)
+		if edge.Source.Entry.Name == UnresolvedNodeName {
+			assert.Equal(edge.Source.Name, edge.Source.Entry.IP)
+		} else {
+			assert.Equal(edge.Source.Name, edge.Source.Entry.Name)
+		}
 		assert.Equal(sourceEntryName, edge.Source.Entry.Name)
 
-		// destination
+		// destination node
 		assert.Contains(nodeIds, edge.Destination.Id)
 		assert.LessOrEqual(edge.Destination.Id, expectedNodeCount)
-		assert.Equal(edge.Destination.Name, edge.Destination.Entry.IP)
+		if edge.Destination.Entry.Name == UnresolvedNodeName {
+			assert.Equal(edge.Destination.Name, edge.Destination.Entry.IP)
+		} else {
+			assert.Equal(edge.Destination.Name, edge.Destination.Entry.Name)
+		}
 		assert.Equal(destEntryName, edge.Destination.Entry.Name)
 
 		// protocol

cli/Makefile

@@ -1,3 +1,9 @@
+SHELL=/bin/bash
+
+.PHONY: help
+.DEFAULT_GOAL := help
+.ONESHELL:
+
 SUFFIX=$(GOOS)_$(GOARCH)
 COMMIT_HASH=$(shell git rev-parse HEAD)
 GIT_BRANCH=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
@@ -5,9 +11,6 @@ GIT_VERSION=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
 BUILD_TIMESTAMP=$(shell date +%s)
 export VER?=0.0
 
-.PHONY: help
-.DEFAULT_GOAL := help
-
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
 
@@ -16,15 +19,20 @@ install:
 
 build-debug:  ## Build mizu CLI for debug
 	export GCLFAGS='-gcflags="all=-N -l"'
-	${MAKE} build
+	${MAKE} build-base
 
-build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
-	go build ${GCLFAGS} -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
-					   -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
-					   -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
-					   -X 'github.com/up9inc/mizu/cli/mizu.Platform=$(SUFFIX)' \
-					   -X 'github.com/up9inc/mizu/cli/mizu.Ver=$(VER)'" \
-					   -o bin/mizu_$(SUFFIX) mizu.go
+build:
+	export LDFLAGS_EXT='-s -w'
+	${MAKE} build-base
+
+build-base: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables).
+	go build ${GCLFAGS} -ldflags="${LDFLAGS_EXT} \
+					-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
+					-X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
+					-X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
+					-X 'github.com/up9inc/mizu/cli/mizu.Platform=$(SUFFIX)' \
+					-X 'github.com/up9inc/mizu/cli/mizu.Ver=$(VER)'" \
+					-o bin/mizu_$(SUFFIX) mizu.go
 	(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
 
 build-all: ## Build for all supported platforms.

cli/README.md.TEMPLATE

@@ -1,5 +1,5 @@
 # Mizu release _VER_
-Full changelog for stable release see in [docs](https://github.com/up9inc/mizu/blob/main/docs/CHANGELOG.md)
+Mizu CHANGELOG is now part of [Mizu wiki](https://github.com/up9inc/mizu/wiki/CHANGELOG)
 
 ## Download Mizu for your platform
 
@@ -10,7 +10,7 @@ curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_darwin
 
 **Mac** (AArch64/Apple M1 silicon)
 ```
-curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_darwin_arm64 && chmod 755 mizu
+rm -f mizu && curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_darwin_arm64 && chmod 755 mizu
 ```
 
 **Linux** (x86-64)

cli/apiserver/provider.go

@@ -4,9 +4,11 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/up9inc/mizu/shared/kubernetes"
@@ -57,10 +59,8 @@ func (provider *Provider) TestConnection() error {
 
 func (provider *Provider) isReachable() (bool, error) {
 	echoUrl := fmt.Sprintf("%s/echo", provider.url)
-	if response, err := provider.client.Get(echoUrl); err != nil {
+	if _, err := provider.get(echoUrl); err != nil {
 		return false, err
-	} else if response.StatusCode != 200 {
-		return false, fmt.Errorf("invalid status code %v", response.StatusCode)
 	} else {
 		return true, nil
 	}
@@ -72,10 +72,8 @@ func (provider *Provider) ReportTapperStatus(tapperStatus shared.TapperStatus) e
 	if jsonValue, err := json.Marshal(tapperStatus); err != nil {
 		return fmt.Errorf("failed Marshal the tapper status %w", err)
 	} else {
-		if response, err := provider.client.Post(tapperStatusUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+		if _, err := provider.post(tapperStatusUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
 			return fmt.Errorf("failed sending to API server the tapped pods %w", err)
-		} else if response.StatusCode != 200 {
-			return fmt.Errorf("failed sending to API server the tapper status, response status code %v", response.StatusCode)
 		} else {
 			logger.Log.Debugf("Reported to server API about tapper status: %v", tapperStatus)
 			return nil
@@ -91,10 +89,8 @@ func (provider *Provider) ReportTappedPods(pods []core.Pod) error {
 	if jsonValue, err := json.Marshal(podInfos); err != nil {
 		return fmt.Errorf("failed Marshal the tapped pods %w", err)
 	} else {
-		if response, err := provider.client.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+		if _, err := provider.post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
 			return fmt.Errorf("failed sending to API server the tapped pods %w", err)
-		} else if response.StatusCode != 200 {
-			return fmt.Errorf("failed sending to API server the tapped pods, response status code %v", response.StatusCode)
 		} else {
 			logger.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos))
 			return nil
@@ -105,11 +101,9 @@ func (provider *Provider) ReportTappedPods(pods []core.Pod) error {
 func (provider *Provider) GetGeneralStats() (map[string]interface{}, error) {
 	generalStatsUrl := fmt.Sprintf("%s/status/general", provider.url)
 
-	response, requestErr := provider.client.Get(generalStatsUrl)
+	response, requestErr := provider.get(generalStatsUrl)
 	if requestErr != nil {
 		return nil, fmt.Errorf("failed to get general stats for telemetry, err: %w", requestErr)
-	} else if response.StatusCode != 200 {
-		return nil, fmt.Errorf("failed to get general stats for telemetry, status code: %v", response.StatusCode)
 	}
 
 	defer response.Body.Close()
@@ -132,7 +126,7 @@ func (provider *Provider) GetVersion() (string, error) {
 		Method: http.MethodGet,
 		URL:    versionUrl,
 	}
-	statusResp, err := provider.client.Do(req)
+	statusResp, err := provider.do(req)
 	if err != nil {
 		return "", err
 	}
@@ -145,3 +139,40 @@ func (provider *Provider) GetVersion() (string, error) {
 
 	return versionResponse.Ver, nil
 }
+
+// When err is nil, resp always contains a non-nil resp.Body.
+// Caller should close resp.Body when done reading from it.
+func (provider *Provider) get(url string) (*http.Response, error) {
+	return provider.checkError(provider.client.Get(url))
+}
+
+// When err is nil, resp always contains a non-nil resp.Body.
+// Caller should close resp.Body when done reading from it.
+func (provider *Provider) post(url, contentType string, body io.Reader) (*http.Response, error) {
+	return provider.checkError(provider.client.Post(url, contentType, body))
+}
+
+// When err is nil, resp always contains a non-nil resp.Body.
+// Caller should close resp.Body when done reading from it.
+func (provider *Provider) do(req *http.Request) (*http.Response, error) {
+	return provider.checkError(provider.client.Do(req))
+}
+
+func (provider *Provider) checkError(response *http.Response, errInOperation error) (*http.Response, error) {
+	if (errInOperation != nil) {
+		return response, errInOperation
+	// Check only if status != 200 (and not status >= 300). Agent APIs return only 200 on success.
+	} else if response.StatusCode != http.StatusOK {
+		body, err := ioutil.ReadAll(response.Body)
+		response.Body.Close()
+		response.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
+		if err != nil {
+			return response, err
+		}
+
+		errorMsg := strings.ReplaceAll((string(body)), "\n", ";")
+		return response, fmt.Errorf("got response with status code: %d, body: %s", response.StatusCode, errorMsg)
+	}
+
+	return response, nil
+}

cli/cmd/check.go

@@ -1,8 +1,11 @@
 package cmd
 
 import (
+	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
+	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/telemetry"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var checkCmd = &cobra.Command{
@@ -17,4 +20,11 @@ var checkCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(checkCmd)
+
+	defaultCheckConfig := configStructs.CheckConfig{}
+	if err := defaults.Set(&defaultCheckConfig); err != nil {
+		logger.Log.Debug(err)
+	}
+
+	checkCmd.Flags().Bool(configStructs.PreTapCheckName, defaultCheckConfig.PreTap, "Check pre-tap Mizu installation for potential problems")
 }

cli/cmd/checkRunner.go

@@ -2,7 +2,11 @@ package cmd
 
 import (
 	"context"
+	"embed"
 	"fmt"
+	rbac "k8s.io/api/rbac/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/scheme"
 	"regexp"
 
 	"github.com/up9inc/mizu/cli/apiserver"
@@ -13,8 +17,13 @@ import (
 	"github.com/up9inc/mizu/shared/semver"
 )
 
+var (
+	//go:embed permissionFiles
+	embedFS embed.FS
+)
+
 func runMizuCheck() {
-	logger.Log.Infof("Mizu install checks\n===================")
+	logger.Log.Infof("Mizu checks\n===================")
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel() // cancel will be called when this function exits
@@ -25,17 +34,18 @@ func runMizuCheck() {
 		checkPassed = checkKubernetesVersion(kubernetesVersion)
 	}
 
-	var isInstallCommand bool
-	if checkPassed {
-		checkPassed, isInstallCommand = checkMizuMode(ctx, kubernetesProvider)
-	}
+	if config.Config.Check.PreTap {
+		if checkPassed {
+			checkPassed = checkK8sTapPermissions(ctx, kubernetesProvider)
+		}
+	} else {
+		if checkPassed {
+			checkPassed = checkK8sResources(ctx, kubernetesProvider)
+		}
 
-	if checkPassed {
-		checkPassed = checkK8sResources(ctx, kubernetesProvider, isInstallCommand)
-	}
-
-	if checkPassed {
-		checkPassed = checkServerConnection(kubernetesProvider)
+		if checkPassed {
+			checkPassed = checkServerConnection(kubernetesProvider)
+		}
 	}
 
 	if checkPassed {
@@ -65,27 +75,6 @@ func checkKubernetesApi() (*kubernetes.Provider, *semver.SemVersion, bool) {
 	return kubernetesProvider, kubernetesVersion, true
 }
 
-func checkMizuMode(ctx context.Context, kubernetesProvider *kubernetes.Provider) (bool, bool) {
-	logger.Log.Infof("\nmode\n--------------------")
-
-	if exist, err := kubernetesProvider.DoesDeploymentExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
-		logger.Log.Errorf("%v can't check mizu command, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
-		return false, false
-	} else if exist {
-		logger.Log.Infof("%v mizu running with install command", fmt.Sprintf(uiUtils.Green, "√"))
-		return true, true
-	} else if exist, err = kubernetesProvider.DoesPodExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
-		logger.Log.Errorf("%v can't check mizu command, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
-		return false, false
-	} else if exist {
-		logger.Log.Infof("%v mizu running with tap command", fmt.Sprintf(uiUtils.Green, "√"))
-		return true, false
-	} else {
-		logger.Log.Infof("%v mizu is not running", fmt.Sprintf(uiUtils.Red, "✗"))
-		return false, false
-	}
-}
-
 func checkKubernetesVersion(kubernetesVersion *semver.SemVersion) bool {
 	logger.Log.Infof("\nkubernetes-version\n--------------------")
 
@@ -169,7 +158,7 @@ func checkPortForward(serverUrl string, kubernetesProvider *kubernetes.Provider)
 	return nil
 }
 
-func checkK8sResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, isInstallCommand bool) bool {
+func checkK8sResources(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
 	logger.Log.Infof("\nk8s-components\n--------------------")
 
 	exist, err := kubernetesProvider.DoesNamespaceExist(ctx, config.Config.MizuResourcesNamespace)
@@ -198,52 +187,27 @@ func checkK8sResources(ctx context.Context, kubernetesProvider *kubernetes.Provi
 	exist, err = kubernetesProvider.DoesServiceExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
 	allResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "service", exist, err) && allResourcesExist
 
-	if isInstallCommand {
-		allResourcesExist = checkInstallResourcesExist(ctx, kubernetesProvider) && allResourcesExist
-	} else {
-		allResourcesExist = checkTapResourcesExist(ctx, kubernetesProvider) && allResourcesExist
-	}
+	allResourcesExist = checkPodResourcesExist(ctx, kubernetesProvider) && allResourcesExist
 
 	return allResourcesExist
 }
 
-func checkInstallResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
-	exist, err := kubernetesProvider.DoesRoleExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleName)
-	installResourcesExist := checkResourceExist(kubernetes.DaemonRoleName, "role", exist, err)
-
-	exist, err = kubernetesProvider.DoesRoleBindingExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleBindingName)
-	installResourcesExist = checkResourceExist(kubernetes.DaemonRoleBindingName, "role binding", exist, err) && installResourcesExist
-
-	exist, err = kubernetesProvider.DoesPersistentVolumeClaimExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName)
-	installResourcesExist = checkResourceExist(kubernetes.PersistentVolumeClaimName, "persistent volume claim", exist, err) && installResourcesExist
-
-	exist, err = kubernetesProvider.DoesDeploymentExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
-	installResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "deployment", exist, err) && installResourcesExist
-
-	return installResourcesExist
-}
-
-func checkTapResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
-	exist, err := kubernetesProvider.DoesPodExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
-	tapResourcesExist := checkResourceExist(kubernetes.ApiServerPodName, "pod", exist, err)
-
-	if !tapResourcesExist {
+func checkPodResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
+	if pods, err := kubernetesProvider.ListPodsByAppLabel(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
+		logger.Log.Errorf("%v error checking if '%v' pod is running, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), kubernetes.ApiServerPodName, err)
 		return false
-	}
-
-	if pod, err := kubernetesProvider.GetPod(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
-		logger.Log.Errorf("%v error checking if '%v' pod exists, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), kubernetes.ApiServerPodName, err)
+	} else if len(pods) == 0 {
+		logger.Log.Errorf("%v '%v' pod doesn't exist", fmt.Sprintf(uiUtils.Red, "✗"), kubernetes.ApiServerPodName)
 		return false
-	} else if kubernetes.IsPodRunning(pod) {
-		logger.Log.Infof("%v '%v' pod running", fmt.Sprintf(uiUtils.Green, "√"), kubernetes.ApiServerPodName)
-	} else {
+	} else if !kubernetes.IsPodRunning(&pods[0]) {
 		logger.Log.Errorf("%v '%v' pod not running", fmt.Sprintf(uiUtils.Red, "✗"), kubernetes.ApiServerPodName)
 		return false
 	}
 
-	tapperRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", kubernetes.TapperPodName))
-	if pods, err := kubernetesProvider.ListAllPodsMatchingRegex(ctx, tapperRegex, []string{config.Config.MizuResourcesNamespace}); err != nil {
-		logger.Log.Errorf("%v error listing '%v' pods, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), kubernetes.TapperPodName, err)
+	logger.Log.Infof("%v '%v' pod running", fmt.Sprintf(uiUtils.Green, "√"), kubernetes.ApiServerPodName)
+
+	if pods, err := kubernetesProvider.ListPodsByAppLabel(ctx, config.Config.MizuResourcesNamespace, kubernetes.TapperPodName); err != nil {
+		logger.Log.Errorf("%v error checking if '%v' pods are running, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), kubernetes.TapperPodName, err)
 		return false
 	} else {
 		tappers := 0
@@ -256,7 +220,7 @@ func checkTapResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.
 			}
 		}
 
-		if tappers != notRunningTappers {
+		if notRunningTappers > 0 {
 			logger.Log.Errorf("%v '%v' %v/%v pods are not running", fmt.Sprintf(uiUtils.Red, "✗"), kubernetes.TapperPodName, notRunningTappers, tappers)
 			return false
 		}
@@ -273,9 +237,81 @@ func checkResourceExist(resourceName string, resourceType string, exist bool, er
 	} else if !exist {
 		logger.Log.Errorf("%v '%v' %v doesn't exist", fmt.Sprintf(uiUtils.Red, "✗"), resourceName, resourceType)
 		return false
-	} else {
-		logger.Log.Infof("%v '%v' %v exists", fmt.Sprintf(uiUtils.Green, "√"), resourceName, resourceType)
 	}
 
+	logger.Log.Infof("%v '%v' %v exists", fmt.Sprintf(uiUtils.Green, "√"), resourceName, resourceType)
+	return true
+}
+
+func checkK8sTapPermissions(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
+	logger.Log.Infof("\nkubernetes-permissions\n--------------------")
+
+	var filePath string
+	if config.Config.IsNsRestrictedMode() {
+		filePath = "permissionFiles/permissions-ns-tap.yaml"
+	} else {
+		filePath = "permissionFiles/permissions-all-namespaces-tap.yaml"
+	}
+
+	data, err := embedFS.ReadFile(filePath)
+	if err != nil {
+		logger.Log.Errorf("%v error while checking kubernetes permissions, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+		return false
+	}
+
+	obj, err := getDecodedObject(data)
+	if err != nil {
+		logger.Log.Errorf("%v error while checking kubernetes permissions, err: %v", fmt.Sprintf(uiUtils.Red, "✗"), err)
+		return false
+	}
+
+	var rules []rbac.PolicyRule
+	if config.Config.IsNsRestrictedMode() {
+		rules = obj.(*rbac.Role).Rules
+	} else {
+		rules = obj.(*rbac.ClusterRole).Rules
+	}
+
+	return checkPermissions(ctx, kubernetesProvider, rules)
+}
+
+func getDecodedObject(data []byte) (runtime.Object, error) {
+	decode := scheme.Codecs.UniversalDeserializer().Decode
+
+	obj, _, err := decode(data, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return obj, nil
+}
+
+func checkPermissions(ctx context.Context, kubernetesProvider *kubernetes.Provider, rules []rbac.PolicyRule) bool {
+	permissionsExist := true
+
+	for _, rule := range rules {
+		for _, group := range rule.APIGroups {
+			for _, resource := range rule.Resources {
+				for _, verb := range rule.Verbs {
+					exist, err := kubernetesProvider.CanI(ctx, config.Config.MizuResourcesNamespace, resource, verb, group)
+					permissionsExist = checkPermissionExist(group, resource, verb, exist, err) && permissionsExist
+				}
+			}
+		}
+	}
+
+	return permissionsExist
+}
+
+func checkPermissionExist(group string, resource string, verb string, exist bool, err error) bool {
+	if err != nil {
+		logger.Log.Errorf("%v error checking permission for %v %v in group '%v', err: %v", fmt.Sprintf(uiUtils.Red, "✗"), verb, resource, group, err)
+		return false
+	} else if !exist {
+		logger.Log.Errorf("%v can't %v %v in group '%v'", fmt.Sprintf(uiUtils.Red, "✗"), verb, resource, group)
+		return false
+	}
+
+	logger.Log.Infof("%v can %v %v in group '%v'", fmt.Sprintf(uiUtils.Green, "√"), verb, resource, group)
 	return true
 }

cli/cmd/install.go

@@ -1,10 +1,9 @@
 package cmd
 
 import (
-	"fmt"
 	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/telemetry"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var installCmd = &cobra.Command{
@@ -12,13 +11,13 @@ var installCmd = &cobra.Command{
 	Short: "Installs mizu components",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		go telemetry.ReportRun("install", nil)
-		runMizuInstall()
-		return nil
-	},
-	PreRunE: func(cmd *cobra.Command, args []string) error {
-		if config.Config.IsNsRestrictedMode() {
-			return fmt.Errorf("install is not supported in restricted namespace mode")
-		}
+		logger.Log.Infof("This command has been deprecated, please use helm as described below.\n\n")
+
+		logger.Log.Infof("To install stable build of Mizu on your cluster using helm, run the following command:")
+		logger.Log.Infof("    helm install mizu up9mizu --repo https://static.up9.com/mizu/helm --namespace=mizu --create-namespace\n\n")
+
+		logger.Log.Infof("To install development build of Mizu on your cluster using helm, run the following command:")
+		logger.Log.Infof("    helm install mizu up9mizu --repo https://static.up9.com/mizu/helm-develop --namespace=mizu --create-namespace\n")
 
 		return nil
 	},
@@ -27,4 +26,3 @@ var installCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(installCmd)
 }
-

cli/cmd/installRunner.go

@@ -1,81 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"errors"
-	"fmt"
-
-	"github.com/creasty/defaults"
-	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/cli/errormessage"
-	"github.com/up9inc/mizu/cli/resources"
-	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-func runMizuInstall() {
-	kubernetesProvider, err := getKubernetesProviderForCli()
-	if err != nil {
-		return
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel() // cancel will be called when this function exits
-
-	var serializedValidationRules string
-	var serializedContract string
-
-	var defaultMaxEntriesDBSizeBytes int64 = 200 * 1000 * 1000
-
-	defaultResources := shared.Resources{}
-	if err := defaults.Set(&defaultResources); err != nil {
-		logger.Log.Debug(err)
-	}
-
-	mizuAgentConfig := getInstallMizuAgentConfig(defaultMaxEntriesDBSizeBytes, defaultResources)
-	serializedMizuConfig, err := getSerializedMizuAgentConfig(mizuAgentConfig)
-	if err != nil {
-		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error serializing mizu config: %v", errormessage.FormatError(err)))
-		return
-	}
-
-	if err = resources.CreateInstallMizuResources(ctx, kubernetesProvider, serializedValidationRules,
-		serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(),
-		config.Config.MizuResourcesNamespace, config.Config.AgentImage,
-		config.Config.KratosImage, config.Config.KetoImage,
-		nil, defaultMaxEntriesDBSizeBytes, defaultResources, config.Config.ImagePullPolicy(),
-		config.Config.LogLevel(), false); err != nil {
-		var statusError *k8serrors.StatusError
-		if errors.As(err, &statusError) && (statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists) {
-			logger.Log.Info("Mizu is already running in this namespace, run `mizu clean` to remove the currently running Mizu instance")
-		} else {
-			defer resources.CleanUpMizuResources(ctx, cancel, kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
-		}
-
-		return
-	}
-
-	logger.Log.Infof(uiUtils.Magenta, "Installation completed, run `mizu view` to connect to the mizu daemon instance")
-}
-
-func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Resources) *shared.MizuAgentConfig {
-	mizuAgentConfig := shared.MizuAgentConfig{
-		MaxDBSizeBytes:         maxDBSizeBytes,
-		AgentImage:             config.Config.AgentImage,
-		PullPolicy:             config.Config.ImagePullPolicyStr,
-		LogLevel:               config.Config.LogLevel(),
-		TapperResources:        tapperResources,
-		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
-		AgentDatabasePath:      shared.DataDirPath,
-		StandaloneMode:         true,
-		ServiceMap:             config.Config.ServiceMap,
-		OAS:                    config.Config.OAS,
-		Elastic:                config.Config.Elastic,
-	}
-
-	return &mizuAgentConfig
-}

cli/cmd/permissionFiles/permissions-all-namespaces-debug-optional.yaml

@@ -0,0 +1,25 @@
+# This example shows permissions that enrich the logs with additional info
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-debug-clusterrole
+rules:
+- apiGroups: ["events.k8s.io"]
+  resources: ["events"]
+  verbs: ["watch"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-debug-clusterrolebindings
+subjects:
+- kind: User
+  name: user1
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: mizu-runner-debug-clusterrole
+  apiGroup: rbac.authorization.k8s.io

cli/cmd/permissionFiles/permissions-all-namespaces-ip-resolution-optional.yaml

@@ -0,0 +1,37 @@
+# This example shows permissions that are required for Mizu to resolve IPs to service names
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-resolver-clusterrole
+rules:
+- apiGroups: [""]
+  resources: ["serviceaccounts"]
+  verbs: ["get", "create"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["clusterroles"]
+  verbs: ["get", "list", "create", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["clusterrolebindings"]
+  verbs: ["get", "list", "create", "delete"]
+- apiGroups: ["", "apps", "extensions"]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["", "apps", "extensions"]
+  resources: ["services"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["", "apps", "extensions"]
+  resources: ["endpoints"]
+  verbs: ["get", "list", "watch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-resolver-clusterrolebindings
+subjects:
+- kind: User
+  name: user1
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: mizu-resolver-clusterrole
+  apiGroup: rbac.authorization.k8s.io

cli/cmd/permissionFiles/permissions-all-namespaces-tap.yaml

@@ -1,5 +1,4 @@
-# This example shows the roles required for a user to be able to use Mizu in all namespaces with IP resolution disabled.
-# (Traffic will be recorded, but Mizu will not translate IP addresses to names)
+# This example shows the permissions that are required in order to run the `mizu tap` command
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -7,25 +6,22 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["pods"]
-  verbs: ["list", "watch", "create", "delete"]
+  verbs: ["list", "watch", "create"]
 - apiGroups: [""]
   resources: ["services"]
-  verbs: ["create", "delete"]
+  verbs: ["get", "create"]
 - apiGroups: ["apps"]
   resources: ["daemonsets"]
-  verbs: ["create", "patch", "delete"]
+  verbs: ["create", "patch"]
 - apiGroups: [""]
   resources: ["namespaces"]
-  verbs: ["get", "list", "watch", "create", "delete"]
+  verbs: ["list", "watch", "create", "delete"]
 - apiGroups: [""]
   resources: ["services/proxy"]
-  verbs: ["get"]
+  verbs: ["get", "create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  verbs: ["get", "create", "delete"]
-  - apiGroups: ["events.k8s.io"]
-    resources: ["events"]
-    verbs: ["list", "watch"]
+  verbs: ["create"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

cli/cmd/permissionFiles/permissions-ns-debug-optional.yaml

@@ -0,0 +1,27 @@
+# This example shows permissions that enrich the logs with additional info in namespace-restricted mode
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-debug-role
+  namespace: user1
+rules:
+- apiGroups: ["events.k8s.io"]
+  resources: ["events"]
+  verbs: ["watch"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-runner-debug-rolebindings
+  namespace: user1
+subjects:
+- kind: User
+  name: user1
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: Role
+  name: mizu-runner-debug-role
+  apiGroup: rbac.authorization.k8s.io

cli/cmd/permissionFiles/permissions-ns-ip-resolution-optional.yaml

@@ -0,0 +1,39 @@
+# This example shows permissions that are required for Mizu to resolve IPs to service names in namespace-restricted mode
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-resolver-role
+  namespace: user1
+rules:
+- apiGroups: [""]
+  resources: ["serviceaccounts"]
+  verbs: ["get", "list", "create", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["roles"]
+  verbs: ["get", "list", "create", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["rolebindings"]
+  verbs: ["get", "list", "create", "delete"]
+- apiGroups: ["", "apps", "extensions"]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["", "apps", "extensions"]
+  resources: ["services"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["", "apps", "extensions"]
+  resources: ["endpoints"]
+  verbs: ["get", "list", "watch"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: mizu-resolver-rolebindings
+  namespace: user1
+subjects:
+- kind: User
+  name: user1
+  apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: Role
+  name: mizu-resolver-role
+  apiGroup: rbac.authorization.k8s.io

cli/cmd/permissionFiles/permissions-ns-tap.yaml

@@ -1,4 +1,4 @@
-# This example shows the roles required for a user to be able to use Mizu in a single namespace with IP resolution disabled.
+# This example shows the permissions that are required in order to run the `mizu tap` command in namespace-restricted mode
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -7,22 +7,19 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["pods"]
-  verbs: ["get", "list", "watch", "create", "delete"]
+  verbs: ["list", "watch", "create"]
 - apiGroups: [""]
   resources: ["services"]
   verbs: ["get", "create", "delete"]
 - apiGroups: ["apps"]
   resources: ["daemonsets"]
-  verbs: ["get", "create", "patch", "delete"]
+  verbs: ["create", "patch", "delete"]
 - apiGroups: [""]
   resources: ["services/proxy"]
-  verbs: ["get"]
+  verbs: ["get", "create", "delete"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["events.k8s.io"]
-  resources: ["events"]
-  verbs: ["list", "watch"]
+  verbs: ["create", "delete"]
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

cli/cmd/tap.go

@@ -120,4 +120,5 @@ func init() {
 	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules")
 	tapCmd.Flags().String(configStructs.ContractFile, defaultTapConfig.ContractFile, "OAS/Swagger file to validate to monitor the contracts")
 	tapCmd.Flags().Bool(configStructs.ServiceMeshName, defaultTapConfig.ServiceMesh, "Record decrypted traffic if the cluster is configured with a service mesh and with mtls")
+	tapCmd.Flags().Bool(configStructs.TlsName, defaultTapConfig.Tls, "Record tls traffic")
 }

cli/cmd/tapRunner.go

@@ -162,6 +162,7 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 		AgentDatabasePath:      shared.DataDirPath,
 		ServiceMap:             config.Config.ServiceMap,
 		OAS:                    config.Config.OAS,
+		Telemetry:              config.Config.Telemetry,
 		Elastic:                config.Config.Elastic,
 	}
 
@@ -200,6 +201,7 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider
 		MizuApiFilteringOptions:  mizuApiFilteringOptions,
 		MizuServiceAccountExists: state.mizuServiceAccountExists,
 		ServiceMesh:              config.Config.Tap.ServiceMesh,
+		Tls:                      config.Config.Tap.Tls,
 	}, startTime)
 
 	if err != nil {

cli/cmd/viewRunner.go

@@ -3,13 +3,13 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"github.com/up9inc/mizu/cli/utils"
 	"net/http"
 
+	"github.com/up9inc/mizu/cli/utils"
+
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
-	"github.com/up9inc/mizu/cli/mizu/version"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared/kubernetes"
 	"github.com/up9inc/mizu/shared/logger"
@@ -62,14 +62,5 @@ func runMizuView() {
 		uiUtils.OpenBrowser(url)
 	}
 
-	if isCompatible, err := version.CheckVersionCompatibility(apiServerProvider); err != nil {
-		logger.Log.Errorf("Failed to check versions compatibility %v", err)
-		cancel()
-		return
-	} else if !isCompatible {
-		cancel()
-		return
-	}
-
 	utils.WaitForFinish(ctx, cancel)
 }

cli/config/configStruct.go

@@ -22,6 +22,7 @@ const (
 
 type ConfigStruct struct {
 	Tap                    configStructs.TapConfig     `yaml:"tap"`
+	Check                  configStructs.CheckConfig   `yaml:"check"`
 	Version                configStructs.VersionConfig `yaml:"version"`
 	View                   configStructs.ViewConfig    `yaml:"view"`
 	Logs                   configStructs.LogsConfig    `yaml:"logs"`
@@ -38,7 +39,7 @@ type ConfigStruct struct {
 	ConfigFilePath         string                      `yaml:"config-path,omitempty" readonly:""`
 	HeadlessMode           bool                        `yaml:"headless" default:"false"`
 	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
-	ServiceMap             bool                        `yaml:"service-map" default:"false"`
+	ServiceMap             bool                        `yaml:"service-map" default:"true"`
 	OAS                    bool                        `yaml:"oas,omitempty" default:"false" readonly:""`
 	Elastic                shared.ElasticConfig        `yaml:"elastic"`
 }

cli/config/configStructs/checkConfig.go

@@ -0,0 +1,9 @@
+package configStructs
+
+const (
+	PreTapCheckName = "pre-tap"
+)
+
+type CheckConfig struct {
+	PreTap bool `yaml:"pre-tap"`
+}

cli/config/configStructs/tapConfig.go

@@ -23,6 +23,7 @@ const (
 	EnforcePolicyFile             = "traffic-validation-file"
 	ContractFile                  = "contract"
 	ServiceMeshName               = "service-mesh"
+	TlsName                       = "tls"
 )
 
 type TapConfig struct {
@@ -45,6 +46,7 @@ type TapConfig struct {
 	ApiServerResources      shared.Resources `yaml:"api-server-resources"`
 	TapperResources         shared.Resources `yaml:"tapper-resources"`
 	ServiceMesh             bool             `yaml:"service-mesh" default:"false"`
+	Tls                     bool             `yaml:"tls" default:"false"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {

docs/CHANGELOG.md

@@ -1,72 +1 @@
-# CHANGELOG
-This document summarizes main and fixes changes published in stable (aka `main`) branch of this project.
-Ongoing work and development releases are under `develop` branch.
-
-## 0.24.0
-
-### main features
-* ARM64 support -- Mizu is now available for ARM 64bit architecture
-  * Now you can run Mizu with `minikube` on your Apple M1 laptop or any other ARM-based hosts
-* New command helps user verify Mizu deployment
-  * Run `mizu check` to verify Mizu was deployed successfully
-  * `mizu check` verifies version compatibility, resources and permissions required by Mizu
-* EXPERIMENTAL: Service Map - graph of all service interactions
-  * Arrow direction show client to server connection
-  * Graph edge width reflects volume of traffic captured between the services
-  * to enable this experimental feature use `--set service-map=true` flag
-
-### improvements
-* Mizu container images are now served from [Docker Hub](https://hub.docker.com/r/up9inc/mizu), as multi-architecture images (arm64, amd64)
-* in Mizu GUI the filter query can now be applied by pressing CONTROL/COMMAND + ENTER
-* try port-forwarding if http-proxy connection to Mizu API server is not available
-
-### notable bug fixes
-* Fixed HTTP/1.0 presentation which was shown as HTTP/1.1
-* Fixed handling of long-living TCP connections, improves capturing gRPC and HTTP/2 traffic, and helps in service-mesh setups (istio, linkerd)
-
-
-## 0.23.0
-### notable bug fixes
-* fixed errors in Redis protocol parser (better handling of Array and Bulk String message types)
-
-
-
-## 0.22.0
-
-### main features
-* Service Mesh support -- mizu is now capable to tap mTLS traffic between pods connected by Istio service mesh
-  * Use `--service-mesh` option to enable this feature
-* New installation option - have the same Mizu functionality as long living pods in your cluster, with password protection
-  * To install use `mizu install` command
-  * To access use `mizu view` or `kubectl -n mizu port-forward svc/mizu-api-server`
-  * To uninstall run `mizu clean`
-* At first login
-  * Set admin password as prompted, use it to login to mizu later on.
-  * After login, user should select cluster namespaces to tap: by default all namespaces in the cluster are selected, user can select/unselect according to their needs. These settings are retained and can be modified at any time via Settings menu (cog icon on the top-right)
-
-
-### improvements
-* improved Mizu permissions/roles logic to support clusters with strict PodSecurityPolicy (PSP) -- see [PERMISSIONS](PERMISSIONS.md) doc for more details
- 
-### notable bug fixes
-* mizu now works properly when API service is exposed via HTTPS url
-* mizu now properly displays KAFKA message body 
-
-
-
-
-## 0.21.0
-
-### main features
-* New traffic search & stream exprience
-* Rich query language with full-text search capabilities on headers & body
-* Distinct live-streaming vs paging/browsing modes, all with filter applied
-
-### improvements
-* GUI - source and destination IP addresses & service names for each traffic item
-* GUI - Mizu health - display warning sign in top bar when not all requested pods are successfully tapped
-* GUI - pod tapping status reflected in the list (ok or problem)
-* Mizu telemetry - report platform type
-
-### fixes
-* Request duration and body size properly shown in GUI (instead of -1)
+Mizu CHANGELOG is now part of [Mizu wiki](https://github.com/up9inc/mizu/wiki/CHANGELOG)

docs/PERMISSIONS.md

@@ -80,327 +80,9 @@ Notes:
 
 ## List of permissions
 
-We broke down this list into few categories:
+The permissions that are required to run Mizu depend on the configuration.
+By default Mizu requires cluster-wide permissions.
+If these are not available to the user, it is possible to run Mizu in namespace-restricted mode which has a reduced set of requirements.
+This is done by by setting the `mizu-resources-namespace` config option. See [configuration](CONFIGURATION.md) for instructions.
 
-- Required - what is needed for `mizu` to run properly on your k8s cluster
-- Optional - permissions needed for proper name resolving for service & pod IPs
-  - addition required for policy validation
-
-### Required permissions
-
-Mizu needs following permissions on your Kubernetes cluster to run properly
-
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - create
-  - delete
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - create
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - namespaces
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services/proxy
-  verbs:
-  - get
-```
-
-#### Permissions required running with install command or (optional) for service / pod name resolving
-
-Mandatory permissions for running with install command.
-
-Optional for service/pod name resolving in non install standalone
-
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - create
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - namespaces
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services/proxy
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - serviceaccounts
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - clusterroles
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - clusterrolebindings
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - roles
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - rolebindings
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  - apps
-  - extensions
-  resources:
-  - endpoints
-  verbs:
-  - get
-  - list
-  - watch
-```
-
-#### Permissions for Policy rules validation feature (opt)
-
-Optionally, in order to use the policy rules validation feature, Mizu requires the following additional permissions:
-
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-  - create
-  - delete
-```
-
-- - -
-
-#### Namespace-Restricted mode
-
-Alternatively, in order to restrict Mizu to one namespace only (by setting `agent.namespace` in the config file), Mizu needs the following permissions in that namespace:
-
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - get
-  - create
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services/proxy
-  verbs:
-  - get
-```
-
-##### Name resolving in Namespace-Restricted mode (opt)
-
-To restrict Mizu to one namespace while also resolving IPs, Mizu needs the following permissions in that namespace:
-
-```yaml
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - apps
-  resources:
-  - daemonsets
-  verbs:
-  - get
-  - create
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services/proxy
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - serviceaccounts
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - roles
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - rolebindings
-  verbs:
-  - get
-  - create
-  - delete
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - apps
-  - extensions
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  - apps
-  - extensions
-  resources:
-  - endpoints
-  verbs:
-  - get
-  - list
-  - watch
-```
+The different requirements are listed in [the permission templates dir](../cli/cmd/permissionFiles)

examples/roles/permissions-all-namespaces-daemon.yaml

@@ -1,67 +0,0 @@
-# This example shows the roles required for a user to be able to use Mizu in all namespaces.
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: mizu-runner-clusterrole
-rules:
-  - apiGroups: [""]
-    resources: ["pods"]
-    verbs: ["get", "list", "watch", "delete"]
-  - apiGroups: [ "apps" ]
-    resources: [ "deployments" ]
-    verbs: [ "get", "create", "delete" ]
-  - apiGroups: [""]
-    resources: ["services"]
-    verbs: ["get", "list", "watch", "create", "delete"]
-  - apiGroups: ["apps"]
-    resources: ["daemonsets"]
-    verbs: ["get", "create", "patch", "delete", "list"]
-  - apiGroups: [""]
-    resources: ["namespaces"]
-    verbs: ["get", "list", "watch", "create", "delete"]
-  - apiGroups: [""]
-    resources: ["services/proxy"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["get", "create", "delete"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts"]
-    verbs: ["get", "create", "delete"]
-  - apiGroups: ["rbac.authorization.k8s.io"]
-    resources: ["clusterroles"]
-    verbs: ["get", "create", "delete"]
-  - apiGroups: ["rbac.authorization.k8s.io"]
-    resources: ["clusterrolebindings"]
-    verbs: ["get", "create", "delete"]
-  - apiGroups: ["rbac.authorization.k8s.io"]
-    resources: ["roles"]
-    verbs: ["get", "create", "delete"]
-  - apiGroups: ["rbac.authorization.k8s.io"]
-    resources: ["rolebindings"]
-    verbs: ["get", "create", "delete"]
-  - apiGroups: ["apps", "extensions"]
-    resources: ["pods"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["apps", "extensions"]
-    resources: ["services"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["", "apps", "extensions"]
-    resources: ["endpoints"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["events.k8s.io"]
-    resources: ["events"]
-    verbs: ["list", "watch"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: mizu-runner-clusterrolebindings
-subjects:
-  - kind: User
-    name: user1
-    apiGroup: rbac.authorization.k8s.io
-roleRef:
-  kind: ClusterRole
-  name: mizu-runner-clusterrole
-  apiGroup: rbac.authorization.k8s.io

examples/roles/permissions-all-namespaces.yaml

@@ -1,64 +0,0 @@
-# This example shows the roles required for a user to be able to use Mizu in all namespaces.
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: mizu-runner-clusterrole
-rules:
-- apiGroups: [""]
-  resources: ["pods"]
-  verbs: ["get", "list", "watch", "create", "delete"]
-- apiGroups: [""]
-  resources: ["services"]
-  verbs: ["get", "list", "watch", "create", "delete"]
-- apiGroups: ["apps"]
-  resources: ["daemonsets"]
-  verbs: ["create", "patch", "delete"]
-- apiGroups: [""]
-  resources: ["namespaces"]
-  verbs: ["get", "list", "watch", "create", "delete"]
-- apiGroups: [""]
-  resources: ["services/proxy"]
-  verbs: ["get"]
-- apiGroups: [""]
-  resources: ["configmaps"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: [""]
-  resources: ["serviceaccounts"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["rbac.authorization.k8s.io"]
-  resources: ["clusterroles"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["rbac.authorization.k8s.io"]
-  resources: ["clusterrolebindings"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["rbac.authorization.k8s.io"]
-  resources: ["roles"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["rbac.authorization.k8s.io"]
-  resources: ["rolebindings"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["apps", "extensions"]
-  resources: ["pods"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["apps", "extensions"]
-  resources: ["services"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["", "apps", "extensions"]
-  resources: ["endpoints"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["events.k8s.io"]
-  resources: ["events"]
-  verbs: ["list", "watch"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: mizu-runner-clusterrolebindings
-subjects:
-- kind: User
-  name: user1
-  apiGroup: rbac.authorization.k8s.io
-roleRef:
-  kind: ClusterRole
-  name: mizu-runner-clusterrole
-  apiGroup: rbac.authorization.k8s.io

examples/roles/permissions-ns-daemon.yaml

@@ -1,60 +0,0 @@
-# This example shows the roles required for a user to be able to use Mizu in a single namespace.
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: mizu-runner-role
-  namespace: user1
-rules:
-- apiGroups: [""]
-  resources: ["pods"]
-  verbs: ["get", "list", "watch", "delete"]
-- apiGroups: [ "apps" ]
-  resources: [ "deployments" ]
-  verbs: [ "get", "create", "delete" ]
-- apiGroups: [""]
-  resources: ["services"]
-  verbs: ["get", "list", "watch", "create", "delete"]
-- apiGroups: ["apps"]
-  resources: ["daemonsets"]
-  verbs: ["get", "create", "patch", "delete", "list"]
-- apiGroups: [""]
-  resources: ["services/proxy"]
-  verbs: ["get"]
-- apiGroups: [""]
-  resources: ["configmaps"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: [""]
-  resources: ["serviceaccounts"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["rbac.authorization.k8s.io"]
-  resources: ["roles"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["rbac.authorization.k8s.io"]
-  resources: ["rolebindings"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["apps", "extensions", ""]
-  resources: ["pods"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["apps", "extensions"]
-  resources: ["services"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["", "apps", "extensions"]
-  resources: ["endpoints"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["events.k8s.io"]
-  resources: ["events"]
-  verbs: ["list", "watch"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: mizu-runner-rolebindings
-  namespace: user1
-subjects:
-- kind: User
-  name: user1
-  apiGroup: rbac.authorization.k8s.io
-roleRef:
-  kind: Role
-  name: mizu-runner-role
-  apiGroup: rbac.authorization.k8s.io

examples/roles/permissions-ns-with-validation.yaml

@@ -1,57 +0,0 @@
-# This example shows the roles required for a user to be able to use Mizu in a single namespace.
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: mizu-runner-role
-  namespace: user1
-rules:
-- apiGroups: [""]
-  resources: ["pods"]
-  verbs: ["get", "list", "watch", "create", "delete"]
-- apiGroups: [""]
-  resources: ["services"]
-  verbs: ["get", "list", "watch", "create", "delete"]
-- apiGroups: ["apps"]
-  resources: ["daemonsets"]
-  verbs: ["get", "create", "patch", "delete"]
-- apiGroups: [""]
-  resources: ["services/proxy"]
-  verbs: ["get"]
-- apiGroups: [""]
-  resources: ["configmaps"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: [""]
-  resources: ["serviceaccounts"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["rbac.authorization.k8s.io"]
-  resources: ["roles"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["rbac.authorization.k8s.io"]
-  resources: ["rolebindings"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["apps", "extensions"]
-  resources: ["pods"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["apps", "extensions"]
-  resources: ["services"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["", "apps", "extensions"]
-  resources: ["endpoints"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["events.k8s.io"]
-  resources: ["events"]
-  verbs: ["list", "watch"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: mizu-runner-rolebindings
-  namespace: user1
-subjects:
-- kind: User
-  name: user1
-  apiGroup: rbac.authorization.k8s.io
-roleRef:
-  kind: Role
-  name: mizu-runner-role
-  apiGroup: rbac.authorization.k8s.io

examples/roles/permissions-ns.yaml

@@ -1,57 +0,0 @@
-# This example shows the roles required for a user to be able to use Mizu in a single namespace.
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: mizu-runner-role
-  namespace: user1
-rules:
-- apiGroups: [""]
-  resources: ["pods"]
-  verbs: ["get", "list", "watch", "create", "delete"]
-- apiGroups: [""]
-  resources: ["services"]
-  verbs: ["get", "list", "watch", "create", "delete"]
-- apiGroups: ["apps"]
-  resources: ["daemonsets"]
-  verbs: ["get", "create", "patch", "delete"]
-- apiGroups: [""]
-  resources: ["services/proxy"]
-  verbs: ["get"]
-- apiGroups: [ "" ]
-  resources: [ "configmaps" ]
-  verbs: [ "get", "create", "delete" ]
-- apiGroups: [""]
-  resources: ["serviceaccounts"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["rbac.authorization.k8s.io"]
-  resources: ["roles"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["rbac.authorization.k8s.io"]
-  resources: ["rolebindings"]
-  verbs: ["get", "create", "delete"]
-- apiGroups: ["apps", "extensions"]
-  resources: ["pods"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["apps", "extensions"]
-  resources: ["services"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["", "apps", "extensions"]
-  resources: ["endpoints"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["events.k8s.io"]
-  resources: ["events"]
-  verbs: ["list", "watch"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: mizu-runner-rolebindings
-  namespace: user1
-subjects:
-- kind: User
-  name: user1
-  apiGroup: rbac.authorization.k8s.io
-roleRef:
-  kind: Role
-  name: mizu-runner-role
-  apiGroup: rbac.authorization.k8s.io

shared/kubernetes/mizuTapperSyncer.go

@@ -46,6 +46,7 @@ type TapperSyncerConfig struct {
 	MizuApiFilteringOptions  api.TrafficFilteringOptions
 	MizuServiceAccountExists bool
 	ServiceMesh              bool
+	Tls                      bool
 }
 
 func CreateAndStartMizuTapperSyncer(ctx context.Context, kubernetesProvider *Provider, config TapperSyncerConfig, startTime time.Time) (*MizuTapperSyncer, error) {
@@ -324,6 +325,7 @@ func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error {
 			tapperSyncer.config.MizuApiFilteringOptions,
 			tapperSyncer.config.LogLevel,
 			tapperSyncer.config.ServiceMesh,
+			tapperSyncer.config.Tls,
 		); err != nil {
 			return err
 		}

shared/kubernetes/provider.go

@@ -17,6 +17,7 @@ import (
 	"github.com/up9inc/mizu/shared/semver"
 	"github.com/up9inc/mizu/tap/api"
 	v1 "k8s.io/api/apps/v1"
+	auth "k8s.io/api/authorization/v1"
 	core "k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -51,6 +52,8 @@ const (
 	fieldManagerName = "mizu-manager"
 	procfsVolumeName = "proc"
 	procfsMountPath  = "/hostproc"
+	sysfsVolumeName  = "sys"
+	sysfsMountPath   = "/sys"
 )
 
 func NewProvider(kubeConfigPath string) (*Provider, error) {
@@ -76,6 +79,8 @@ func NewProvider(kubeConfigPath string) (*Provider, error) {
 			"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 	}
 
+	logger.Log.Debugf("K8s client config, host: %s, api path: %s, user agent: %s", restClientConfig.Host, restClientConfig.APIPath, restClientConfig.UserAgent)
+
 	return &Provider{
 		clientSet:        clientSet,
 		kubernetesConfig: kubernetesConfig,
@@ -439,6 +444,26 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s
 	return provider.clientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{})
 }
 
+func (provider *Provider) CanI(ctx context.Context, namespace string, resource string, verb string, group string) (bool, error) {
+	selfSubjectAccessReview := &auth.SelfSubjectAccessReview{
+		Spec: auth.SelfSubjectAccessReviewSpec{
+			ResourceAttributes: &auth.ResourceAttributes{
+				Namespace: namespace,
+				Resource: resource,
+				Verb: verb,
+				Group: group,
+			},
+		},
+	}
+
+	response, err := provider.clientSet.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, selfSubjectAccessReview, metav1.CreateOptions{})
+	if err != nil {
+		return false, err
+	}
+
+	return response.Status.Allowed, nil
+}
+
 func (provider *Provider) DoesNamespaceExist(ctx context.Context, name string) (bool, error) {
 	namespaceResource, err := provider.clientSet.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{})
 	return provider.doesResourceExist(namespaceResource, err)
@@ -464,11 +489,6 @@ func (provider *Provider) DoesDeploymentExist(ctx context.Context, namespace str
 	return provider.doesResourceExist(deploymentResource, err)
 }
 
-func (provider *Provider) DoesPodExist(ctx context.Context, namespace string, name string) (bool, error) {
-	podResource, err := provider.clientSet.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
-	return provider.doesResourceExist(podResource, err)
-}
-
 func (provider *Provider) DoesServiceExist(ctx context.Context, namespace string, name string) (bool, error) {
 	serviceResource, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{})
 	return provider.doesResourceExist(serviceResource, err)
@@ -793,7 +813,7 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
 	return nil
 }
 
-func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodMap map[string][]core.Pod, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, logLevel logging.Level, serviceMesh bool) error {
+func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodMap map[string][]core.Pod, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, logLevel logging.Level, serviceMesh bool, tls bool) error {
 	logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodMap), namespace, daemonSetName, podImage, tapperPodName)
 
 	if len(nodeToTappedPodMap) == 0 {
@@ -819,7 +839,15 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	}
 
 	if serviceMesh {
-		mizuCmd = append(mizuCmd, "--procfs", procfsMountPath, "--servicemesh")
+		mizuCmd = append(mizuCmd, "--servicemesh")
+	}
+
+	if tls {
+		mizuCmd = append(mizuCmd, "--tls")
+	}
+
+	if serviceMesh || tls {
+		mizuCmd = append(mizuCmd, "--procfs", procfsMountPath)
 	}
 
 	agentContainer := applyconfcore.Container()
@@ -827,12 +855,21 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	agentContainer.WithImage(podImage)
 	agentContainer.WithImagePullPolicy(imagePullPolicy)
 
-	caps := applyconfcore.Capabilities().WithDrop("ALL").WithAdd("NET_RAW").WithAdd("NET_ADMIN")
+	caps := applyconfcore.Capabilities().WithDrop("ALL")
 
-	if serviceMesh {
-		caps = caps.WithAdd("SYS_ADMIN")    // for reading /proc/PID/net/ns
-		caps = caps.WithAdd("SYS_PTRACE")   // for setting netns to other process
-		caps = caps.WithAdd("DAC_OVERRIDE") // for reading /proc/PID/environ
+	caps = caps.WithAdd("NET_RAW").WithAdd("NET_ADMIN") // to listen to traffic using libpcap
+
+	if serviceMesh || tls {
+		caps = caps.WithAdd("SYS_ADMIN")  // to read /proc/PID/net/ns + to install eBPF programs (kernel < 5.8)
+		caps = caps.WithAdd("SYS_PTRACE") // to set netns to other process + to open libssl.so of other process
+
+		if serviceMesh {
+			caps = caps.WithAdd("DAC_OVERRIDE") // to read /proc/PID/environ
+		}
+
+		if tls {
+			caps = caps.WithAdd("SYS_RESOURCE") // to change rlimits for eBPF
+		}
 	}
 
 	agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithCapabilities(caps))
@@ -908,26 +945,15 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	//
 	procfsVolume := applyconfcore.Volume()
 	procfsVolume.WithName(procfsVolumeName).WithHostPath(applyconfcore.HostPathVolumeSource().WithPath("/proc"))
-	volumeMount := applyconfcore.VolumeMount().WithName(procfsVolumeName).WithMountPath(procfsMountPath).WithReadOnly(true)
-	agentContainer.WithVolumeMounts(volumeMount)
+	procfsVolumeMount := applyconfcore.VolumeMount().WithName(procfsVolumeName).WithMountPath(procfsMountPath).WithReadOnly(true)
+	agentContainer.WithVolumeMounts(procfsVolumeMount)
 
-	volumeName := ConfigMapName
-	configMapVolume := applyconfcore.VolumeApplyConfiguration{
-		Name: &volumeName,
-		VolumeSourceApplyConfiguration: applyconfcore.VolumeSourceApplyConfiguration{
-			ConfigMap: &applyconfcore.ConfigMapVolumeSourceApplyConfiguration{
-				LocalObjectReferenceApplyConfiguration: applyconfcore.LocalObjectReferenceApplyConfiguration{
-					Name: &volumeName,
-				},
-			},
-		},
-	}
-	mountPath := shared.ConfigDirPath
-	configMapVolumeMount := applyconfcore.VolumeMountApplyConfiguration{
-		Name:      &volumeName,
-		MountPath: &mountPath,
-	}
-	agentContainer.WithVolumeMounts(&configMapVolumeMount)
+	// We need access to /sys in order to install certain eBPF tracepoints
+	//
+	sysfsVolume := applyconfcore.Volume()
+	sysfsVolume.WithName(sysfsVolumeName).WithHostPath(applyconfcore.HostPathVolumeSource().WithPath("/sys"))
+	sysfsVolumeMount := applyconfcore.VolumeMount().WithName(sysfsVolumeName).WithMountPath(sysfsMountPath).WithReadOnly(true)
+	agentContainer.WithVolumeMounts(sysfsVolumeMount)
 
 	podSpec := applyconfcore.PodSpec()
 	podSpec.WithHostNetwork(true)
@@ -939,7 +965,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	podSpec.WithContainers(agentContainer)
 	podSpec.WithAffinity(affinity)
 	podSpec.WithTolerations(noExecuteToleration, noScheduleToleration)
-	podSpec.WithVolumes(&configMapVolume, procfsVolume)
+	podSpec.WithVolumes(procfsVolume, sysfsVolume)
 
 	podTemplate := applyconfcore.PodTemplateSpec()
 	podTemplate.WithLabels(map[string]string{
@@ -952,6 +978,11 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	labelSelector := applyconfmeta.LabelSelector()
 	labelSelector.WithMatchLabels(map[string]string{"app": tapperPodName})
 
+	applyOptions := metav1.ApplyOptions{
+		Force:        true,
+		FieldManager: fieldManagerName,
+	}
+
 	daemonSet := applyconfapp.DaemonSet(daemonSetName, namespace)
 	daemonSet.
 		WithLabels(map[string]string{
@@ -960,7 +991,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 		}).
 		WithSpec(applyconfapp.DaemonSetSpec().WithSelector(labelSelector).WithTemplate(podTemplate))
 
-	_, err = provider.clientSet.AppsV1().DaemonSets(namespace).Apply(ctx, daemonSet, metav1.ApplyOptions{FieldManager: fieldManagerName})
+	_, err = provider.clientSet.AppsV1().DaemonSets(namespace).Apply(ctx, daemonSet, applyOptions)
 	return err
 }
 
@@ -1007,6 +1038,15 @@ func (provider *Provider) ListAllRunningPodsMatchingRegex(ctx context.Context, r
 	return matchingPods, nil
 }
 
+func(provider *Provider) ListPodsByAppLabel(ctx context.Context, namespaces string, labelName string) ([]core.Pod, error) {
+	pods, err := provider.clientSet.CoreV1().Pods(namespaces).List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", labelName)})
+	if err != nil {
+		return nil, err
+	}
+
+	return pods.Items, err
+}
+
 func (provider *Provider) ListAllNamespaces(ctx context.Context) ([]core.Namespace, error) {
 	namespaces, err := provider.clientSet.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
 	if err != nil {

shared/kubernetes/proxy.go

@@ -128,9 +128,14 @@ func getHttpDialer(kubernetesProvider *Provider, namespace string, podName strin
 		return nil, err
 	}
 
-	path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward", namespace, podName)
-	hostIP := strings.TrimLeft(kubernetesProvider.clientConfig.Host, "htps:/") // no need specify "t" twice
-	serverURL := url.URL{Scheme: "https", Path: path, Host: hostIP}
+	clientConfigHostUrl, err := url.Parse(kubernetesProvider.clientConfig.Host)
+	if err != nil {
+		return nil, fmt.Errorf("Failed parsing client config host URL %s, error %w", kubernetesProvider.clientConfig.Host, err)
+	}
+	path := fmt.Sprintf("%s/api/v1/namespaces/%s/pods/%s/portforward", clientConfigHostUrl.Path, namespace, podName)
+
+	serverURL := url.URL{Scheme: "https", Path: path, Host: clientConfigHostUrl.Host}
+	logger.Log.Debugf("Http dialer url %v", serverURL)
 
 	return spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, &serverURL), nil
 }

shared/kubernetes/utils.go

@@ -30,11 +30,24 @@ func getMinimizedPod(fullPod core.Pod) core.Pod {
 			Name: fullPod.Name,
 		},
 		Status: v1.PodStatus{
-			PodIP: fullPod.Status.PodIP,
+			PodIP:             fullPod.Status.PodIP,
+			ContainerStatuses: getMinimizedContainerStatuses(fullPod),
 		},
 	}
 }
 
+func getMinimizedContainerStatuses(fullPod core.Pod) []v1.ContainerStatus {
+	result := make([]v1.ContainerStatus, len(fullPod.Status.ContainerStatuses))
+
+	for i, container := range fullPod.Status.ContainerStatuses {
+		result[i] = v1.ContainerStatus{
+			ContainerID: container.ContainerID,
+		}
+	}
+
+	return result
+}
+
 func excludeMizuPods(pods []core.Pod) []core.Pod {
 	mizuPrefixRegex := regexp.MustCompile("^" + MizuResourcesPrefix)
 

shared/logger/logger.go

@@ -29,10 +29,21 @@ func InitLogger(logPath string) {
 	logging.SetBackend(backend1Leveled, backend2Formatter)
 }
 
-func InitLoggerStderrOnly(level logging.Level) {
-	backend := logging.NewLogBackend(os.Stderr, "", 0)
-	backendFormatter := logging.NewBackendFormatter(backend, format)
+func InitLoggerStd(level logging.Level) {
+	var backends []logging.Backend
 
-	logging.SetBackend(backendFormatter)
-	logging.SetLevel(level, "")
+	stderrBackend := logging.NewLogBackend(os.Stderr, "", 0)
+	stderrFormater := logging.NewBackendFormatter(stderrBackend, format)
+	stderrLeveled := logging.AddModuleLevel(stderrFormater)
+	stderrLeveled.SetLevel(logging.ERROR, "")
+	backends = append(backends, stderrLeveled)
+
+	if level >= logging.WARNING {
+		stdoutBackend := logging.NewLogBackend(os.Stdout, "", 0)
+		stdoutFormater := logging.NewBackendFormatter(stdoutBackend, format)
+		stdoutLeveled := logging.AddModuleLevel(stdoutFormater)
+		stdoutLeveled.SetLevel(level, "")
+		backends = append(backends, stdoutLeveled)
+	}
+	logging.SetBackend(backends...)
 }

shared/models.go

@@ -15,6 +15,7 @@ type WebSocketMessageType string
 
 const (
 	WebSocketMessageTypeEntry         WebSocketMessageType = "entry"
+	WebSocketMessageTypeFullEntry     WebSocketMessageType = "fullEntry"
 	WebSocketMessageTypeTappedEntry   WebSocketMessageType = "tappedEntry"
 	WebSocketMessageTypeUpdateStatus  WebSocketMessageType = "status"
 	WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus"
@@ -43,6 +44,7 @@ type MizuAgentConfig struct {
 	StandaloneMode         bool          `json:"standaloneMode"`
 	ServiceMap             bool          `json:"serviceMap"`
 	OAS                    bool          `json:"oas"`
+	Telemetry              bool          `json:"telemetry"`
 	Elastic                ElasticConfig `json:"elastic"`
 }
 

tap/api/api.go

@@ -39,10 +39,9 @@ type TCP struct {
 }
 
 type Extension struct {
-	Protocol   *Protocol
-	Path       string
-	Dissector  Dissector
-	MatcherMap *sync.Map
+	Protocol  *Protocol
+	Path      string
+	Dissector Dissector
 }
 
 type ConnectionInfo struct {
@@ -62,7 +61,6 @@ type TcpID struct {
 }
 
 type CounterPair struct {
-	StreamId int64
 	Request  uint
 	Response uint
 	sync.Mutex
@@ -100,10 +98,16 @@ type SuperIdentifier struct {
 type Dissector interface {
 	Register(*Extension)
 	Ping()
-	Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID, counterPair *CounterPair, superTimer *SuperTimer, superIdentifier *SuperIdentifier, emitter Emitter, options *TrafficFilteringOptions) error
-	Analyze(item *OutputChannelItem, resolvedSource string, resolvedDestination string) *Entry
+	Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID, counterPair *CounterPair, superTimer *SuperTimer, superIdentifier *SuperIdentifier, emitter Emitter, options *TrafficFilteringOptions, reqResMatcher RequestResponseMatcher) error
+	Analyze(item *OutputChannelItem, resolvedSource string, resolvedDestination string, namespace string) *Entry
 	Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error)
 	Macros() map[string]string
+	NewResponseRequestMatcher() RequestResponseMatcher
+}
+
+type RequestResponseMatcher interface {
+	GetMap() *sync.Map
+	SetMaxTry(value int)
 }
 
 type Emitting struct {
@@ -125,6 +129,7 @@ type Entry struct {
 	Protocol               Protocol               `json:"proto"`
 	Source                 *TCP                   `json:"src"`
 	Destination            *TCP                   `json:"dst"`
+	Namespace              string                 `json:"namespace,omitempty"`
 	Outgoing               bool                   `json:"outgoing"`
 	Timestamp              int64                  `json:"timestamp"`
 	StartTime              time.Time              `json:"startTime"`

tap/cleaner.go

@@ -22,6 +22,7 @@ type Cleaner struct {
 	connectionTimeout time.Duration
 	stats             CleanerStats
 	statsMutex        sync.Mutex
+	streamsMap        *tcpStreamMap
 }
 
 func (cl *Cleaner) clean() {
@@ -32,10 +33,15 @@ func (cl *Cleaner) clean() {
 	flushed, closed := cl.assembler.FlushCloseOlderThan(startCleanTime.Add(-cl.connectionTimeout))
 	cl.assemblerMutex.Unlock()
 
-	for _, extension := range extensions {
-		deleted := deleteOlderThan(extension.MatcherMap, startCleanTime.Add(-cl.connectionTimeout))
+	cl.streamsMap.streams.Range(func(k, v interface{}) bool {
+		reqResMatcher := v.(*tcpStreamWrapper).reqResMatcher
+		if reqResMatcher == nil {
+			return true
+		}
+		deleted := deleteOlderThan(reqResMatcher.GetMap(), startCleanTime.Add(-cl.connectionTimeout))
 		cl.stats.deleted += deleted
-	}
+		return true
+	})
 
 	cl.statsMutex.Lock()
 	logger.Log.Debugf("Assembler Stats after cleaning %s", cl.assembler.Dump())

tap/extensions/amqp/Makefile

@@ -0,0 +1,16 @@
+skipbin := $$(find bin -mindepth 1 -maxdepth 1)
+skipexpect := $$(find expect -mindepth 1 -maxdepth 1)
+
+test: test-pull-bin test-pull-expect
+	@MIZU_TEST=1 go test -v ./... -coverpkg=./... -race -coverprofile=coverage.out -covermode=atomic
+
+test-update: test-pull-bin
+	@MIZU_TEST=1 TEST_UPDATE=1 go test -v ./... -coverpkg=./... -coverprofile=coverage.out -covermode=atomic
+
+test-pull-bin:
+	@mkdir -p bin
+	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp gs://static.up9.io/mizu/test-pcap/bin/amqp/\*.bin bin
+
+test-pull-expect:
+	@mkdir -p expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect/amqp/\* expect

tap/extensions/amqp/go.mod

@@ -2,8 +2,16 @@ module github.com/up9inc/mizu/tap/extensions/amqp
 
 go 1.17
 
-require github.com/up9inc/mizu/tap/api v0.0.0
+require (
+	github.com/stretchr/testify v1.7.0
+	github.com/up9inc/mizu/tap/api v0.0.0
+)
 
-require github.com/google/martian v2.1.0+incompatible // indirect
+require (
+	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/google/martian v2.1.0+incompatible // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+)
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api

tap/extensions/amqp/go.sum

@@ -1,2 +1,13 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tap/extensions/amqp/helpers.go

@@ -3,6 +3,7 @@ package amqp
 import (
 	"encoding/json"
 	"fmt"
+	"sort"
 	"strconv"
 	"time"
 
@@ -282,6 +283,9 @@ func representBasicPublish(event map[string]interface{}) []interface{} {
 				Selector: fmt.Sprintf(`request.properties.headers["%s"]`, name),
 			})
 		}
+		sort.Slice(headers, func(i, j int) bool {
+			return headers[i].Name < headers[j].Name
+		})
 		headersMarshaled, _ := json.Marshal(headers)
 		rep = append(rep, api.SectionData{
 			Type:  api.TABLE,
@@ -366,6 +370,9 @@ func representBasicDeliver(event map[string]interface{}) []interface{} {
 				Selector: fmt.Sprintf(`request.properties.headers["%s"]`, name),
 			})
 		}
+		sort.Slice(headers, func(i, j int) bool {
+			return headers[i].Name < headers[j].Name
+		})
 		headersMarshaled, _ := json.Marshal(headers)
 		rep = append(rep, api.SectionData{
 			Type:  api.TABLE,
@@ -438,6 +445,9 @@ func representQueueDeclare(event map[string]interface{}) []interface{} {
 				Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
 			})
 		}
+		sort.Slice(headers, func(i, j int) bool {
+			return headers[i].Name < headers[j].Name
+		})
 		headersMarshaled, _ := json.Marshal(headers)
 		rep = append(rep, api.SectionData{
 			Type:  api.TABLE,
@@ -504,6 +514,9 @@ func representExchangeDeclare(event map[string]interface{}) []interface{} {
 				Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
 			})
 		}
+		sort.Slice(headers, func(i, j int) bool {
+			return headers[i].Name < headers[j].Name
+		})
 		headersMarshaled, _ := json.Marshal(headers)
 		rep = append(rep, api.SectionData{
 			Type:  api.TABLE,
@@ -565,6 +578,9 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
 				Selector: fmt.Sprintf(`request.serverProperties["%s"]`, name),
 			})
 		}
+		sort.Slice(headers, func(i, j int) bool {
+			return headers[i].Name < headers[j].Name
+		})
 		headersMarshaled, _ := json.Marshal(headers)
 		rep = append(rep, api.SectionData{
 			Type:  api.TABLE,
@@ -656,6 +672,9 @@ func representQueueBind(event map[string]interface{}) []interface{} {
 				Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
 			})
 		}
+		sort.Slice(headers, func(i, j int) bool {
+			return headers[i].Name < headers[j].Name
+		})
 		headersMarshaled, _ := json.Marshal(headers)
 		rep = append(rep, api.SectionData{
 			Type:  api.TABLE,
@@ -717,6 +736,9 @@ func representBasicConsume(event map[string]interface{}) []interface{} {
 				Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
 			})
 		}
+		sort.Slice(headers, func(i, j int) bool {
+			return headers[i].Name < headers[j].Name
+		})
 		headersMarshaled, _ := json.Marshal(headers)
 		rep = append(rep, api.SectionData{
 			Type:  api.TABLE,

tap/extensions/amqp/main.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"log"
 	"strconv"
+	"time"
 
 	"github.com/up9inc/mizu/tap/api"
 )
@@ -26,10 +27,6 @@ var protocol api.Protocol = api.Protocol{
 	Priority:        1,
 }
 
-func init() {
-	log.Println("Initializing AMQP extension...")
-}
-
 type dissecting string
 
 func (d dissecting) Register(extension *api.Extension) {
@@ -42,7 +39,7 @@ func (d dissecting) Ping() {
 
 const amqpRequest string = "amqp_request"
 
-func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
+func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions, _reqResMatcher api.RequestResponseMatcher) error {
 	r := AmqpReader{b}
 
 	var remaining int
@@ -85,7 +82,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 		frame, err := r.ReadFrame()
 		if err == io.EOF {
 			// We must read until we see an EOF... very important!
-			return errors.New("AMQP EOF")
+			return nil
 		}
 
 		switch f := frame.(type) {
@@ -96,6 +93,12 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			// start content state
 			header = f
 			remaining = int(header.Size)
+
+			// Workaround for `Time.MarshalJSON: year outside of range [0,9999]` error
+			if header.Properties.Timestamp.Year() > 9999 {
+				header.Properties.Timestamp = time.Time{}.UTC()
+			}
+
 			switch lastMethodFrameMessage.(type) {
 			case *BasicPublish:
 				eventBasicPublish.Properties = header.Properties
@@ -212,7 +215,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.Entry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string, namespace string) *api.Entry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
 
@@ -254,6 +257,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 			IP:   item.ConnectionInfo.ServerIP,
 			Port: item.ConnectionInfo.ServerPort,
 		},
+		Namespace:   namespace,
 		Outgoing:    item.ConnectionInfo.IsOutgoing,
 		Request:     reqDetails,
 		Method:      request["method"].(string),
@@ -300,6 +304,10 @@ func (d dissecting) Macros() map[string]string {
 	}
 }
 
+func (d dissecting) NewResponseRequestMatcher() api.RequestResponseMatcher {
+	return nil
+}
+
 var Dissector dissecting
 
 func NewDissector() api.Dissector {

tap/extensions/amqp/main_test.go

@@ -0,0 +1,289 @@
+package amqp
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/up9inc/mizu/tap/api"
+)
+
+const (
+	binDir          = "bin"
+	patternBin      = "*_req.bin"
+	patternDissect  = "*.json"
+	msgDissecting   = "Dissecting:"
+	msgAnalyzing    = "Analyzing:"
+	msgRepresenting = "Representing:"
+	respSuffix      = "_res.bin"
+	expectDir       = "expect"
+	dissectDir      = "dissect"
+	analyzeDir      = "analyze"
+	representDir    = "represent"
+	testUpdate      = "TEST_UPDATE"
+)
+
+func TestRegister(t *testing.T) {
+	dissector := NewDissector()
+	extension := &api.Extension{}
+	dissector.Register(extension)
+	assert.Equal(t, "amqp", extension.Protocol.Name)
+}
+
+func TestMacros(t *testing.T) {
+	expectedMacros := map[string]string{
+		"amqp": `proto.name == "amqp"`,
+	}
+	dissector := NewDissector()
+	macros := dissector.Macros()
+	assert.Equal(t, expectedMacros, macros)
+}
+
+func TestPing(t *testing.T) {
+	dissector := NewDissector()
+	dissector.Ping()
+}
+
+func TestDissect(t *testing.T) {
+	_, testUpdateEnabled := os.LookupEnv(testUpdate)
+
+	expectDirDissect := path.Join(expectDir, dissectDir)
+
+	if testUpdateEnabled {
+		os.RemoveAll(expectDirDissect)
+		err := os.MkdirAll(expectDirDissect, 0775)
+		assert.Nil(t, err)
+	}
+
+	dissector := NewDissector()
+	paths, err := filepath.Glob(path.Join(binDir, patternBin))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	options := &api.TrafficFilteringOptions{
+		IgnoredUserAgents: []string{},
+	}
+
+	for _, _path := range paths {
+		basePath := _path[:len(_path)-8]
+
+		// Channel to verify the output
+		itemChannel := make(chan *api.OutputChannelItem)
+		var emitter api.Emitter = &api.Emitting{
+			AppStats:      &api.AppStats{},
+			OutputChannel: itemChannel,
+		}
+
+		var items []*api.OutputChannelItem
+		stop := make(chan bool)
+
+		go func() {
+			for {
+				select {
+				case <-stop:
+					return
+				case item := <-itemChannel:
+					items = append(items, item)
+				}
+			}
+		}()
+
+		// Stream level
+		counterPair := &api.CounterPair{
+			Request:  0,
+			Response: 0,
+		}
+		superIdentifier := &api.SuperIdentifier{}
+
+		// Request
+		pathClient := _path
+		fmt.Printf("%s %s\n", msgDissecting, pathClient)
+		fileClient, err := os.Open(pathClient)
+		assert.Nil(t, err)
+
+		bufferClient := bufio.NewReader(fileClient)
+		tcpIDClient := &api.TcpID{
+			SrcIP:   "1",
+			DstIP:   "2",
+			SrcPort: "1",
+			DstPort: "2",
+		}
+		reqResMatcher := dissector.NewResponseRequestMatcher()
+		err = dissector.Dissect(bufferClient, true, tcpIDClient, counterPair, &api.SuperTimer{}, superIdentifier, emitter, options, reqResMatcher)
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+			panic(err)
+		}
+
+		// Response
+		pathServer := basePath + respSuffix
+		fmt.Printf("%s %s\n", msgDissecting, pathServer)
+		fileServer, err := os.Open(pathServer)
+		assert.Nil(t, err)
+
+		bufferServer := bufio.NewReader(fileServer)
+		tcpIDServer := &api.TcpID{
+			SrcIP:   "2",
+			DstIP:   "1",
+			SrcPort: "2",
+			DstPort: "1",
+		}
+		err = dissector.Dissect(bufferServer, false, tcpIDServer, counterPair, &api.SuperTimer{}, superIdentifier, emitter, options, reqResMatcher)
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+			panic(err)
+		}
+
+		fileClient.Close()
+		fileServer.Close()
+
+		pathExpect := path.Join(expectDirDissect, fmt.Sprintf("%s.json", basePath[4:]))
+
+		time.Sleep(10 * time.Millisecond)
+
+		stop <- true
+
+		marshaled, err := json.Marshal(items)
+		assert.Nil(t, err)
+
+		if testUpdateEnabled {
+			if len(items) > 0 {
+				err = os.WriteFile(pathExpect, marshaled, 0644)
+				assert.Nil(t, err)
+			}
+		} else {
+			if _, err := os.Stat(pathExpect); errors.Is(err, os.ErrNotExist) {
+				assert.Len(t, items, 0)
+			} else {
+				expectedBytes, err := ioutil.ReadFile(pathExpect)
+				assert.Nil(t, err)
+
+				assert.JSONEq(t, string(expectedBytes), string(marshaled))
+			}
+		}
+	}
+}
+
+func TestAnalyze(t *testing.T) {
+	_, testUpdateEnabled := os.LookupEnv(testUpdate)
+
+	expectDirDissect := path.Join(expectDir, dissectDir)
+	expectDirAnalyze := path.Join(expectDir, analyzeDir)
+
+	if testUpdateEnabled {
+		os.RemoveAll(expectDirAnalyze)
+		err := os.MkdirAll(expectDirAnalyze, 0775)
+		assert.Nil(t, err)
+	}
+
+	dissector := NewDissector()
+	paths, err := filepath.Glob(path.Join(expectDirDissect, patternDissect))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, _path := range paths {
+		fmt.Printf("%s %s\n", msgAnalyzing, _path)
+
+		bytes, err := ioutil.ReadFile(_path)
+		assert.Nil(t, err)
+
+		var items []*api.OutputChannelItem
+		err = json.Unmarshal(bytes, &items)
+		assert.Nil(t, err)
+
+		var entries []*api.Entry
+		for _, item := range items {
+			entry := dissector.Analyze(item, "", "", "")
+			entries = append(entries, entry)
+		}
+
+		pathExpect := path.Join(expectDirAnalyze, filepath.Base(_path))
+
+		marshaled, err := json.Marshal(entries)
+		assert.Nil(t, err)
+
+		if testUpdateEnabled {
+			if len(entries) > 0 {
+				err = os.WriteFile(pathExpect, marshaled, 0644)
+				assert.Nil(t, err)
+			}
+		} else {
+			if _, err := os.Stat(pathExpect); errors.Is(err, os.ErrNotExist) {
+				assert.Len(t, items, 0)
+			} else {
+				expectedBytes, err := ioutil.ReadFile(pathExpect)
+				assert.Nil(t, err)
+
+				assert.JSONEq(t, string(expectedBytes), string(marshaled))
+			}
+		}
+	}
+}
+
+func TestRepresent(t *testing.T) {
+	_, testUpdateEnabled := os.LookupEnv(testUpdate)
+
+	expectDirAnalyze := path.Join(expectDir, analyzeDir)
+	expectDirRepresent := path.Join(expectDir, representDir)
+
+	if testUpdateEnabled {
+		os.RemoveAll(expectDirRepresent)
+		err := os.MkdirAll(expectDirRepresent, 0775)
+		assert.Nil(t, err)
+	}
+
+	dissector := NewDissector()
+	paths, err := filepath.Glob(path.Join(expectDirAnalyze, patternDissect))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, _path := range paths {
+		fmt.Printf("%s %s\n", msgRepresenting, _path)
+
+		bytes, err := ioutil.ReadFile(_path)
+		assert.Nil(t, err)
+
+		var entries []*api.Entry
+		err = json.Unmarshal(bytes, &entries)
+		assert.Nil(t, err)
+
+		var objects []string
+		for _, entry := range entries {
+			object, _, err := dissector.Represent(entry.Request, entry.Response)
+			assert.Nil(t, err)
+			objects = append(objects, string(object))
+		}
+
+		pathExpect := path.Join(expectDirRepresent, filepath.Base(_path))
+
+		marshaled, err := json.Marshal(objects)
+		assert.Nil(t, err)
+
+		if testUpdateEnabled {
+			if len(objects) > 0 {
+				err = os.WriteFile(pathExpect, marshaled, 0644)
+				assert.Nil(t, err)
+			}
+		} else {
+			if _, err := os.Stat(pathExpect); errors.Is(err, os.ErrNotExist) {
+				assert.Len(t, objects, 0)
+			} else {
+				expectedBytes, err := ioutil.ReadFile(pathExpect)
+				assert.Nil(t, err)
+
+				assert.JSONEq(t, string(expectedBytes), string(marshaled))
+			}
+		}
+	}
+}

tap/extensions/amqp/read.go

@@ -140,7 +140,7 @@ func readTimestamp(r io.Reader) (v time.Time, err error) {
 	if err = binary.Read(r, binary.BigEndian, &sec); err != nil {
 		return
 	}
-	return time.Unix(sec, 0), nil
+	return time.Unix(sec, 0).UTC(), nil
 }
 
 /*

tap/extensions/amqp/types.go

@@ -223,41 +223,8 @@ type Decimal struct {
 //
 type Table map[string]interface{}
 
-func validateField(f interface{}) error {
-	switch fv := f.(type) {
-	case nil, bool, byte, int, int16, int32, int64, float32, float64, string, []byte, Decimal, time.Time:
-		return nil
-
-	case []interface{}:
-		for _, v := range fv {
-			if err := validateField(v); err != nil {
-				return fmt.Errorf("in array %s", err)
-			}
-		}
-		return nil
-
-	case Table:
-		for k, v := range fv {
-			if err := validateField(v); err != nil {
-				return fmt.Errorf("table field %q %s", k, err)
-			}
-		}
-		return nil
-	}
-
-	return fmt.Errorf("value %T not supported", f)
-}
-
-// Validate returns and error if any Go types in the table are incompatible with AMQP types.
-func (t Table) Validate() error {
-	return validateField(t)
-}
-
 type Message interface {
-	id() (uint16, uint16)
-	wait() bool
 	read(io.Reader) error
-	write(io.Writer) error
 }
 
 /*
@@ -286,8 +253,6 @@ system calls to read a frame.
 
 */
 type frame interface {
-	write(io.Writer) error
-	channel() uint16
 }
 
 type AmqpReader struct {
@@ -323,8 +288,6 @@ type MethodFrame struct {
 	Method    Message
 }
 
-func (f *MethodFrame) channel() uint16 { return f.ChannelId }
-
 /*
 Heartbeating is a technique designed to undo one of TCP/IP's features, namely
 its ability to recover from a broken physical connection by closing only after
@@ -338,8 +301,6 @@ type HeartbeatFrame struct {
 	ChannelId uint16
 }
 
-func (f *HeartbeatFrame) channel() uint16 { return f.ChannelId }
-
 /*
 Certain methods (such as Basic.Publish, Basic.Deliver, etc.) are formally
 defined as carrying content.  When a peer sends such a method frame, it always
@@ -367,8 +328,6 @@ type HeaderFrame struct {
 	Properties Properties
 }
 
-func (f *HeaderFrame) channel() uint16 { return f.ChannelId }
-
 /*
 Content is the application data we carry from client-to-client via the AMQP
 server.  Content is, roughly speaking, a set of properties plus a binary data
@@ -388,5 +347,3 @@ type BodyFrame struct {
 	ChannelId uint16
 	Body      []byte
 }
-
-func (f *BodyFrame) channel() uint16 { return f.ChannelId }

tap/extensions/amqp/write.go

@@ -1,403 +0,0 @@
-// Copyright (c) 2012, Sean Treadway, SoundCloud Ltd.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-// Source code and contact info at http://github.com/streadway/amqp
-
-package amqp
-
-import (
-	"bytes"
-	"encoding/binary"
-	"errors"
-	"io"
-	"math"
-	"time"
-)
-
-func (f *MethodFrame) write(w io.Writer) (err error) {
-	var payload bytes.Buffer
-
-	if f.Method == nil {
-		return errors.New("malformed frame: missing method")
-	}
-
-	class, method := f.Method.id()
-
-	if err = binary.Write(&payload, binary.BigEndian, class); err != nil {
-		return
-	}
-
-	if err = binary.Write(&payload, binary.BigEndian, method); err != nil {
-		return
-	}
-
-	if err = f.Method.write(&payload); err != nil {
-		return
-	}
-
-	return writeFrame(w, frameMethod, f.ChannelId, payload.Bytes())
-}
-
-// Heartbeat
-//
-// Payload is empty
-func (f *HeartbeatFrame) write(w io.Writer) (err error) {
-	return writeFrame(w, frameHeartbeat, f.ChannelId, []byte{})
-}
-
-// CONTENT HEADER
-// 0          2        4           12               14
-// +----------+--------+-----------+----------------+------------- - -
-// | class-id | weight | body size | property flags | property list...
-// +----------+--------+-----------+----------------+------------- - -
-//    short     short    long long       short        remainder...
-//
-func (f *HeaderFrame) write(w io.Writer) (err error) {
-	var payload bytes.Buffer
-	var zeroTime time.Time
-
-	if err = binary.Write(&payload, binary.BigEndian, f.ClassId); err != nil {
-		return
-	}
-
-	if err = binary.Write(&payload, binary.BigEndian, f.weight); err != nil {
-		return
-	}
-
-	if err = binary.Write(&payload, binary.BigEndian, f.Size); err != nil {
-		return
-	}
-
-	// First pass will build the mask to be serialized, second pass will serialize
-	// each of the fields that appear in the mask.
-
-	var mask uint16
-
-	if len(f.Properties.ContentType) > 0 {
-		mask = mask | flagContentType
-	}
-	if len(f.Properties.ContentEncoding) > 0 {
-		mask = mask | flagContentEncoding
-	}
-	if f.Properties.Headers != nil && len(f.Properties.Headers) > 0 {
-		mask = mask | flagHeaders
-	}
-	if f.Properties.DeliveryMode > 0 {
-		mask = mask | flagDeliveryMode
-	}
-	if f.Properties.Priority > 0 {
-		mask = mask | flagPriority
-	}
-	if len(f.Properties.CorrelationId) > 0 {
-		mask = mask | flagCorrelationId
-	}
-	if len(f.Properties.ReplyTo) > 0 {
-		mask = mask | flagReplyTo
-	}
-	if len(f.Properties.Expiration) > 0 {
-		mask = mask | flagExpiration
-	}
-	if len(f.Properties.MessageId) > 0 {
-		mask = mask | flagMessageId
-	}
-	if f.Properties.Timestamp != zeroTime {
-		mask = mask | flagTimestamp
-	}
-	if len(f.Properties.Type) > 0 {
-		mask = mask | flagType
-	}
-	if len(f.Properties.UserId) > 0 {
-		mask = mask | flagUserId
-	}
-	if len(f.Properties.AppId) > 0 {
-		mask = mask | flagAppId
-	}
-
-	if err = binary.Write(&payload, binary.BigEndian, mask); err != nil {
-		return
-	}
-
-	if hasProperty(mask, flagContentType) {
-		if err = writeShortstr(&payload, f.Properties.ContentType); err != nil {
-			return
-		}
-	}
-	if hasProperty(mask, flagContentEncoding) {
-		if err = writeShortstr(&payload, f.Properties.ContentEncoding); err != nil {
-			return
-		}
-	}
-	if hasProperty(mask, flagHeaders) {
-		if err = writeTable(&payload, f.Properties.Headers); err != nil {
-			return
-		}
-	}
-	if hasProperty(mask, flagDeliveryMode) {
-		if err = binary.Write(&payload, binary.BigEndian, f.Properties.DeliveryMode); err != nil {
-			return
-		}
-	}
-	if hasProperty(mask, flagPriority) {
-		if err = binary.Write(&payload, binary.BigEndian, f.Properties.Priority); err != nil {
-			return
-		}
-	}
-	if hasProperty(mask, flagCorrelationId) {
-		if err = writeShortstr(&payload, f.Properties.CorrelationId); err != nil {
-			return
-		}
-	}
-	if hasProperty(mask, flagReplyTo) {
-		if err = writeShortstr(&payload, f.Properties.ReplyTo); err != nil {
-			return
-		}
-	}
-	if hasProperty(mask, flagExpiration) {
-		if err = writeShortstr(&payload, f.Properties.Expiration); err != nil {
-			return
-		}
-	}
-	if hasProperty(mask, flagMessageId) {
-		if err = writeShortstr(&payload, f.Properties.MessageId); err != nil {
-			return
-		}
-	}
-	if hasProperty(mask, flagTimestamp) {
-		if err = binary.Write(&payload, binary.BigEndian, uint64(f.Properties.Timestamp.Unix())); err != nil {
-			return
-		}
-	}
-	if hasProperty(mask, flagType) {
-		if err = writeShortstr(&payload, f.Properties.Type); err != nil {
-			return
-		}
-	}
-	if hasProperty(mask, flagUserId) {
-		if err = writeShortstr(&payload, f.Properties.UserId); err != nil {
-			return
-		}
-	}
-	if hasProperty(mask, flagAppId) {
-		if err = writeShortstr(&payload, f.Properties.AppId); err != nil {
-			return
-		}
-	}
-
-	return writeFrame(w, frameHeader, f.ChannelId, payload.Bytes())
-}
-
-// Body
-//
-// Payload is one byterange from the full body who's size is declared in the
-// Header frame
-func (f *BodyFrame) write(w io.Writer) (err error) {
-	return writeFrame(w, frameBody, f.ChannelId, f.Body)
-}
-
-func writeFrame(w io.Writer, typ uint8, channel uint16, payload []byte) (err error) {
-	end := []byte{frameEnd}
-	size := uint(len(payload))
-
-	_, err = w.Write([]byte{
-		byte(typ),
-		byte((channel & 0xff00) >> 8),
-		byte((channel & 0x00ff) >> 0),
-		byte((size & 0xff000000) >> 24),
-		byte((size & 0x00ff0000) >> 16),
-		byte((size & 0x0000ff00) >> 8),
-		byte((size & 0x000000ff) >> 0),
-	})
-
-	if err != nil {
-		return
-	}
-
-	if _, err = w.Write(payload); err != nil {
-		return
-	}
-
-	if _, err = w.Write(end); err != nil {
-		return
-	}
-
-	return
-}
-
-func writeShortstr(w io.Writer, s string) (err error) {
-	b := []byte(s)
-
-	var length = uint8(len(b))
-
-	if err = binary.Write(w, binary.BigEndian, length); err != nil {
-		return
-	}
-
-	if _, err = w.Write(b[:length]); err != nil {
-		return
-	}
-
-	return
-}
-
-func writeLongstr(w io.Writer, s string) (err error) {
-	b := []byte(s)
-
-	var length = uint32(len(b))
-
-	if err = binary.Write(w, binary.BigEndian, length); err != nil {
-		return
-	}
-
-	if _, err = w.Write(b[:length]); err != nil {
-		return
-	}
-
-	return
-}
-
-/*
-'A': []interface{}
-'D': Decimal
-'F': Table
-'I': int32
-'S': string
-'T': time.Time
-'V': nil
-'b': byte
-'d': float64
-'f': float32
-'l': int64
-'s': int16
-'t': bool
-'x': []byte
-*/
-func writeField(w io.Writer, value interface{}) (err error) {
-	var buf [9]byte
-	var enc []byte
-
-	switch v := value.(type) {
-	case bool:
-		buf[0] = 't'
-		if v {
-			buf[1] = byte(1)
-		} else {
-			buf[1] = byte(0)
-		}
-		enc = buf[:2]
-
-	case byte:
-		buf[0] = 'b'
-		buf[1] = byte(v)
-		enc = buf[:2]
-
-	case int16:
-		buf[0] = 's'
-		binary.BigEndian.PutUint16(buf[1:3], uint16(v))
-		enc = buf[:3]
-
-	case int:
-		buf[0] = 'I'
-		binary.BigEndian.PutUint32(buf[1:5], uint32(v))
-		enc = buf[:5]
-
-	case int32:
-		buf[0] = 'I'
-		binary.BigEndian.PutUint32(buf[1:5], uint32(v))
-		enc = buf[:5]
-
-	case int64:
-		buf[0] = 'l'
-		binary.BigEndian.PutUint64(buf[1:9], uint64(v))
-		enc = buf[:9]
-
-	case float32:
-		buf[0] = 'f'
-		binary.BigEndian.PutUint32(buf[1:5], math.Float32bits(v))
-		enc = buf[:5]
-
-	case float64:
-		buf[0] = 'd'
-		binary.BigEndian.PutUint64(buf[1:9], math.Float64bits(v))
-		enc = buf[:9]
-
-	case Decimal:
-		buf[0] = 'D'
-		buf[1] = byte(v.Scale)
-		binary.BigEndian.PutUint32(buf[2:6], uint32(v.Value))
-		enc = buf[:6]
-
-	case string:
-		buf[0] = 'S'
-		binary.BigEndian.PutUint32(buf[1:5], uint32(len(v)))
-		enc = append(buf[:5], []byte(v)...)
-
-	case []interface{}: // field-array
-		buf[0] = 'A'
-
-		sec := new(bytes.Buffer)
-		for _, val := range v {
-			if err = writeField(sec, val); err != nil {
-				return
-			}
-		}
-
-		binary.BigEndian.PutUint32(buf[1:5], uint32(sec.Len()))
-		if _, err = w.Write(buf[:5]); err != nil {
-			return
-		}
-
-		if _, err = w.Write(sec.Bytes()); err != nil {
-			return
-		}
-
-		return
-
-	case time.Time:
-		buf[0] = 'T'
-		binary.BigEndian.PutUint64(buf[1:9], uint64(v.Unix()))
-		enc = buf[:9]
-
-	case Table:
-		if _, err = w.Write([]byte{'F'}); err != nil {
-			return
-		}
-		return writeTable(w, v)
-
-	case []byte:
-		buf[0] = 'x'
-		binary.BigEndian.PutUint32(buf[1:5], uint32(len(v)))
-		if _, err = w.Write(buf[0:5]); err != nil {
-			return
-		}
-		if _, err = w.Write(v); err != nil {
-			return
-		}
-		return
-
-	case nil:
-		buf[0] = 'V'
-		enc = buf[:1]
-
-	default:
-		return ErrFieldType
-	}
-
-	_, err = w.Write(enc)
-
-	return
-}
-
-func writeTable(w io.Writer, table Table) (err error) {
-	var buf bytes.Buffer
-
-	for key, val := range table {
-		if err = writeShortstr(&buf, key); err != nil {
-			return
-		}
-		if err = writeField(&buf, val); err != nil {
-			return
-		}
-	}
-
-	return writeLongstr(w, buf.String())
-}

tap/extensions/http/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect/http/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect2/http/\* expect

tap/extensions/http/handlers.go

@@ -47,7 +47,7 @@ func replaceForwardedFor(item *api.OutputChannelItem) {
 	item.ConnectionInfo.ClientPort = ""
 }
 
-func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
+func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions, reqResMatcher *requestResponseMatcher) error {
 	streamID, messageHTTP1, isGrpc, err := http2Assembler.readMessage()
 	if err != nil {
 		return err
@@ -58,7 +58,7 @@ func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTi
 	switch messageHTTP1 := messageHTTP1.(type) {
 	case http.Request:
 		ident := fmt.Sprintf(
-			"%s->%s %s->%s %d %s",
+			"%s_%s_%s_%s_%d_%s",
 			tcpID.SrcIP,
 			tcpID.DstIP,
 			tcpID.SrcPort,
@@ -78,7 +78,7 @@ func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTi
 		}
 	case http.Response:
 		ident := fmt.Sprintf(
-			"%s->%s %s->%s %d %s",
+			"%s_%s_%s_%s_%d_%s",
 			tcpID.DstIP,
 			tcpID.SrcIP,
 			tcpID.DstPort,
@@ -110,7 +110,7 @@ func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTi
 	return nil
 }
 
-func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) (switchingProtocolsHTTP2 bool, req *http.Request, err error) {
+func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions, reqResMatcher *requestResponseMatcher) (switchingProtocolsHTTP2 bool, req *http.Request, err error) {
 	req, err = http.ReadRequest(b)
 	if err != nil {
 		return
@@ -130,8 +130,7 @@ func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api
 	req.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
 
 	ident := fmt.Sprintf(
-		"%d_%s:%s_%s:%s_%d_%s",
-		counterPair.StreamId,
+		"%s_%s_%s_%s_%d_%s",
 		tcpID.SrcIP,
 		tcpID.DstIP,
 		tcpID.SrcPort,
@@ -153,7 +152,7 @@ func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api
 	return
 }
 
-func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) (switchingProtocolsHTTP2 bool, err error) {
+func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions, reqResMatcher *requestResponseMatcher) (switchingProtocolsHTTP2 bool, err error) {
 	var res *http.Response
 	res, err = http.ReadResponse(b, nil)
 	if err != nil {
@@ -174,8 +173,7 @@ func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api
 	res.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
 
 	ident := fmt.Sprintf(
-		"%d_%s:%s_%s:%s_%d_%s",
-		counterPair.StreamId,
+		"%s_%s_%s_%s_%d_%s",
 		tcpID.DstIP,
 		tcpID.SrcIP,
 		tcpID.DstPort,

tap/extensions/http/main.go

@@ -76,22 +76,19 @@ const (
 	TypeHttpResponse
 )
 
-func init() {
-	log.Println("Initializing HTTP extension...")
-}
-
 type dissecting string
 
 func (d dissecting) Register(extension *api.Extension) {
 	extension.Protocol = &http11protocol
-	extension.MatcherMap = reqResMatcher.openMessagesMap
 }
 
 func (d dissecting) Ping() {
 	log.Printf("pong %s", http11protocol.Name)
 }
 
-func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
+func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions, _reqResMatcher api.RequestResponseMatcher) error {
+	reqResMatcher := _reqResMatcher.(*requestResponseMatcher)
+
 	var err error
 	isHTTP2, _ := checkIsHTTP2Connection(b, isClient)
 
@@ -124,7 +121,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 		}
 
 		if isHTTP2 {
-			err = handleHTTP2Stream(http2Assembler, tcpID, superTimer, emitter, options)
+			err = handleHTTP2Stream(http2Assembler, tcpID, superTimer, emitter, options, reqResMatcher)
 			if err == io.EOF || err == io.ErrUnexpectedEOF {
 				break
 			} else if err != nil {
@@ -133,7 +130,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			superIdentifier.Protocol = &http11protocol
 		} else if isClient {
 			var req *http.Request
-			switchingProtocolsHTTP2, req, err = handleHTTP1ClientStream(b, tcpID, counterPair, superTimer, emitter, options)
+			switchingProtocolsHTTP2, req, err = handleHTTP1ClientStream(b, tcpID, counterPair, superTimer, emitter, options, reqResMatcher)
 			if err == io.EOF || err == io.ErrUnexpectedEOF {
 				break
 			} else if err != nil {
@@ -144,7 +141,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			// In case of an HTTP2 upgrade, duplicate the HTTP1 request into HTTP2 with stream ID 1
 			if switchingProtocolsHTTP2 {
 				ident := fmt.Sprintf(
-					"%s->%s %s->%s 1 %s",
+					"%s_%s_%s_%s_1_%s",
 					tcpID.SrcIP,
 					tcpID.DstIP,
 					tcpID.SrcPort,
@@ -164,7 +161,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 				}
 			}
 		} else {
-			switchingProtocolsHTTP2, err = handleHTTP1ServerStream(b, tcpID, counterPair, superTimer, emitter, options)
+			switchingProtocolsHTTP2, err = handleHTTP1ServerStream(b, tcpID, counterPair, superTimer, emitter, options, reqResMatcher)
 			if err == io.EOF || err == io.ErrUnexpectedEOF {
 				break
 			} else if err != nil {
@@ -181,7 +178,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	return nil
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.Entry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string, namespace string) *api.Entry {
 	var host, authority, path string
 
 	request := item.Pair.Request.Payload.(map[string]interface{})
@@ -279,6 +276,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 			IP:   item.ConnectionInfo.ServerIP,
 			Port: item.ConnectionInfo.ServerPort,
 		},
+		Namespace:   namespace,
 		Outgoing:    item.ConnectionInfo.IsOutgoing,
 		Request:     reqDetails,
 		Response:    resDetails,
@@ -472,6 +470,10 @@ func (d dissecting) Macros() map[string]string {
 	}
 }
 
+func (d dissecting) NewResponseRequestMatcher() api.RequestResponseMatcher {
+	return createResponseRequestMatcher()
+}
+
 var Dissector dissecting
 
 func NewDissector() api.Dissector {

tap/extensions/http/main_test.go

@@ -11,7 +11,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"sort"
 	"testing"
 	"time"
 
@@ -39,7 +38,6 @@ func TestRegister(t *testing.T) {
 	extension := &api.Extension{}
 	dissector.Register(extension)
 	assert.Equal(t, "http", extension.Protocol.Name)
-	assert.NotNil(t, extension.MatcherMap)
 }
 
 func TestMacros(t *testing.T) {
@@ -123,7 +121,8 @@ func TestDissect(t *testing.T) {
 			SrcPort: "1",
 			DstPort: "2",
 		}
-		err = dissector.Dissect(bufferClient, true, tcpIDClient, counterPair, &api.SuperTimer{}, superIdentifier, emitter, options)
+		reqResMatcher := dissector.NewResponseRequestMatcher()
+		err = dissector.Dissect(bufferClient, true, tcpIDClient, counterPair, &api.SuperTimer{}, superIdentifier, emitter, options, reqResMatcher)
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			panic(err)
 		}
@@ -141,7 +140,7 @@ func TestDissect(t *testing.T) {
 			SrcPort: "2",
 			DstPort: "1",
 		}
-		err = dissector.Dissect(bufferServer, false, tcpIDServer, counterPair, &api.SuperTimer{}, superIdentifier, emitter, options)
+		err = dissector.Dissect(bufferServer, false, tcpIDServer, counterPair, &api.SuperTimer{}, superIdentifier, emitter, options, reqResMatcher)
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			panic(err)
 		}
@@ -155,14 +154,6 @@ func TestDissect(t *testing.T) {
 
 		stop <- true
 
-		sort.Slice(items, func(i, j int) bool {
-			iMarshaled, err := json.Marshal(items[i])
-			assert.Nil(t, err)
-			jMarshaled, err := json.Marshal(items[j])
-			assert.Nil(t, err)
-			return len(iMarshaled) < len(jMarshaled)
-		})
-
 		marshaled, err := json.Marshal(items)
 		assert.Nil(t, err)
 
@@ -214,7 +205,7 @@ func TestAnalyze(t *testing.T) {
 
 		var entries []*api.Entry
 		for _, item := range items {
-			entry := dissector.Analyze(item, "", "")
+			entry := dissector.Analyze(item, "", "", "")
 			entries = append(entries, entry)
 		}
 

tap/extensions/http/matcher.go

@@ -8,16 +8,20 @@ import (
 	"github.com/up9inc/mizu/tap/api"
 )
 
-var reqResMatcher = createResponseRequestMatcher() // global
-
-// Key is {client_addr}:{client_port}->{dest_addr}:{dest_port}_{incremental_counter}
+// Key is {client_addr}_{client_port}_{dest_addr}_{dest_port}_{incremental_counter}_{proto_ident}
 type requestResponseMatcher struct {
 	openMessagesMap *sync.Map
 }
 
-func createResponseRequestMatcher() requestResponseMatcher {
-	newMatcher := &requestResponseMatcher{openMessagesMap: &sync.Map{}}
-	return *newMatcher
+func createResponseRequestMatcher() api.RequestResponseMatcher {
+	return &requestResponseMatcher{openMessagesMap: &sync.Map{}}
+}
+
+func (matcher *requestResponseMatcher) GetMap() *sync.Map {
+	return matcher.openMessagesMap
+}
+
+func (matcher *requestResponseMatcher) SetMaxTry(value int) {
 }
 
 func (matcher *requestResponseMatcher) registerRequest(ident string, request *http.Request, captureTime time.Time, protoMinor int) *api.OutputChannelItem {

tap/extensions/kafka/Makefile

@@ -0,0 +1,16 @@
+skipbin := $$(find bin -mindepth 1 -maxdepth 1)
+skipexpect := $$(find expect -mindepth 1 -maxdepth 1)
+
+test: test-pull-bin test-pull-expect
+	@MIZU_TEST=1 go test -v ./... -coverpkg=./... -race -coverprofile=coverage.out -covermode=atomic
+
+test-update: test-pull-bin
+	@MIZU_TEST=1 TEST_UPDATE=1 go test -v ./... -coverpkg=./... -coverprofile=coverage.out -covermode=atomic
+
+test-pull-bin:
+	@mkdir -p bin
+	@[ "${skipbin}" ] && echo "Skipping downloading BINs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp gs://static.up9.io/mizu/test-pcap/bin/kafka/\*.bin bin
+
+test-pull-expect:
+	@mkdir -p expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect/kafka/\* expect

tap/extensions/kafka/buffer.go

@@ -1,584 +0,0 @@
-package kafka
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"sync"
-	"sync/atomic"
-)
-
-// Bytes is an interface implemented by types that represent immutable
-// sequences of bytes.
-//
-// Bytes values are used to abstract the location where record keys and
-// values are read from (e.g. in-memory buffers, network sockets, files).
-//
-// The Close method should be called to release resources held by the object
-// when the program is done with it.
-//
-// Bytes values are generally not safe to use concurrently from multiple
-// goroutines.
-type Bytes interface {
-	io.ReadCloser
-	// Returns the number of bytes remaining to be read from the payload.
-	Len() int
-}
-
-// NewBytes constructs a Bytes value from b.
-//
-// The returned value references b, it does not make a copy of the backing
-// array.
-//
-// If b is nil, nil is returned to represent a null BYTES value in the kafka
-// protocol.
-func NewBytes(b []byte) Bytes {
-	if b == nil {
-		return nil
-	}
-	r := new(bytesReader)
-	r.Reset(b)
-	return r
-}
-
-// ReadAll is similar to ioutil.ReadAll, but it takes advantage of knowing the
-// length of b to minimize the memory footprint.
-//
-// The function returns a nil slice if b is nil.
-// func ReadAll(b Bytes) ([]byte, error) {
-// 	if b == nil {
-// 		return nil, nil
-// 	}
-// 	s := make([]byte, b.Len())
-// 	_, err := io.ReadFull(b, s)
-// 	return s, err
-// }
-
-type bytesReader struct{ bytes.Reader }
-
-func (*bytesReader) Close() error { return nil }
-
-type refCount uintptr
-
-func (rc *refCount) ref() { atomic.AddUintptr((*uintptr)(rc), 1) }
-
-func (rc *refCount) unref(onZero func()) {
-	if atomic.AddUintptr((*uintptr)(rc), ^uintptr(0)) == 0 {
-		onZero()
-	}
-}
-
-const (
-	// Size of the memory buffer for a single page. We use a farily
-	// large size here (64 KiB) because batches exchanged with kafka
-	// tend to be multiple kilobytes in size, sometimes hundreds.
-	// Using large pages amortizes the overhead of the page metadata
-	// and algorithms to manage the pages.
-	pageSize = 65536
-)
-
-type page struct {
-	refc   refCount
-	offset int64
-	length int
-	buffer *[pageSize]byte
-}
-
-func newPage(offset int64) *page {
-	p, _ := pagePool.Get().(*page)
-	if p != nil {
-		p.offset = offset
-		p.length = 0
-		p.ref()
-	} else {
-		p = &page{
-			refc:   1,
-			offset: offset,
-			buffer: &[pageSize]byte{},
-		}
-	}
-	return p
-}
-
-func (p *page) ref() { p.refc.ref() }
-
-func (p *page) unref() { p.refc.unref(func() { pagePool.Put(p) }) }
-
-func (p *page) slice(begin, end int64) []byte {
-	i, j := begin-p.offset, end-p.offset
-
-	if i < 0 {
-		i = 0
-	} else if i > pageSize {
-		i = pageSize
-	}
-
-	if j < 0 {
-		j = 0
-	} else if j > pageSize {
-		j = pageSize
-	}
-
-	if i < j {
-		return p.buffer[i:j]
-	}
-
-	return nil
-}
-
-func (p *page) Cap() int { return pageSize }
-
-func (p *page) Len() int { return p.length }
-
-func (p *page) Size() int64 { return int64(p.length) }
-
-func (p *page) Truncate(n int) {
-	if n < p.length {
-		p.length = n
-	}
-}
-
-func (p *page) ReadAt(b []byte, off int64) (int, error) {
-	if off -= p.offset; off < 0 || off > pageSize {
-		panic("offset out of range")
-	}
-	if off > int64(p.length) {
-		return 0, nil
-	}
-	return copy(b, p.buffer[off:p.length]), nil
-}
-
-func (p *page) ReadFrom(r io.Reader) (int64, error) {
-	n, err := io.ReadFull(r, p.buffer[p.length:])
-	if err == io.EOF || err == io.ErrUnexpectedEOF {
-		err = nil
-	}
-	p.length += n
-	return int64(n), err
-}
-
-func (p *page) WriteAt(b []byte, off int64) (int, error) {
-	if off -= p.offset; off < 0 || off > pageSize {
-		panic("offset out of range")
-	}
-	n := copy(p.buffer[off:], b)
-	if end := int(off) + n; end > p.length {
-		p.length = end
-	}
-	return n, nil
-}
-
-func (p *page) Write(b []byte) (int, error) {
-	return p.WriteAt(b, p.offset+int64(p.length))
-}
-
-var (
-	_ io.ReaderAt   = (*page)(nil)
-	_ io.ReaderFrom = (*page)(nil)
-	_ io.Writer     = (*page)(nil)
-	_ io.WriterAt   = (*page)(nil)
-)
-
-type pageBuffer struct {
-	refc   refCount
-	pages  contiguousPages
-	length int
-	cursor int
-}
-
-func newPageBuffer() *pageBuffer {
-	b, _ := pageBufferPool.Get().(*pageBuffer)
-	if b != nil {
-		b.cursor = 0
-		b.refc.ref()
-	} else {
-		b = &pageBuffer{
-			refc:  1,
-			pages: make(contiguousPages, 0, 16),
-		}
-	}
-	return b
-}
-
-func (pb *pageBuffer) unref() {
-	pb.refc.unref(func() {
-		pb.pages.unref()
-		pb.pages.clear()
-		pb.pages = pb.pages[:0]
-		pb.length = 0
-		pageBufferPool.Put(pb)
-	})
-}
-
-func (pb *pageBuffer) newPage() *page {
-	return newPage(int64(pb.length))
-}
-
-func (pb *pageBuffer) Close() error {
-	return nil
-}
-
-func (pb *pageBuffer) Len() int {
-	return pb.length - pb.cursor
-}
-
-func (pb *pageBuffer) Size() int64 {
-	return int64(pb.length)
-}
-
-func (pb *pageBuffer) Discard(n int) (int, error) {
-	remain := pb.length - pb.cursor
-	if remain < n {
-		n = remain
-	}
-	pb.cursor += n
-	return n, nil
-}
-
-func (pb *pageBuffer) Truncate(n int) {
-	if n < pb.length {
-		pb.length = n
-
-		if n < pb.cursor {
-			pb.cursor = n
-		}
-
-		for i := range pb.pages {
-			if p := pb.pages[i]; p.length <= n {
-				n -= p.length
-			} else {
-				if n > 0 {
-					pb.pages[i].Truncate(n)
-					i++
-				}
-				pb.pages[i:].unref()
-				pb.pages[i:].clear()
-				pb.pages = pb.pages[:i]
-				break
-			}
-		}
-	}
-}
-
-func (pb *pageBuffer) Seek(offset int64, whence int) (int64, error) {
-	c, err := seek(int64(pb.cursor), int64(pb.length), offset, whence)
-	if err != nil {
-		return -1, err
-	}
-	pb.cursor = int(c)
-	return c, nil
-}
-
-func (pb *pageBuffer) ReadByte() (byte, error) {
-	b := [1]byte{}
-	_, err := pb.Read(b[:])
-	return b[0], err
-}
-
-func (pb *pageBuffer) Read(b []byte) (int, error) {
-	if pb.cursor >= pb.length {
-		return 0, io.EOF
-	}
-	n, err := pb.ReadAt(b, int64(pb.cursor))
-	pb.cursor += n
-	return n, err
-}
-
-func (pb *pageBuffer) ReadAt(b []byte, off int64) (int, error) {
-	return pb.pages.ReadAt(b, off)
-}
-
-func (pb *pageBuffer) ReadFrom(r io.Reader) (int64, error) {
-	if len(pb.pages) == 0 {
-		pb.pages = append(pb.pages, pb.newPage())
-	}
-
-	rn := int64(0)
-
-	for {
-		tail := pb.pages[len(pb.pages)-1]
-		free := tail.Cap() - tail.Len()
-
-		if free == 0 {
-			tail = pb.newPage()
-			free = pageSize
-			pb.pages = append(pb.pages, tail)
-		}
-
-		n, err := tail.ReadFrom(r)
-		pb.length += int(n)
-		rn += n
-		if n < int64(free) {
-			return rn, err
-		}
-	}
-}
-
-func (pb *pageBuffer) WriteString(s string) (int, error) {
-	return pb.Write([]byte(s))
-}
-
-func (pb *pageBuffer) Write(b []byte) (int, error) {
-	wn := len(b)
-	if wn == 0 {
-		return 0, nil
-	}
-
-	if len(pb.pages) == 0 {
-		pb.pages = append(pb.pages, pb.newPage())
-	}
-
-	for len(b) != 0 {
-		tail := pb.pages[len(pb.pages)-1]
-		free := tail.Cap() - tail.Len()
-
-		if len(b) <= free {
-			_, _ = tail.Write(b)
-			pb.length += len(b)
-			break
-		}
-
-		_, _ = tail.Write(b[:free])
-		b = b[free:]
-
-		pb.length += free
-		pb.pages = append(pb.pages, pb.newPage())
-	}
-
-	return wn, nil
-}
-
-func (pb *pageBuffer) WriteAt(b []byte, off int64) (int, error) {
-	n, err := pb.pages.WriteAt(b, off)
-	if err != nil {
-		return n, err
-	}
-	if n < len(b) {
-		_, _ = pb.Write(b[n:])
-	}
-	return len(b), nil
-}
-
-func (pb *pageBuffer) WriteTo(w io.Writer) (int64, error) {
-	var wn int
-	var err error
-	pb.pages.scan(int64(pb.cursor), int64(pb.length), func(b []byte) bool {
-		var n int
-		n, err = w.Write(b)
-		wn += n
-		return err == nil
-	})
-	pb.cursor += wn
-	return int64(wn), err
-}
-
-var (
-	_ io.ReaderAt     = (*pageBuffer)(nil)
-	_ io.ReaderFrom   = (*pageBuffer)(nil)
-	_ io.StringWriter = (*pageBuffer)(nil)
-	_ io.Writer       = (*pageBuffer)(nil)
-	_ io.WriterAt     = (*pageBuffer)(nil)
-	_ io.WriterTo     = (*pageBuffer)(nil)
-
-	pagePool       sync.Pool
-	pageBufferPool sync.Pool
-)
-
-type contiguousPages []*page
-
-func (pages contiguousPages) unref() {
-	for _, p := range pages {
-		p.unref()
-	}
-}
-
-func (pages contiguousPages) clear() {
-	for i := range pages {
-		pages[i] = nil
-	}
-}
-
-func (pages contiguousPages) ReadAt(b []byte, off int64) (int, error) {
-	rn := 0
-
-	for _, p := range pages.slice(off, off+int64(len(b))) {
-		n, _ := p.ReadAt(b, off)
-		b = b[n:]
-		rn += n
-		off += int64(n)
-	}
-
-	return rn, nil
-}
-
-func (pages contiguousPages) WriteAt(b []byte, off int64) (int, error) {
-	wn := 0
-
-	for _, p := range pages.slice(off, off+int64(len(b))) {
-		n, _ := p.WriteAt(b, off)
-		b = b[n:]
-		wn += n
-		off += int64(n)
-	}
-
-	return wn, nil
-}
-
-func (pages contiguousPages) slice(begin, end int64) contiguousPages {
-	i := pages.indexOf(begin)
-	j := pages.indexOf(end)
-	if j < len(pages) {
-		j++
-	}
-	return pages[i:j]
-}
-
-func (pages contiguousPages) indexOf(offset int64) int {
-	if len(pages) == 0 {
-		return 0
-	}
-	return int((offset - pages[0].offset) / pageSize)
-}
-
-func (pages contiguousPages) scan(begin, end int64, f func([]byte) bool) {
-	for _, p := range pages.slice(begin, end) {
-		if !f(p.slice(begin, end)) {
-			break
-		}
-	}
-}
-
-var (
-	_ io.ReaderAt = contiguousPages{}
-	_ io.WriterAt = contiguousPages{}
-)
-
-type pageRef struct {
-	pages  contiguousPages
-	offset int64
-	cursor int64
-	length uint32
-	once   uint32
-}
-
-func (ref *pageRef) unref() {
-	if atomic.CompareAndSwapUint32(&ref.once, 0, 1) {
-		ref.pages.unref()
-		ref.pages.clear()
-		ref.pages = nil
-		ref.offset = 0
-		ref.cursor = 0
-		ref.length = 0
-	}
-}
-
-func (ref *pageRef) Len() int { return int(ref.Size() - ref.cursor) }
-
-func (ref *pageRef) Size() int64 { return int64(ref.length) }
-
-func (ref *pageRef) Close() error { ref.unref(); return nil }
-
-func (ref *pageRef) String() string {
-	return fmt.Sprintf("[offset=%d cursor=%d length=%d]", ref.offset, ref.cursor, ref.length)
-}
-
-func (ref *pageRef) Seek(offset int64, whence int) (int64, error) {
-	c, err := seek(ref.cursor, int64(ref.length), offset, whence)
-	if err != nil {
-		return -1, err
-	}
-	ref.cursor = c
-	return c, nil
-}
-
-func (ref *pageRef) ReadByte() (byte, error) {
-	var c byte
-	var ok bool
-	ref.scan(ref.cursor, func(b []byte) bool {
-		c, ok = b[0], true
-		return false
-	})
-	if ok {
-		ref.cursor++
-	} else {
-		return 0, io.EOF
-	}
-	return c, nil
-}
-
-func (ref *pageRef) Read(b []byte) (int, error) {
-	if ref.cursor >= int64(ref.length) {
-		return 0, io.EOF
-	}
-	n, err := ref.ReadAt(b, ref.cursor)
-	ref.cursor += int64(n)
-	return n, err
-}
-
-func (ref *pageRef) ReadAt(b []byte, off int64) (int, error) {
-	limit := ref.offset + int64(ref.length)
-	off += ref.offset
-
-	if off >= limit {
-		return 0, io.EOF
-	}
-
-	if off+int64(len(b)) > limit {
-		b = b[:limit-off]
-	}
-
-	if len(b) == 0 {
-		return 0, nil
-	}
-
-	n, err := ref.pages.ReadAt(b, off)
-	if n == 0 && err == nil {
-		err = io.EOF
-	}
-	return n, err
-}
-
-func (ref *pageRef) WriteTo(w io.Writer) (wn int64, err error) {
-	ref.scan(ref.cursor, func(b []byte) bool {
-		var n int
-		n, err = w.Write(b)
-		wn += int64(n)
-		return err == nil
-	})
-	ref.cursor += wn
-	return
-}
-
-func (ref *pageRef) scan(off int64, f func([]byte) bool) {
-	begin := ref.offset + off
-	end := ref.offset + int64(ref.length)
-	ref.pages.scan(begin, end, f)
-}
-
-var (
-	_ io.Closer   = (*pageRef)(nil)
-	_ io.Seeker   = (*pageRef)(nil)
-	_ io.Reader   = (*pageRef)(nil)
-	_ io.ReaderAt = (*pageRef)(nil)
-	_ io.WriterTo = (*pageRef)(nil)
-)
-
-func seek(cursor, limit, offset int64, whence int) (int64, error) {
-	switch whence {
-	case io.SeekStart:
-		// absolute offset
-	case io.SeekCurrent:
-		offset = cursor + offset
-	case io.SeekEnd:
-		offset = limit - offset
-	default:
-		return -1, fmt.Errorf("seek: invalid whence value: %d", whence)
-	}
-	if offset < 0 {
-		offset = 0
-	}
-	if offset > limit {
-		offset = limit
-	}
-	return offset, nil
-}

tap/extensions/kafka/cluster.go

@@ -1,143 +0,0 @@
-package kafka
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-	"text/tabwriter"
-)
-
-type Cluster struct {
-	ClusterID  string
-	Controller int32
-	Brokers    map[int32]Broker
-	Topics     map[string]Topic
-}
-
-func (c Cluster) BrokerIDs() []int32 {
-	brokerIDs := make([]int32, 0, len(c.Brokers))
-	for id := range c.Brokers {
-		brokerIDs = append(brokerIDs, id)
-	}
-	sort.Slice(brokerIDs, func(i, j int) bool {
-		return brokerIDs[i] < brokerIDs[j]
-	})
-	return brokerIDs
-}
-
-func (c Cluster) TopicNames() []string {
-	topicNames := make([]string, 0, len(c.Topics))
-	for name := range c.Topics {
-		topicNames = append(topicNames, name)
-	}
-	sort.Strings(topicNames)
-	return topicNames
-}
-
-func (c Cluster) IsZero() bool {
-	return c.ClusterID == "" && c.Controller == 0 && len(c.Brokers) == 0 && len(c.Topics) == 0
-}
-
-func (c Cluster) Format(w fmt.State, _ rune) {
-	tw := new(tabwriter.Writer)
-	fmt.Fprintf(w, "CLUSTER: %q\n\n", c.ClusterID)
-
-	tw.Init(w, 0, 8, 2, ' ', 0)
-	fmt.Fprint(tw, "  BROKER\tHOST\tPORT\tRACK\tCONTROLLER\n")
-
-	for _, id := range c.BrokerIDs() {
-		broker := c.Brokers[id]
-		fmt.Fprintf(tw, "  %d\t%s\t%d\t%s\t%t\n", broker.ID, broker.Host, broker.Port, broker.Rack, broker.ID == c.Controller)
-	}
-
-	tw.Flush()
-	fmt.Fprintln(w)
-
-	tw.Init(w, 0, 8, 2, ' ', 0)
-	fmt.Fprint(tw, "  TOPIC\tPARTITIONS\tBROKERS\n")
-	topicNames := c.TopicNames()
-	brokers := make(map[int32]struct{}, len(c.Brokers))
-	brokerIDs := make([]int32, 0, len(c.Brokers))
-
-	for _, name := range topicNames {
-		topic := c.Topics[name]
-
-		for _, p := range topic.Partitions {
-			for _, id := range p.Replicas {
-				brokers[id] = struct{}{}
-			}
-		}
-
-		for id := range brokers {
-			brokerIDs = append(brokerIDs, id)
-		}
-
-		fmt.Fprintf(tw, "  %s\t%d\t%s\n", topic.Name, len(topic.Partitions), formatBrokerIDs(brokerIDs, -1))
-
-		for id := range brokers {
-			delete(brokers, id)
-		}
-
-		brokerIDs = brokerIDs[:0]
-	}
-
-	tw.Flush()
-	fmt.Fprintln(w)
-
-	if w.Flag('+') {
-		for _, name := range topicNames {
-			fmt.Fprintf(w, "  TOPIC: %q\n\n", name)
-
-			tw.Init(w, 0, 8, 2, ' ', 0)
-			fmt.Fprint(tw, "    PARTITION\tREPLICAS\tISR\tOFFLINE\n")
-
-			for _, p := range c.Topics[name].Partitions {
-				fmt.Fprintf(tw, "    %d\t%s\t%s\t%s\n", p.ID,
-					formatBrokerIDs(p.Replicas, -1),
-					formatBrokerIDs(p.ISR, p.Leader),
-					formatBrokerIDs(p.Offline, -1),
-				)
-			}
-
-			tw.Flush()
-			fmt.Fprintln(w)
-		}
-	}
-}
-
-func formatBrokerIDs(brokerIDs []int32, leader int32) string {
-	if len(brokerIDs) == 0 {
-		return ""
-	}
-
-	if len(brokerIDs) == 1 {
-		return itoa(brokerIDs[0])
-	}
-
-	sort.Slice(brokerIDs, func(i, j int) bool {
-		id1 := brokerIDs[i]
-		id2 := brokerIDs[j]
-
-		if id1 == leader {
-			return true
-		}
-
-		if id2 == leader {
-			return false
-		}
-
-		return id1 < id2
-	})
-
-	brokerNames := make([]string, len(brokerIDs))
-
-	for i, id := range brokerIDs {
-		brokerNames[i] = itoa(id)
-	}
-
-	return strings.Join(brokerNames, ",")
-}
-
-var (
-	_ fmt.Formatter = Cluster{}
-)

tap/extensions/kafka/decode.go

@@ -1,7 +1,6 @@
 package kafka
 
 import (
-	"bytes"
 	"encoding/binary"
 	"fmt"
 	"hash/crc32"
@@ -9,8 +8,6 @@ import (
 	"io/ioutil"
 	"reflect"
 	"strings"
-	"sync"
-	"sync/atomic"
 )
 
 type discarder interface {
@@ -26,15 +23,6 @@ type decoder struct {
 	crc32  uint32
 }
 
-func (d *decoder) Reset(r io.Reader, n int) {
-	d.reader = r
-	d.remain = n
-	d.buffer = [8]byte{}
-	d.err = nil
-	d.table = nil
-	d.crc32 = 0
-}
-
 func (d *decoder) Read(b []byte) (int, error) {
 	if d.err != nil {
 		return 0, d.err
@@ -483,52 +471,3 @@ func decodeReadInt32(b []byte) int32 {
 func decodeReadInt64(b []byte) int64 {
 	return int64(binary.BigEndian.Uint64(b))
 }
-
-func Unmarshal(data []byte, version int16, value interface{}) error {
-	typ := elemTypeOf(value)
-	cache, _ := unmarshalers.Load().(map[versionedType]decodeFunc)
-	key := versionedType{typ: typ, version: version}
-	decode := cache[key]
-
-	if decode == nil {
-		decode = decodeFuncOf(reflect.TypeOf(value).Elem(), version, false, structTag{
-			MinVersion: -1,
-			MaxVersion: -1,
-			TagID:      -2,
-			Compact:    true,
-			Nullable:   true,
-		})
-
-		newCache := make(map[versionedType]decodeFunc, len(cache)+1)
-		newCache[key] = decode
-
-		for typ, fun := range cache {
-			newCache[typ] = fun
-		}
-
-		unmarshalers.Store(newCache)
-	}
-
-	d, _ := decoders.Get().(*decoder)
-	if d == nil {
-		d = &decoder{reader: bytes.NewReader(nil)}
-	}
-
-	d.remain = len(data)
-	r, _ := d.reader.(*bytes.Reader)
-	r.Reset(data)
-
-	defer func() {
-		r.Reset(nil)
-		d.Reset(r, 0)
-		decoders.Put(d)
-	}()
-
-	decode(d, valueOf(value))
-	return dontExpectEOF(d.err)
-}
-
-var (
-	decoders     sync.Pool    // *decoder
-	unmarshalers atomic.Value // map[versionedType]decodeFunc
-)

tap/extensions/kafka/discard.go

@@ -1,16 +0,0 @@
-package kafka
-
-import "bufio"
-
-func discardN(r *bufio.Reader, sz int, n int) (int, error) {
-	var err error
-	if n <= sz {
-		n, err = r.Discard(n)
-	} else {
-		n, err = r.Discard(sz)
-		if err == nil {
-			err = errShortRead
-		}
-	}
-	return sz - n, err
-}

tap/extensions/kafka/encode.go

@@ -1,537 +0,0 @@
-package kafka
-
-import (
-	"bytes"
-	"encoding/binary"
-	"hash/crc32"
-	"io"
-	"reflect"
-	"sync"
-	"sync/atomic"
-)
-
-type encoder struct {
-	writer io.Writer
-	err    error
-	table  *crc32.Table
-	crc32  uint32
-	buffer [32]byte
-}
-
-type encoderChecksum struct {
-	reader  io.Reader
-	encoder *encoder
-}
-
-func (e *encoderChecksum) Read(b []byte) (int, error) {
-	n, err := e.reader.Read(b)
-	if n > 0 {
-		e.encoder.update(b[:n])
-	}
-	return n, err
-}
-
-func (e *encoder) Reset(w io.Writer) {
-	e.writer = w
-	e.err = nil
-	e.table = nil
-	e.crc32 = 0
-	e.buffer = [32]byte{}
-}
-
-func (e *encoder) ReadFrom(r io.Reader) (int64, error) {
-	if e.table != nil {
-		r = &encoderChecksum{
-			reader:  r,
-			encoder: e,
-		}
-	}
-	return io.Copy(e.writer, r)
-}
-
-func (e *encoder) Write(b []byte) (int, error) {
-	if e.err != nil {
-		return 0, e.err
-	}
-	n, err := e.writer.Write(b)
-	if n > 0 {
-		e.update(b[:n])
-	}
-	if err != nil {
-		e.err = err
-	}
-	return n, err
-}
-
-func (e *encoder) WriteByte(b byte) error {
-	e.buffer[0] = b
-	_, err := e.Write(e.buffer[:1])
-	return err
-}
-
-func (e *encoder) WriteString(s string) (int, error) {
-	// This implementation is an optimization to avoid the heap allocation that
-	// would occur when converting the string to a []byte to call crc32.Update.
-	//
-	// Strings are rarely long in the kafka protocol, so the use of a 32 byte
-	// buffer is a good comprise between keeping the encoder value small and
-	// limiting the number of calls to Write.
-	//
-	// We introduced this optimization because memory profiles on the benchmarks
-	// showed that most heap allocations were caused by this code path.
-	n := 0
-
-	for len(s) != 0 {
-		c := copy(e.buffer[:], s)
-		w, err := e.Write(e.buffer[:c])
-		n += w
-		if err != nil {
-			return n, err
-		}
-		s = s[c:]
-	}
-
-	return n, nil
-}
-
-func (e *encoder) update(b []byte) {
-	if e.table != nil {
-		e.crc32 = crc32.Update(e.crc32, e.table, b)
-	}
-}
-
-func (e *encoder) encodeBool(v value) {
-	b := int8(0)
-	if v.bool() {
-		b = 1
-	}
-	e.writeInt8(b)
-}
-
-func (e *encoder) encodeInt8(v value) {
-	e.writeInt8(v.int8())
-}
-
-func (e *encoder) encodeInt16(v value) {
-	e.writeInt16(v.int16())
-}
-
-func (e *encoder) encodeInt32(v value) {
-	e.writeInt32(v.int32())
-}
-
-func (e *encoder) encodeInt64(v value) {
-	e.writeInt64(v.int64())
-}
-
-func (e *encoder) encodeString(v value) {
-	e.writeString(v.string())
-}
-
-func (e *encoder) encodeCompactString(v value) {
-	e.writeCompactString(v.string())
-}
-
-func (e *encoder) encodeNullString(v value) {
-	e.writeNullString(v.string())
-}
-
-func (e *encoder) encodeCompactNullString(v value) {
-	e.writeCompactNullString(v.string())
-}
-
-func (e *encoder) encodeBytes(v value) {
-	e.writeBytes(v.bytes())
-}
-
-func (e *encoder) encodeCompactBytes(v value) {
-	e.writeCompactBytes(v.bytes())
-}
-
-func (e *encoder) encodeNullBytes(v value) {
-	e.writeNullBytes(v.bytes())
-}
-
-func (e *encoder) encodeCompactNullBytes(v value) {
-	e.writeCompactNullBytes(v.bytes())
-}
-
-func (e *encoder) encodeArray(v value, elemType reflect.Type, encodeElem encodeFunc) {
-	a := v.array(elemType)
-	n := a.length()
-	e.writeInt32(int32(n))
-
-	for i := 0; i < n; i++ {
-		encodeElem(e, a.index(i))
-	}
-}
-
-func (e *encoder) encodeCompactArray(v value, elemType reflect.Type, encodeElem encodeFunc) {
-	a := v.array(elemType)
-	n := a.length()
-	e.writeUnsignedVarInt(uint64(n + 1))
-
-	for i := 0; i < n; i++ {
-		encodeElem(e, a.index(i))
-	}
-}
-
-func (e *encoder) encodeNullArray(v value, elemType reflect.Type, encodeElem encodeFunc) {
-	a := v.array(elemType)
-	if a.isNil() {
-		e.writeInt32(-1)
-		return
-	}
-
-	n := a.length()
-	e.writeInt32(int32(n))
-
-	for i := 0; i < n; i++ {
-		encodeElem(e, a.index(i))
-	}
-}
-
-func (e *encoder) encodeCompactNullArray(v value, elemType reflect.Type, encodeElem encodeFunc) {
-	a := v.array(elemType)
-	if a.isNil() {
-		e.writeUnsignedVarInt(0)
-		return
-	}
-
-	n := a.length()
-	e.writeUnsignedVarInt(uint64(n + 1))
-	for i := 0; i < n; i++ {
-		encodeElem(e, a.index(i))
-	}
-}
-
-func (e *encoder) writeInt8(i int8) {
-	writeInt8(e.buffer[:1], i)
-	_, _ = e.Write(e.buffer[:1])
-}
-
-func (e *encoder) writeInt16(i int16) {
-	writeInt16(e.buffer[:2], i)
-	_, _ = e.Write(e.buffer[:2])
-}
-
-func (e *encoder) writeInt32(i int32) {
-	writeInt32(e.buffer[:4], i)
-	_, _ = e.Write(e.buffer[:4])
-}
-
-func (e *encoder) writeInt64(i int64) {
-	writeInt64(e.buffer[:8], i)
-	_, _ = e.Write(e.buffer[:8])
-}
-
-func (e *encoder) writeString(s string) {
-	e.writeInt16(int16(len(s)))
-	_, _ = e.WriteString(s)
-}
-
-func (e *encoder) writeCompactString(s string) {
-	e.writeUnsignedVarInt(uint64(len(s)) + 1)
-	_, _ = e.WriteString(s)
-}
-
-func (e *encoder) writeNullString(s string) {
-	if s == "" {
-		e.writeInt16(-1)
-	} else {
-		e.writeInt16(int16(len(s)))
-		_, _ = e.WriteString(s)
-	}
-}
-
-func (e *encoder) writeCompactNullString(s string) {
-	if s == "" {
-		e.writeUnsignedVarInt(0)
-	} else {
-		e.writeUnsignedVarInt(uint64(len(s)) + 1)
-		_, _ = e.WriteString(s)
-	}
-}
-
-func (e *encoder) writeBytes(b []byte) {
-	e.writeInt32(int32(len(b)))
-	_, _ = e.Write(b)
-}
-
-func (e *encoder) writeCompactBytes(b []byte) {
-	e.writeUnsignedVarInt(uint64(len(b)) + 1)
-	_, _ = e.Write(b)
-}
-
-func (e *encoder) writeNullBytes(b []byte) {
-	if b == nil {
-		e.writeInt32(-1)
-	} else {
-		e.writeInt32(int32(len(b)))
-		_, _ = e.Write(b)
-	}
-}
-
-func (e *encoder) writeCompactNullBytes(b []byte) {
-	if b == nil {
-		e.writeUnsignedVarInt(0)
-	} else {
-		e.writeUnsignedVarInt(uint64(len(b)) + 1)
-		_, _ = e.Write(b)
-	}
-}
-
-func (e *encoder) writeUnsignedVarInt(i uint64) {
-	b := e.buffer[:]
-	n := 0
-
-	for i >= 0x80 && n < len(b) {
-		b[n] = byte(i) | 0x80
-		i >>= 7
-		n++
-	}
-
-	if n < len(b) {
-		b[n] = byte(i)
-		n++
-	}
-
-	_, _ = e.Write(b[:n])
-}
-
-type encodeFunc func(*encoder, value)
-
-var (
-	_ io.ReaderFrom   = (*encoder)(nil)
-	_ io.Writer       = (*encoder)(nil)
-	_ io.ByteWriter   = (*encoder)(nil)
-	_ io.StringWriter = (*encoder)(nil)
-
-	writerTo = reflect.TypeOf((*io.WriterTo)(nil)).Elem()
-)
-
-func encodeFuncOf(typ reflect.Type, version int16, flexible bool, tag structTag) encodeFunc {
-	if reflect.PtrTo(typ).Implements(writerTo) {
-		return writerEncodeFuncOf(typ)
-	}
-	switch typ.Kind() {
-	case reflect.Bool:
-		return (*encoder).encodeBool
-	case reflect.Int8:
-		return (*encoder).encodeInt8
-	case reflect.Int16:
-		return (*encoder).encodeInt16
-	case reflect.Int32:
-		return (*encoder).encodeInt32
-	case reflect.Int64:
-		return (*encoder).encodeInt64
-	case reflect.String:
-		return stringEncodeFuncOf(flexible, tag)
-	case reflect.Struct:
-		return structEncodeFuncOf(typ, version, flexible)
-	case reflect.Slice:
-		if typ.Elem().Kind() == reflect.Uint8 { // []byte
-			return bytesEncodeFuncOf(flexible, tag)
-		}
-		return arrayEncodeFuncOf(typ, version, flexible, tag)
-	default:
-		panic("unsupported type: " + typ.String())
-	}
-}
-
-func stringEncodeFuncOf(flexible bool, tag structTag) encodeFunc {
-	switch {
-	case flexible && tag.Nullable:
-		// In flexible messages, all strings are compact
-		return (*encoder).encodeCompactNullString
-	case flexible:
-		// In flexible messages, all strings are compact
-		return (*encoder).encodeCompactString
-	case tag.Nullable:
-		return (*encoder).encodeNullString
-	default:
-		return (*encoder).encodeString
-	}
-}
-
-func bytesEncodeFuncOf(flexible bool, tag structTag) encodeFunc {
-	switch {
-	case flexible && tag.Nullable:
-		// In flexible messages, all arrays are compact
-		return (*encoder).encodeCompactNullBytes
-	case flexible:
-		// In flexible messages, all arrays are compact
-		return (*encoder).encodeCompactBytes
-	case tag.Nullable:
-		return (*encoder).encodeNullBytes
-	default:
-		return (*encoder).encodeBytes
-	}
-}
-
-func structEncodeFuncOf(typ reflect.Type, version int16, flexible bool) encodeFunc {
-	type field struct {
-		encode encodeFunc
-		index  index
-		tagID  int
-	}
-
-	var fields []field
-	var taggedFields []field
-
-	forEachStructField(typ, func(typ reflect.Type, index index, tag string) {
-		if typ.Size() != 0 { // skip struct{}
-			forEachStructTag(tag, func(tag structTag) bool {
-				if tag.MinVersion <= version && version <= tag.MaxVersion {
-					f := field{
-						encode: encodeFuncOf(typ, version, flexible, tag),
-						index:  index,
-						tagID:  tag.TagID,
-					}
-
-					if tag.TagID < -1 {
-						// Normal required field
-						fields = append(fields, f)
-					} else {
-						// Optional tagged field (flexible messages only)
-						taggedFields = append(taggedFields, f)
-					}
-					return false
-				}
-				return true
-			})
-		}
-	})
-
-	return func(e *encoder, v value) {
-		for i := range fields {
-			f := &fields[i]
-			f.encode(e, v.fieldByIndex(f.index))
-		}
-
-		if flexible {
-			// See https://cwiki.apache.org/confluence/display/KAFKA/KIP-482%3A+The+Kafka+Protocol+should+Support+Optional+Tagged+Fields
-			// for details of tag buffers in "flexible" messages.
-			e.writeUnsignedVarInt(uint64(len(taggedFields)))
-
-			for i := range taggedFields {
-				f := &taggedFields[i]
-				e.writeUnsignedVarInt(uint64(f.tagID))
-
-				buf := &bytes.Buffer{}
-				se := &encoder{writer: buf}
-				f.encode(se, v.fieldByIndex(f.index))
-				e.writeUnsignedVarInt(uint64(buf.Len()))
-				_, _ = e.Write(buf.Bytes())
-			}
-		}
-	}
-}
-
-func arrayEncodeFuncOf(typ reflect.Type, version int16, flexible bool, tag structTag) encodeFunc {
-	elemType := typ.Elem()
-	elemFunc := encodeFuncOf(elemType, version, flexible, tag)
-	switch {
-	case flexible && tag.Nullable:
-		// In flexible messages, all arrays are compact
-		return func(e *encoder, v value) { e.encodeCompactNullArray(v, elemType, elemFunc) }
-	case flexible:
-		// In flexible messages, all arrays are compact
-		return func(e *encoder, v value) { e.encodeCompactArray(v, elemType, elemFunc) }
-	case tag.Nullable:
-		return func(e *encoder, v value) { e.encodeNullArray(v, elemType, elemFunc) }
-	default:
-		return func(e *encoder, v value) { e.encodeArray(v, elemType, elemFunc) }
-	}
-}
-
-func writerEncodeFuncOf(typ reflect.Type) encodeFunc {
-	typ = reflect.PtrTo(typ)
-	return func(e *encoder, v value) {
-		// Optimization to write directly into the buffer when the encoder
-		// does no need to compute a crc32 checksum.
-		w := io.Writer(e)
-		if e.table == nil {
-			w = e.writer
-		}
-		_, err := v.iface(typ).(io.WriterTo).WriteTo(w)
-		if err != nil {
-			e.err = err
-		}
-	}
-}
-
-func writeInt8(b []byte, i int8) {
-	b[0] = byte(i)
-}
-
-func writeInt16(b []byte, i int16) {
-	binary.BigEndian.PutUint16(b, uint16(i))
-}
-
-func writeInt32(b []byte, i int32) {
-	binary.BigEndian.PutUint32(b, uint32(i))
-}
-
-func writeInt64(b []byte, i int64) {
-	binary.BigEndian.PutUint64(b, uint64(i))
-}
-
-func Marshal(version int16, value interface{}) ([]byte, error) {
-	typ := typeOf(value)
-	cache, _ := marshalers.Load().(map[versionedType]encodeFunc)
-	key := versionedType{typ: typ, version: version}
-	encode := cache[key]
-
-	if encode == nil {
-		encode = encodeFuncOf(reflect.TypeOf(value), version, false, structTag{
-			MinVersion: -1,
-			MaxVersion: -1,
-			TagID:      -2,
-			Compact:    true,
-			Nullable:   true,
-		})
-
-		newCache := make(map[versionedType]encodeFunc, len(cache)+1)
-		newCache[key] = encode
-
-		for typ, fun := range cache {
-			newCache[typ] = fun
-		}
-
-		marshalers.Store(newCache)
-	}
-
-	e, _ := encoders.Get().(*encoder)
-	if e == nil {
-		e = &encoder{writer: new(bytes.Buffer)}
-	}
-
-	b, _ := e.writer.(*bytes.Buffer)
-	defer func() {
-		b.Reset()
-		e.Reset(b)
-		encoders.Put(e)
-	}()
-
-	encode(e, nonAddressableValueOf(value))
-
-	if e.err != nil {
-		return nil, e.err
-	}
-
-	buf := b.Bytes()
-	out := make([]byte, len(buf))
-	copy(out, buf)
-	return out, nil
-}
-
-type versionedType struct {
-	typ     _type
-	version int16
-}
-
-var (
-	encoders   sync.Pool    // *encoder
-	marshalers atomic.Value // map[versionedType]encodeFunc
-)

tap/extensions/kafka/error.go

@@ -1,91 +0,0 @@
-package kafka
-
-import (
-	"fmt"
-)
-
-// Error represents client-side protocol errors.
-type Error string
-
-func (e Error) Error() string { return string(e) }
-
-func Errorf(msg string, args ...interface{}) Error {
-	return Error(fmt.Sprintf(msg, args...))
-}
-
-const (
-	// ErrNoTopic is returned when a request needs to be sent to a specific
-	ErrNoTopic Error = "topic not found"
-
-	// ErrNoPartition is returned when a request needs to be sent to a specific
-	// partition, but the client did not find it in the cluster metadata.
-	ErrNoPartition Error = "topic partition not found"
-
-	// ErrNoLeader is returned when a request needs to be sent to a partition
-	// leader, but the client could not determine what the leader was at this
-	// time.
-	ErrNoLeader Error = "topic partition has no leader"
-
-	// ErrNoRecord is returned when attempting to write a message containing an
-	// empty record set (which kafka forbids).
-	//
-	// We handle this case client-side because kafka will close the connection
-	// that it received an empty produce request on, causing all concurrent
-	// requests to be aborted.
-	ErrNoRecord Error = "record set contains no records"
-
-	// ErrNoReset is returned by ResetRecordReader when the record reader does
-	// not support being reset.
-	ErrNoReset Error = "record sequence does not support reset"
-)
-
-type TopicError struct {
-	Topic string
-	Err   error
-}
-
-func NewTopicError(topic string, err error) *TopicError {
-	return &TopicError{Topic: topic, Err: err}
-}
-
-func NewErrNoTopic(topic string) *TopicError {
-	return NewTopicError(topic, ErrNoTopic)
-}
-
-func (e *TopicError) Error() string {
-	return fmt.Sprintf("%v (topic=%q)", e.Err, e.Topic)
-}
-
-func (e *TopicError) Unwrap() error {
-	return e.Err
-}
-
-type TopicPartitionError struct {
-	Topic     string
-	Partition int32
-	Err       error
-}
-
-func NewTopicPartitionError(topic string, partition int32, err error) *TopicPartitionError {
-	return &TopicPartitionError{
-		Topic:     topic,
-		Partition: partition,
-		Err:       err,
-	}
-}
-
-func NewErrNoPartition(topic string, partition int32) *TopicPartitionError {
-	return NewTopicPartitionError(topic, partition, ErrNoPartition)
-}
-
-func NewErrNoLeader(topic string, partition int32) *TopicPartitionError {
-	return NewTopicPartitionError(topic, partition, ErrNoLeader)
-}
-
-func (e *TopicPartitionError) Error() string {
-	return fmt.Sprintf("%v (topic=%q partition=%d)", e.Err, e.Topic, e.Partition)
-}
-
-func (e *TopicPartitionError) Unwrap() error {
-	return e.Err
-}

tap/extensions/kafka/go.mod

@@ -6,14 +6,18 @@ require (
 	github.com/fatih/camelcase v1.0.0
 	github.com/ohler55/ojg v1.12.12
 	github.com/segmentio/kafka-go v0.4.27
+	github.com/stretchr/testify v1.6.1
 	github.com/up9inc/mizu/tap/api v0.0.0
 )
 
 require (
+	github.com/davecgh/go-spew v1.1.0 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/martian v2.1.0+incompatible // indirect
 	github.com/klauspost/compress v1.14.2 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api

tap/extensions/kafka/go.sum

@@ -1,3 +1,4 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
@@ -25,10 +26,12 @@ github.com/ohler55/ojg v1.12.12/go.mod h1:LBbIVRAgoFbYBXQhRhuEpaJIqq+goSO63/FQ+n
 github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
 github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/segmentio/kafka-go v0.4.27 h1:sIhEozeL/TLN2mZ5dkG462vcGEWYKS+u31sXPjKhAM4=
 github.com/segmentio/kafka-go v0.4.27/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
@@ -40,5 +43,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tap/extensions/kafka/helpers.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"reflect"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -897,6 +898,10 @@ func representMapAsTable(mapData map[string]interface{}, selectorPrefix string,
 		})
 	}
 
+	sort.Slice(table, func(i, j int) bool {
+		return table[i].Name < table[j].Name
+	})
+
 	obj, _ := json.Marshal(table)
 	representation = string(obj)
 	return

tap/extensions/kafka/main.go

@@ -25,35 +25,31 @@ var _protocol api.Protocol = api.Protocol{
 	Priority:        2,
 }
 
-func init() {
-	log.Println("Initializing Kafka extension...")
-}
-
 type dissecting string
 
 func (d dissecting) Register(extension *api.Extension) {
 	extension.Protocol = &_protocol
-	extension.MatcherMap = reqResMatcher.openMessagesMap
 }
 
 func (d dissecting) Ping() {
 	log.Printf("pong %s", _protocol.Name)
 }
 
-func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
+func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions, _reqResMatcher api.RequestResponseMatcher) error {
+	reqResMatcher := _reqResMatcher.(*requestResponseMatcher)
 	for {
 		if superIdentifier.Protocol != nil && superIdentifier.Protocol != &_protocol {
 			return errors.New("Identified by another protocol")
 		}
 
 		if isClient {
-			_, _, err := ReadRequest(b, tcpID, counterPair, superTimer)
+			_, _, err := ReadRequest(b, tcpID, counterPair, superTimer, reqResMatcher)
 			if err != nil {
 				return err
 			}
 			superIdentifier.Protocol = &_protocol
 		} else {
-			err := ReadResponse(b, tcpID, counterPair, superTimer, emitter)
+			err := ReadResponse(b, tcpID, counterPair, superTimer, emitter, reqResMatcher)
 			if err != nil {
 				return err
 			}
@@ -62,7 +58,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 	}
 }
 
-func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.Entry {
+func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string, namespace string) *api.Entry {
 	request := item.Pair.Request.Payload.(map[string]interface{})
 	reqDetails := request["details"].(map[string]interface{})
 	apiKey := ApiKey(reqDetails["apiKey"].(float64))
@@ -158,6 +154,7 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
 			IP:   item.ConnectionInfo.ServerIP,
 			Port: item.ConnectionInfo.ServerPort,
 		},
+		Namespace:   namespace,
 		Outgoing:    item.ConnectionInfo.IsOutgoing,
 		Request:     reqDetails,
 		Response:    item.Pair.Response.Payload.(map[string]interface{})["details"].(map[string]interface{}),
@@ -215,6 +212,10 @@ func (d dissecting) Macros() map[string]string {
 	}
 }
 
+func (d dissecting) NewResponseRequestMatcher() api.RequestResponseMatcher {
+	return createResponseRequestMatcher()
+}
+
 var Dissector dissecting
 
 func NewDissector() api.Dissector {

tap/extensions/kafka/main_test.go

@@ -0,0 +1,290 @@
+package kafka
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/up9inc/mizu/tap/api"
+)
+
+const (
+	binDir          = "bin"
+	patternBin      = "*_req.bin"
+	patternDissect  = "*.json"
+	msgDissecting   = "Dissecting:"
+	msgAnalyzing    = "Analyzing:"
+	msgRepresenting = "Representing:"
+	respSuffix      = "_res.bin"
+	expectDir       = "expect"
+	dissectDir      = "dissect"
+	analyzeDir      = "analyze"
+	representDir    = "represent"
+	testUpdate      = "TEST_UPDATE"
+)
+
+func TestRegister(t *testing.T) {
+	dissector := NewDissector()
+	extension := &api.Extension{}
+	dissector.Register(extension)
+	assert.Equal(t, "kafka", extension.Protocol.Name)
+}
+
+func TestMacros(t *testing.T) {
+	expectedMacros := map[string]string{
+		"kafka": `proto.name == "kafka"`,
+	}
+	dissector := NewDissector()
+	macros := dissector.Macros()
+	assert.Equal(t, expectedMacros, macros)
+}
+
+func TestPing(t *testing.T) {
+	dissector := NewDissector()
+	dissector.Ping()
+}
+
+func TestDissect(t *testing.T) {
+	_, testUpdateEnabled := os.LookupEnv(testUpdate)
+
+	expectDirDissect := path.Join(expectDir, dissectDir)
+
+	if testUpdateEnabled {
+		os.RemoveAll(expectDirDissect)
+		err := os.MkdirAll(expectDirDissect, 0775)
+		assert.Nil(t, err)
+	}
+
+	dissector := NewDissector()
+	paths, err := filepath.Glob(path.Join(binDir, patternBin))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	options := &api.TrafficFilteringOptions{
+		IgnoredUserAgents: []string{},
+	}
+
+	for _, _path := range paths {
+		basePath := _path[:len(_path)-8]
+
+		// Channel to verify the output
+		itemChannel := make(chan *api.OutputChannelItem)
+		var emitter api.Emitter = &api.Emitting{
+			AppStats:      &api.AppStats{},
+			OutputChannel: itemChannel,
+		}
+
+		var items []*api.OutputChannelItem
+		stop := make(chan bool)
+
+		go func() {
+			for {
+				select {
+				case <-stop:
+					return
+				case item := <-itemChannel:
+					items = append(items, item)
+				}
+			}
+		}()
+
+		// Stream level
+		counterPair := &api.CounterPair{
+			Request:  0,
+			Response: 0,
+		}
+		superIdentifier := &api.SuperIdentifier{}
+
+		// Request
+		pathClient := _path
+		fmt.Printf("%s %s\n", msgDissecting, pathClient)
+		fileClient, err := os.Open(pathClient)
+		assert.Nil(t, err)
+
+		bufferClient := bufio.NewReader(fileClient)
+		tcpIDClient := &api.TcpID{
+			SrcIP:   "1",
+			DstIP:   "2",
+			SrcPort: "1",
+			DstPort: "2",
+		}
+		reqResMatcher := dissector.NewResponseRequestMatcher()
+		reqResMatcher.SetMaxTry(10)
+		err = dissector.Dissect(bufferClient, true, tcpIDClient, counterPair, &api.SuperTimer{}, superIdentifier, emitter, options, reqResMatcher)
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+			log.Println(err)
+		}
+
+		// Response
+		pathServer := basePath + respSuffix
+		fmt.Printf("%s %s\n", msgDissecting, pathServer)
+		fileServer, err := os.Open(pathServer)
+		assert.Nil(t, err)
+
+		bufferServer := bufio.NewReader(fileServer)
+		tcpIDServer := &api.TcpID{
+			SrcIP:   "2",
+			DstIP:   "1",
+			SrcPort: "2",
+			DstPort: "1",
+		}
+		err = dissector.Dissect(bufferServer, false, tcpIDServer, counterPair, &api.SuperTimer{}, superIdentifier, emitter, options, reqResMatcher)
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+			log.Println(err)
+		}
+
+		fileClient.Close()
+		fileServer.Close()
+
+		pathExpect := path.Join(expectDirDissect, fmt.Sprintf("%s.json", basePath[4:]))
+
+		time.Sleep(10 * time.Millisecond)
+
+		stop <- true
+
+		marshaled, err := json.Marshal(items)
+		assert.Nil(t, err)
+
+		if testUpdateEnabled {
+			if len(items) > 0 {
+				err = os.WriteFile(pathExpect, marshaled, 0644)
+				assert.Nil(t, err)
+			}
+		} else {
+			if _, err := os.Stat(pathExpect); errors.Is(err, os.ErrNotExist) {
+				assert.Len(t, items, 0)
+			} else {
+				expectedBytes, err := ioutil.ReadFile(pathExpect)
+				assert.Nil(t, err)
+
+				assert.JSONEq(t, string(expectedBytes), string(marshaled))
+			}
+		}
+	}
+}
+
+func TestAnalyze(t *testing.T) {
+	_, testUpdateEnabled := os.LookupEnv(testUpdate)
+
+	expectDirDissect := path.Join(expectDir, dissectDir)
+	expectDirAnalyze := path.Join(expectDir, analyzeDir)
+
+	if testUpdateEnabled {
+		os.RemoveAll(expectDirAnalyze)
+		err := os.MkdirAll(expectDirAnalyze, 0775)
+		assert.Nil(t, err)
+	}
+
+	dissector := NewDissector()
+	paths, err := filepath.Glob(path.Join(expectDirDissect, patternDissect))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, _path := range paths {
+		fmt.Printf("%s %s\n", msgAnalyzing, _path)
+
+		bytes, err := ioutil.ReadFile(_path)
+		assert.Nil(t, err)
+
+		var items []*api.OutputChannelItem
+		err = json.Unmarshal(bytes, &items)
+		assert.Nil(t, err)
+
+		var entries []*api.Entry
+		for _, item := range items {
+			entry := dissector.Analyze(item, "", "", "")
+			entries = append(entries, entry)
+		}
+
+		pathExpect := path.Join(expectDirAnalyze, filepath.Base(_path))
+
+		marshaled, err := json.Marshal(entries)
+		assert.Nil(t, err)
+
+		if testUpdateEnabled {
+			if len(entries) > 0 {
+				err = os.WriteFile(pathExpect, marshaled, 0644)
+				assert.Nil(t, err)
+			}
+		} else {
+			if _, err := os.Stat(pathExpect); errors.Is(err, os.ErrNotExist) {
+				assert.Len(t, items, 0)
+			} else {
+				expectedBytes, err := ioutil.ReadFile(pathExpect)
+				assert.Nil(t, err)
+
+				assert.JSONEq(t, string(expectedBytes), string(marshaled))
+			}
+		}
+	}
+}
+
+func TestRepresent(t *testing.T) {
+	_, testUpdateEnabled := os.LookupEnv(testUpdate)
+
+	expectDirAnalyze := path.Join(expectDir, analyzeDir)
+	expectDirRepresent := path.Join(expectDir, representDir)
+
+	if testUpdateEnabled {
+		os.RemoveAll(expectDirRepresent)
+		err := os.MkdirAll(expectDirRepresent, 0775)
+		assert.Nil(t, err)
+	}
+
+	dissector := NewDissector()
+	paths, err := filepath.Glob(path.Join(expectDirAnalyze, patternDissect))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, _path := range paths {
+		fmt.Printf("%s %s\n", msgRepresenting, _path)
+
+		bytes, err := ioutil.ReadFile(_path)
+		assert.Nil(t, err)
+
+		var entries []*api.Entry
+		err = json.Unmarshal(bytes, &entries)
+		assert.Nil(t, err)
+
+		var objects []string
+		for _, entry := range entries {
+			object, _, err := dissector.Represent(entry.Request, entry.Response)
+			assert.Nil(t, err)
+			objects = append(objects, string(object))
+		}
+
+		pathExpect := path.Join(expectDirRepresent, filepath.Base(_path))
+
+		marshaled, err := json.Marshal(objects)
+		assert.Nil(t, err)
+
+		if testUpdateEnabled {
+			if len(objects) > 0 {
+				err = os.WriteFile(pathExpect, marshaled, 0644)
+				assert.Nil(t, err)
+			}
+		} else {
+			if _, err := os.Stat(pathExpect); errors.Is(err, os.ErrNotExist) {
+				assert.Len(t, objects, 0)
+			} else {
+				expectedBytes, err := ioutil.ReadFile(pathExpect)
+				assert.Nil(t, err)
+
+				assert.JSONEq(t, string(expectedBytes), string(marshaled))
+			}
+		}
+	}
+}

tap/extensions/kafka/matcher.go

@@ -3,24 +3,31 @@ package kafka
 import (
 	"sync"
 	"time"
-)
 
-var reqResMatcher = CreateResponseRequestMatcher() // global
-const maxTry int = 3000
+	"github.com/up9inc/mizu/tap/api"
+)
 
 type RequestResponsePair struct {
 	Request  Request
 	Response Response
 }
 
-// Key is {client_addr}:{client_port}->{dest_addr}:{dest_port}::{correlation_id}
+// Key is {client_addr}_{client_port}_{dest_addr}_{dest_port}_{correlation_id}
 type requestResponseMatcher struct {
 	openMessagesMap *sync.Map
+	maxTry          int
 }
 
-func CreateResponseRequestMatcher() requestResponseMatcher {
-	newMatcher := &requestResponseMatcher{openMessagesMap: &sync.Map{}}
-	return *newMatcher
+func createResponseRequestMatcher() api.RequestResponseMatcher {
+	return &requestResponseMatcher{openMessagesMap: &sync.Map{}, maxTry: 3000}
+}
+
+func (matcher *requestResponseMatcher) GetMap() *sync.Map {
+	return matcher.openMessagesMap
+}
+
+func (matcher *requestResponseMatcher) SetMaxTry(value int) {
+	matcher.maxTry = value
 }
 
 func (matcher *requestResponseMatcher) registerRequest(key string, request *Request) *RequestResponsePair {
@@ -40,7 +47,7 @@ func (matcher *requestResponseMatcher) registerResponse(key string, response *Re
 	try := 0
 	for {
 		try++
-		if try > maxTry {
+		if try > matcher.maxTry {
 			return nil
 		}
 		if request, found := matcher.openMessagesMap.LoadAndDelete(key); found {

tap/extensions/kafka/protocol.go

@@ -3,7 +3,6 @@ package kafka
 import (
 	"fmt"
 	"io"
-	"net"
 	"reflect"
 	"strconv"
 	"strings"
@@ -27,29 +26,20 @@ func (k ApiKey) String() string {
 	return strconv.Itoa(int(k))
 }
 
-func (k ApiKey) MinVersion() int16 { return k.apiType().minVersion() }
-
-func (k ApiKey) MaxVersion() int16 { return k.apiType().maxVersion() }
-
-func (k ApiKey) SelectVersion(minVersion, maxVersion int16) int16 {
-	min := k.MinVersion()
-	max := k.MaxVersion()
-	switch {
-	case min > maxVersion:
-		return min
-	case max < maxVersion:
-		return max
-	default:
-		return maxVersion
-	}
-}
-
-func (k ApiKey) apiType() apiType {
-	if i := int(k); i >= 0 && i < len(apiTypes) {
-		return apiTypes[i]
-	}
-	return apiType{}
-}
+const (
+	// v0  = 0
+	v1  = 1
+	v2  = 2
+	v3  = 3
+	v4  = 4
+	v5  = 5
+	v6  = 6
+	v7  = 7
+	v8  = 8
+	v9  = 9
+	v10 = 10
+	v11 = 11
+)
 
 const (
 	Produce                     ApiKey = 0
@@ -164,48 +154,6 @@ type messageType struct {
 	flexible bool
 	gotype   reflect.Type
 	decode   decodeFunc
-	encode   encodeFunc
-}
-
-type apiType struct {
-	requests  []messageType
-	responses []messageType
-}
-
-func (t apiType) minVersion() int16 {
-	if len(t.requests) == 0 {
-		return 0
-	}
-	return t.requests[0].version
-}
-
-func (t apiType) maxVersion() int16 {
-	if len(t.requests) == 0 {
-		return 0
-	}
-	return t.requests[len(t.requests)-1].version
-}
-
-var apiTypes [numApis]apiType
-
-// Register is automatically called by sub-packages are imported to install a
-// new pair of request/response message types.
-func Register(req, res Message) {
-	k1 := req.ApiKey()
-	k2 := res.ApiKey()
-
-	if k1 != k2 {
-		panic(fmt.Sprintf("[%T/%T]: request and response API keys mismatch: %d != %d", req, res, k1, k2))
-	}
-
-	apiTypes[k1] = apiType{
-		requests:  typesOf(req),
-		responses: typesOf(res),
-	}
-}
-
-func typesOf(v interface{}) []messageType {
-	return makeTypes(reflect.TypeOf(v).Elem())
 }
 
 func makeTypes(t reflect.Type) []messageType {
@@ -241,7 +189,6 @@ func makeTypes(t reflect.Type) []messageType {
 			gotype:   t,
 			flexible: flexible,
 			decode:   decodeFuncOf(t, v, flexible, structTag{}),
-			encode:   encodeFuncOf(t, v, flexible, structTag{}),
 		})
 	}
 
@@ -378,31 +325,6 @@ type Broker struct {
 	Rack string
 }
 
-func (b Broker) String() string {
-	return net.JoinHostPort(b.Host, itoa(b.Port))
-}
-
-func (b Broker) Format(w fmt.State, v rune) {
-	switch v {
-	case 'd':
-		_, _ = io.WriteString(w, itoa(b.ID))
-	case 's':
-		_, _ = io.WriteString(w, b.String())
-	case 'v':
-		_, _ = io.WriteString(w, itoa(b.ID))
-		_, _ = io.WriteString(w, " ")
-		_, _ = io.WriteString(w, b.String())
-		if b.Rack != "" {
-			_, _ = io.WriteString(w, " ")
-			_, _ = io.WriteString(w, b.Rack)
-		}
-	}
-}
-
-func itoa(i int32) string {
-	return strconv.Itoa(int(i))
-}
-
 type Topic struct {
 	Name       string
 	Error      int16
@@ -418,14 +340,6 @@ type Partition struct {
 	Offline  []int32
 }
 
-// BrokerMessage is an extension of the Message interface implemented by some
-// request types to customize the broker assignment logic.
-type BrokerMessage interface {
-	// Given a representation of the kafka cluster state as argument, returns
-	// the broker that the message should be routed to.
-	Broker(Cluster) (Broker, error)
-}
-
 // GroupMessage is an extension of the Message interface implemented by some
 // request types to inform the program that they should be routed to a group
 // coordinator.
@@ -443,16 +357,6 @@ type PreparedMessage interface {
 	Prepare(apiVersion int16)
 }
 
-// Splitter is an interface implemented by messages that can be split into
-// multiple requests and have their results merged back by a Merger.
-type Splitter interface {
-	// For a given cluster layout, returns the list of messages constructed
-	// from the receiver for each requests that should be sent to the cluster.
-	// The second return value is a Merger which can be used to merge back the
-	// results of each request into a single message (or an error).
-	Split(Cluster) ([]Message, Merger, error)
-}
-
 // Merger is an interface implemented by messages which can merge multiple
 // results into one response.
 type Merger interface {
@@ -461,16 +365,3 @@ type Merger interface {
 	// values, other types should trigger a panic.
 	Merge(messages []Message, results []interface{}) (Message, error)
 }
-
-// Result converts r to a Message or and error, or panics if r could be be
-// converted to these types.
-func Result(r interface{}) (Message, error) {
-	switch v := r.(type) {
-	case Message:
-		return v, nil
-	case error:
-		return nil, v
-	default:
-		panic(fmt.Errorf("BUG: result must be a message or an error but not %T", v))
-	}
-}