Added redis acceptance test (#750)

* Commit some fixes

* Accept expected

Co-authored-by: Igor Gov <iggvrv@gmail.com>

.github/workflows/release.yml

@@ -32,14 +32,15 @@ jobs:
         id: condval
         with:
           cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "minor"
-          if_false: "patch"
+          if_true: "stable"
+          if_false: "dev"
 
-      - name: Auto increment SemVer action
-        uses: MCKanpolat/auto-semver-action@1.0.5
+      - name: Auto Increment Ver Action
+        uses: docker://igorgov/auto-inc-ver:v2.0.0
         id: versioning
         with:
-          releaseType: ${{ steps.condval.outputs.value }}
+          mode: ${{ steps.condval.outputs.value }}
+          suffix: 'dev'
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Get version parameters
@@ -77,7 +78,7 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             TARGETARCH=${{ matrix.target }}
-            SEM_VER=${{ steps.versioning.outputs.version }}
+            VER=${{ steps.versioning.outputs.version }}
             BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
             GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
             COMMIT_HASH=${{ github.sha }}
@@ -98,25 +99,28 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v2
 
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@master
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
         with:
-          service_account_key: ${{ secrets.GCR_JSON_KEY }}
-          export_default_credentials: true
+          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v0'
 
       - name: Determine versioning strategy
         uses: haya14busa/action-cond@v1
         id: condval
         with:
           cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "minor"
-          if_false: "patch"
+          if_true: "stable"
+          if_false: "dev"
 
-      - name: Auto increment SemVer action
-        uses: MCKanpolat/auto-semver-action@1.0.5
+      - name: Auto Increment Ver Action
+        uses: docker://igorgov/auto-inc-ver:v2.0.0
         id: versioning
         with:
-          releaseType: ${{ steps.condval.outputs.value }}
+          mode: ${{ steps.condval.outputs.value }}
+          suffix: 'dev'
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Get version parameters
@@ -160,7 +164,7 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             TARGETARCH=${{ matrix.target }}
-            SEM_VER=${{ steps.versioning.outputs.version }}
+            VER=${{ steps.versioning.outputs.version }}
             BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
             GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
             COMMIT_HASH=${{ github.sha }}
@@ -175,14 +179,15 @@ jobs:
         id: condval
         with:
           cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "minor"
-          if_false: "patch"
+          if_true: "stable"
+          if_false: "dev"
 
-      - name: Auto increment SemVer action
-        uses: MCKanpolat/auto-semver-action@1.0.5
+      - name: Auto Increment Ver Action
+        uses: docker://igorgov/auto-inc-ver:v2.0.0
         id: versioning
         with:
-          releaseType: ${{ steps.condval.outputs.value }}
+          mode: ${{ steps.condval.outputs.value }}
+          suffix: 'dev'
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Get version parameters
@@ -232,24 +237,28 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v2
 
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@master
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
         with:
-          service_account_key: ${{ secrets.GCR_JSON_KEY }}
-          export_default_credentials: true
+          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
 
-      - uses: haya14busa/action-cond@v1
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v0'
+
+      - name: Determine versioning strategy
+        uses: haya14busa/action-cond@v1
         id: condval
         with:
           cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "minor"
-          if_false: "patch"
+          if_true: "stable"
+          if_false: "dev"
 
-      - name: Auto Increment Semver Action
-        uses: MCKanpolat/auto-semver-action@1.0.5
+      - name: Auto Increment Ver Action
+        uses: docker://igorgov/auto-inc-ver:v2.0.0
         id: versioning
         with:
-          releaseType: ${{ steps.condval.outputs.value }}
+          mode: ${{ steps.condval.outputs.value }}
+          suffix: 'dev'
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Get version parameters
@@ -260,7 +269,7 @@ jobs:
         id: version_parameters
 
       - name: Build and Push CLI
-        run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
+        run: make push-cli VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
 
       - name: Log the version into a .txt file
         shell: bash

.github/workflows/test.yml

@@ -5,32 +5,18 @@ on:
     branches:
       - 'develop'
       - 'main'
+  push: # needed to upload test coverage report to codecov
+    branches:
+      - 'develop'
+      - 'main'
 
 concurrency:
   group: mizu-tests-validation-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  run-tests-cli:
-    name: CLI Tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
-        with:
-          go-version: '^1.16'
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Test
-        run: make test-cli
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2
-
-  run-tests-agent:
-    name: Agent Tests
+  run-unit-tests:
+    name: Unit Tests
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.16
@@ -45,8 +31,15 @@ jobs:
         run: |
           sudo apt-get install libpcap-dev
 
-      - name: Test
+      - name: CLI Test
+        run: make test-cli
+
+      - name: Agent Test
         run: make test-agent
+      
+      - name: Shared Test
+        run: make test-shared
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
+

Dockerfile

@@ -67,7 +67,7 @@ COPY agent .
 ARG COMMIT_HASH
 ARG GIT_BRANCH
 ARG BUILD_TIMESTAMP
-ARG SEM_VER=0.0.0
+ARG VER=0.0
 
 WORKDIR /app/agent-build
 
@@ -75,7 +75,7 @@ RUN go build -ldflags="-extldflags=-static -s -w \
     -X 'github.com/up9inc/mizu/agent/pkg/version.GitCommitHash=${COMMIT_HASH}' \
     -X 'github.com/up9inc/mizu/agent/pkg/version.Branch=${GIT_BRANCH}' \
     -X 'github.com/up9inc/mizu/agent/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
-    -X 'github.com/up9inc/mizu/agent/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
+    -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum
 ADD https://github.com/up9inc/basenine/releases/download/v0.4.13/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}

Makefile

@@ -19,7 +19,7 @@ help: ## This help.
 TS_SUFFIX="$(shell date '+%s')"
 GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
 BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
-export SEM_VER?=0.0.0
+export VER?=0.0
 
 ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
@@ -89,5 +89,8 @@ test-cli:
 test-agent:
 	@echo "running agent tests"; cd agent && $(MAKE) test
 
+test-shared:
+	@echo "running shared tests"; cd shared && $(MAKE) test
+
 acceptance-test:
 	@echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test

README.md

@@ -1,11 +1,17 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
 
 <p align="center">
+    <a href="https://github.com/up9inc/mizu/blob/main/LICENSE">
+        <img alt="GitHub License" src="https://img.shields.io/github/license/up9inc/mizu?logo=GitHub&style=flat-square">
+    </a>
     <a href="https://github.com/up9inc/mizu/releases/latest">
         <img alt="GitHub Latest Release" src="https://img.shields.io/github/v/release/up9inc/mizu?logo=GitHub&style=flat-square">
     </a>
-    <a href="https://github.com/up9inc/mizu/blob/main/LICENSE">
-        <img alt="GitHub License" src="https://img.shields.io/github/license/up9inc/mizu?logo=GitHub&style=flat-square">
+    <a href="https://hub.docker.com/r/up9inc/mizu">
+      <img alt="Docker pulls" src="https://img.shields.io/docker/pulls/up9inc/mizu?color=%23099cec">
+    </a>
+    <a href="https://hub.docker.com/r/up9inc/mizu">
+      <img alt="Image size" src="https://img.shields.io/docker/image-size/up9inc/mizu/latest">
     </a>
     <a href="https://join.slack.com/t/up9/shared_invite/zt-tfjnduli-QzlR8VV4Z1w3YnPIAJfhlQ">
       <img alt="Slack" src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social">

acceptanceTests/cypress.json

@@ -13,13 +13,18 @@
     "tests/Regex.js",
     "tests/RegexMasking.js",
     "tests/IgnoredUserAgents.js",
-    "tests/UiTest.js"
+    "tests/UiTest.js",
+    "tests/Redis.js"
   ],
 
   "env": {
     "testUrl": "http://localhost:8899/",
     "redactHeaderContent": "User-Header[REDACTED]",
     "redactBodyContent": "{ \"User\": \"[REDACTED]\" }",
-    "regexMaskingBodyContent": "[REDACTED]"
+    "regexMaskingBodyContent": "[REDACTED]",
+    "minimumEntries": 25,
+    "greenFilterColor": "rgb(210, 250, 210)",
+    "redFilterColor": "rgb(250, 214, 220)",
+    "bodyJsonClass": ".hljs"
   }
 }

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -16,3 +16,40 @@ export function resizeToNormalMizu() {
     cy.viewport(1920, 1080);
 }
 
+export function verifyMinimumEntries() {
+    const minimumEntries = Cypress.env('minimumEntries');
+    it(`Making sure that mizu shows at least ${minimumEntries} entries`, async function () {
+        cy.get('#total-entries').then(number => {
+            const getNum = () => {
+                const numOfEntries = number.text();
+                return parseInt(numOfEntries);
+            };
+            cy.wrap({there: getNum}).invoke('there').should('be.gte', minimumEntries);
+        });
+    });
+}
+
+export function leftTextCheck(entryNum, path, expectedText) {
+    cy.get(`#list #entry-${entryNum} ${path}`).invoke('text').should('eq', expectedText);
+}
+
+export function leftOnHoverCheck(entryNum, path, filterName) {
+    cy.get(`#list #entry-${entryNum} ${path}`).trigger('mouseover');
+    cy.get(`#list #entry-${entryNum} .Queryable-Tooltip`).invoke('text').should('match', new RegExp(filterName));
+}
+
+export function rightTextCheck(path, expectedText) {
+    cy.get(`#rightSideContainer ${path}`).should('have.text', expectedText);
+}
+
+export function rightOnHoverCheck(path, expectedText) {
+    cy.get(`#rightSideContainer ${path}`).trigger('mouseover');
+    cy.get(`#rightSideContainer .Queryable-Tooltip`).invoke('text').should('match', new RegExp(expectedText));
+}
+
+export function checkThatAllEntriesShown() {
+    cy.get('#entries-length').then(number => {
+        if (number.text() === '1')
+            cy.get('[title="Fetch old records"]').click();
+    });
+}

acceptanceTests/cypress/integration/tests/GuiPort.js

@@ -1,8 +1,16 @@
-it('check', function () {
-    cy.visit(`http://localhost:${Cypress.env('port')}/`);
+import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
+import {verifyMinimumEntries} from "../testHelpers/TrafficHelper";
 
-    cy.get('.header').should('be.visible');
-    cy.get('.TrafficPageHeader').should('be.visible');
-    cy.get('.TrafficPage-ListContainer').should('be.visible');
-    cy.get('.TrafficPage-Container').should('be.visible');
+it('check', function () {
+    const podName = Cypress.env('name'), namespace = Cypress.env('namespace');
+    const port = Cypress.env('port');
+    cy.intercept('GET', `http://localhost:${port}/status/tap`).as('statusTap');
+
+    cy.visit(`http://localhost:${port}`);
+    cy.wait('@statusTap').its('response.statusCode').should('match', /^2\d{2}/);
+
+    verifyMinimumEntries();
+
+    cy.get('.podsCount').trigger('mouseover');
+    findLineAndCheck(getExpectedDetailsDict(podName, namespace));
 });

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -1,29 +1,27 @@
-import {isValueExistsInElement, resizeToHugeMizu} from "../testHelpers/TrafficHelper";
+import {
+    checkThatAllEntriesShown,
+    isValueExistsInElement,
+    resizeToHugeMizu,
+    verifyMinimumEntries
+} from "../testHelpers/TrafficHelper";
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
 });
 
-it('going through each entry', function () {
-    resizeToHugeMizu();
-    cy.get('#total-entries').then(number => {
-        const getNum = () => {
-            const numOfEntries = number.text();
-            return parseInt(numOfEntries);
-        };
-        cy.wrap({ there: getNum }).invoke('there').should('be.gte', 25);
+verifyMinimumEntries();
 
+checkEntries();
+
+function checkEntries() {
+    it('checking all entries', function () {
         checkThatAllEntriesShown();
+        resizeToHugeMizu();
 
-        const entriesNum = getNum();
-        [...Array(entriesNum).keys()].map(checkEntry);
-    });
-});
-
-function checkThatAllEntriesShown() {
-    cy.get('#entries-length').then(number => {
-        if (number.text() === '1')
-            cy.get('[title="Fetch old records"]').click();
+        cy.get('#total-entries').then(number => {
+            const numOfEntries = parseInt(number.text());
+            [...Array(numOfEntries).keys()].map(checkEntry);
+        });
     });
 }
 

acceptanceTests/cypress/integration/tests/NoRedact.js

@@ -1,8 +1,10 @@
-import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
+import {isValueExistsInElement, verifyMinimumEntries} from '../testHelpers/TrafficHelper';
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
-})
+});
+
+verifyMinimumEntries();
 
 isValueExistsInElement(false, Cypress.env('redactHeaderContent'), '#tbody-Headers');
-isValueExistsInElement(false, Cypress.env('redactBodyContent'), '.hljs');
+isValueExistsInElement(false, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/integration/tests/Redact.js

@@ -1,8 +1,10 @@
-import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
+import {isValueExistsInElement, verifyMinimumEntries} from '../testHelpers/TrafficHelper';
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
-})
+});
+
+verifyMinimumEntries();
 
 isValueExistsInElement(true, Cypress.env('redactHeaderContent'), '#tbody-Headers');
-isValueExistsInElement(true, Cypress.env('redactBodyContent'), '.hljs');
+isValueExistsInElement(true, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/integration/tests/Redis.js

@@ -0,0 +1,155 @@
+import {
+    leftOnHoverCheck,
+    leftTextCheck,
+    rightOnHoverCheck,
+    rightTextCheck,
+} from "../testHelpers/TrafficHelper";
+
+const valueTabs = {
+    response: 'RESPONSE',
+    request: 'REQUEST',
+    none: null
+}
+
+it('opening mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+checkRedisFilterByMethod({
+    method: 'PING',
+    shouldCheckSummary: false,
+    valueTab: valueTabs.none
+});
+
+checkRedisFilterByMethod({
+    method: 'SET',
+    shouldCheckSummary: true,
+    valueTab: valueTabs.request,
+    valueRegex: /^\[value, keepttl]$/mg
+});
+
+checkRedisFilterByMethod({
+    method: 'EXISTS',
+    shouldCheckSummary: true,
+    valueTab: valueTabs.response,
+    valueRegex: /^1$/mg
+});
+
+checkRedisFilterByMethod({
+    method: 'GET',
+    shouldCheckSummary: true,
+    valueTab: valueTabs.response,
+    valueRegex: /^value$/mg
+});
+
+checkRedisFilterByMethod({
+    method: 'DEL',
+    shouldCheckSummary: true,
+    valueTab: valueTabs.response,
+    valueRegex: /^1$|^0$/mg
+});
+
+function checkRedisFilterByMethod(funcDict) {
+    const {method, shouldCheckSummary} = funcDict
+    const summaryDict = getSummeryDict();
+    const methodDict = getMethodDict(method);
+    const protocolDict = getProtocolDict();
+
+    it(`Testing the method: ${method}`, function () {
+        // applying filter
+        cy.get('.w-tc-editor-text').clear().type(`method == "${method}"`);
+        cy.get('[type="submit"]').click();
+        cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
+
+        cy.get('#entries-length').then(number => {
+            // if the entries list isn't expanded it expands here
+            if (number.text() === '0' || number.text() === '1') // todo change when TRA-4262 is fixed
+                cy.get('[title="Fetch old records"]').click();
+
+            cy.get('#entries-length').should('not.have.text', '0').and('not.have.text', '1').then(() => {
+                cy.get(`#list [id]`).then(elements => {
+                   const listElmWithIdAttr = Object.values(elements);
+                   let doneCheckOnFirst = false;
+
+                   listElmWithIdAttr.forEach(entry => {
+                       if (entry?.id && entry.id.match(RegExp(/entry-(\d{2}|\d{1})$/gm))) {
+                           const entryNum = getEntryNumById(entry.id);
+
+                           leftTextCheck(entryNum, methodDict.pathLeft, methodDict.expectedText);
+                           leftTextCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedTextLeft);
+                           if (shouldCheckSummary)
+                               leftTextCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedText);
+
+                           if (!doneCheckOnFirst) {
+                               deepCheck(funcDict, protocolDict, methodDict, summaryDict, entry);
+                               doneCheckOnFirst = true;
+                           }
+                       }
+                   });
+                });
+            });
+        });
+    });
+}
+
+function deepCheck(generalDict, protocolDict, methodDict, summaryDict, entry) {
+    const entryNum = getEntryNumById(entry.id);
+    const {shouldCheckSummary, valueTab, valueRegex} = generalDict;
+
+    leftOnHoverCheck(entryNum, methodDict.pathLeft, methodDict.expectedOnHover);
+    leftOnHoverCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedOnHover);
+    if (shouldCheckSummary)
+        leftOnHoverCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedOnHover);
+
+    cy.get(`#${entry.id}`).click();
+
+    rightTextCheck(methodDict.pathRight, methodDict.expectedText);
+    rightTextCheck(protocolDict.pathRight, protocolDict.expectedTextRight);
+    if (shouldCheckSummary)
+        rightTextCheck(summaryDict.pathRight, summaryDict.expectedText);
+
+    rightOnHoverCheck(methodDict.pathRight, methodDict.expectedOnHover);
+    rightOnHoverCheck(protocolDict.pathRight, protocolDict.expectedOnHover);
+    if (shouldCheckSummary)
+        rightOnHoverCheck(summaryDict.pathRight, summaryDict.expectedOnHover);
+
+    if (valueTab) {
+        if (valueTab === valueTabs.response)
+            cy.contains('Response').click();
+        cy.get(Cypress.env('bodyJsonClass')).then(text => {
+            expect(text.text()).to.match(valueRegex)
+        });
+    }
+}
+
+function getSummeryDict() {
+    return {
+        pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+        pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+        expectedText: 'key',
+        expectedOnHover: `summary == "key"`
+    };
+}
+
+function getMethodDict(method) {
+    return {
+        pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+        pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+        expectedText: method,
+        expectedOnHover: `method == "${method}"`
+    };
+}
+
+function getProtocolDict() {
+    return {
+        pathLeft: '> :nth-child(1) > :nth-child(1)',
+        pathRight: '> :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(1)',
+        expectedTextLeft: 'REDIS',
+        expectedTextRight: 'Redis Serialization Protocol',
+        expectedOnHover: `redis`
+    };
+}
+
+function getEntryNumById (id) {
+    return parseInt(id.split('-')[1]);
+}

acceptanceTests/cypress/integration/tests/RegexMasking.js

@@ -1,7 +1,9 @@
-import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
+import {isValueExistsInElement, verifyMinimumEntries} from "../testHelpers/TrafficHelper";
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
-})
+});
 
-isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), '.hljs');
+verifyMinimumEntries();
+
+isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -1,14 +1,21 @@
 import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
-import {resizeToHugeMizu, resizeToNormalMizu} from "../testHelpers/TrafficHelper";
-const greenFilterColor = 'rgb(210, 250, 210)';
-const redFilterColor = 'rgb(250, 214, 220)';
+import {
+    leftTextCheck,
+    resizeToHugeMizu,
+    resizeToNormalMizu,
+    rightOnHoverCheck,
+    leftOnHoverCheck,
+    rightTextCheck,
+    verifyMinimumEntries
+} from "../testHelpers/TrafficHelper";
 const refreshWaitTimeout = 10000;
-const bodyJsonClass = '.hljs';
 
 it('opening mizu', function () {
     cy.visit(Cypress.env('testUrl'));
 });
 
+verifyMinimumEntries();
+
 it('top bar check', function () {
     const podName1 = 'httpbin', namespace1 = 'mizu-tests';
     const podName2 = 'httpbin2', namespace2 = 'mizu-tests';
@@ -131,7 +138,7 @@ function checkFilterNoResults(filterName) {
 
             // applying the filter
             cy.get('.w-tc-editor-text').type(filterName);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', greenFilterColor);
+            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
             cy.get('[type="submit"]').click();
 
             // waiting for the entries number to load
@@ -164,7 +171,7 @@ function checkIllegalFilter(illegalFilterName) {
             const totalEntries = number.text();
 
             cy.get('.w-tc-editor-text').type(illegalFilterName);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', redFilterColor);
+            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('redFilterColor'));
             cy.get('[type="submit"]').click();
 
             cy.get('[role="alert"]').should('be.visible');
@@ -189,7 +196,7 @@ function checkFilter(filterDetails){
 
             // applying the filter with alt+enter or with the button
             cy.get('.w-tc-editor-text').type(`${name}${applyByEnter ? '{alt+enter}' : ''}`);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', greenFilterColor);
+            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
             if (!applyByEnter)
                 cy.get('[type="submit"]').click();
 
@@ -229,37 +236,15 @@ function deeperChcek(leftSidePath, rightSidePath, filterName, leftSideExpectedTe
         cy.get(`#list #entry-${entryNum}`).click();
         rightTextCheck(rightSidePath, rightSideExpectedText);
         rightOnHoverCheck(rightSidePath, filterName);
-        checkRightSideResponseBody();
     });
 }
 
-function leftTextCheck(entryNum, path, expectedText) {
-    cy.get(`#list #entry-${entryNum} ${path}`).invoke('text').should('eq', expectedText);
-}
-
-function leftOnHoverCheck(entryNum, path, filterName) {
-    cy.get(`#list #entry-${entryNum} ${path}`).trigger('mouseover');
-    cy.get(`#list #entry-${entryNum} .Queryable-Tooltip`).should('have.text', filterName);
-}
-
-function rightTextCheck(path, expectedText) {
-    cy.get(`.TrafficPage-Container > :nth-child(2) ${path}`).should('have.text', expectedText);
-}
-
-function rightOnHoverCheck(path, expectedText) {
-    cy.get(`.TrafficPage-Container > :nth-child(2) ${path}`).trigger('mouseover');
-    cy.get(`.TrafficPage-Container > :nth-child(2) .Queryable-Tooltip`).should('have.text', expectedText);
-}
-
-
 function checkRightSideResponseBody() {
     cy.contains('Response').click();
     clickCheckbox('Decode Base64');
 
-    cy.get(`${bodyJsonClass}`).then(value => {
+    cy.get(`${Cypress.env('bodyJsonClass')}`).then(value => {
         const encodedBody = value.text();
-        cy.log(encodedBody);
-
         const decodedBody = atob(encodedBody);
         const responseBody = JSON.parse(decodedBody);
 
@@ -279,11 +264,11 @@ function checkRightSideResponseBody() {
         expect(responseBody.headers['Accept-Encoding']).to.match(expectdJsonBody.headers['Accept-Encoding']);
         expect(responseBody.headers['X-Forwarded-Uri']).to.match(expectdJsonBody.headers['X-Forwarded-Uri']);
 
-        cy.get(`${bodyJsonClass}`).should('have.text', encodedBody);
+        cy.get(`${Cypress.env('bodyJsonClass')}`).should('have.text', encodedBody);
         clickCheckbox('Decode Base64');
 
-        cy.get(`${bodyJsonClass} > `).its('length').should('be.gt', 1).then(linesNum => {
-            cy.get(`${bodyJsonClass} > >`).its('length').should('be.gt', linesNum).then(jsonItemsNum => {
+        cy.get(`${Cypress.env('bodyJsonClass')} > `).its('length').should('be.gt', 1).then(linesNum => {
+            cy.get(`${Cypress.env('bodyJsonClass')} > >`).its('length').should('be.gt', linesNum).then(jsonItemsNum => {
                 checkPrettyAndLineNums(jsonItemsNum, decodedBody);
 
                 clickCheckbox('Line numbers');
@@ -305,7 +290,7 @@ function clickCheckbox(type) {
 
 function checkPrettyAndLineNums(jsonItemsLen, decodedBody) {
     decodedBody = decodedBody.replaceAll(' ', '');
-    cy.get(`${bodyJsonClass} >`).then(elements => {
+    cy.get(`${Cypress.env('bodyJsonClass')} >`).then(elements => {
         const lines = Object.values(elements);
         lines.forEach((line, index) => {
             if (line.getAttribute) {
@@ -325,13 +310,13 @@ function getCleanLine(lineElement) {
 }
 
 function checkPrettyOrNothing(jsonItems, decodedBody) {
-    cy.get(`${bodyJsonClass} > `).should('have.length', jsonItems).then(text => {
+    cy.get(`${Cypress.env('bodyJsonClass')} > `).should('have.length', jsonItems).then(text => {
         const json = text.text();
         expect(json).to.equal(decodedBody);
     });
 }
 
 function checkOnlyLineNumberes(jsonItems, decodedText) {
-    cy.get(`${bodyJsonClass} >`).should('have.length', 1).and('have.text', decodedText);
-    cy.get(`${bodyJsonClass} > >`).should('have.length', jsonItems)
+    cy.get(`${Cypress.env('bodyJsonClass')} >`).should('have.length', 1).and('have.text', decodedText);
+    cy.get(`${Cypress.env('bodyJsonClass')} > >`).should('have.length', jsonItems)
 }

acceptanceTests/extensions_test.go

@@ -0,0 +1,101 @@
+package acceptanceTests
+
+import (
+	"context"
+	"fmt"
+	"github.com/go-redis/redis/v8"
+	"os/exec"
+	"testing"
+)
+
+func TestRedis(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := cleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	ctx := context.Background()
+
+	redisExternalIp, err := getServiceExternalIp(ctx, defaultNamespaceName, "redis")
+	if err != nil {
+		t.Errorf("failed to get redis external ip, err: %v", err)
+		return
+	}
+
+	rdb := redis.NewClient(&redis.Options{
+		Addr: fmt.Sprintf("%v:6379", redisExternalIp),
+	})
+
+	for i := 0; i < defaultEntriesCount/5; i++ {
+		requestErr := rdb.Ping(ctx).Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < defaultEntriesCount/5; i++ {
+		requestErr := rdb.Set(ctx, "key", "value", -1).Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < defaultEntriesCount/5; i++ {
+		requestErr := rdb.Exists(ctx, "key").Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < defaultEntriesCount/5; i++ {
+		requestErr := rdb.Get(ctx, "key").Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < defaultEntriesCount/5; i++ {
+		requestErr := rdb.Del(ctx, "key").Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/Redis.js\"")
+}

acceptanceTests/go.mod

@@ -3,9 +3,11 @@ module github.com/up9inc/mizu/tests
 go 1.16
 
 require (
-	github.com/gorilla/websocket v1.4.2
+	github.com/go-redis/redis/v8 v8.11.4
 	github.com/up9inc/mizu/shared v0.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	k8s.io/apimachinery v0.21.2
+	k8s.io/client-go v0.21.2
 )
 
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared

acceptanceTests/go.sum

@@ -57,8 +57,11 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -79,6 +82,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -96,6 +101,7 @@ github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwo
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -154,7 +160,10 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
 github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
 github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
+github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg=
+github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -185,6 +194,9 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -194,8 +206,10 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -212,10 +226,10 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -243,6 +257,7 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -304,13 +319,23 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c=
+github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
@@ -375,6 +400,7 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
@@ -470,14 +496,17 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -504,10 +533,12 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -522,21 +553,27 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -577,6 +614,7 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -598,6 +636,7 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -635,6 +674,9 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -647,6 +689,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -655,6 +698,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -673,6 +717,7 @@ k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU=
 k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc=
 k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM=
 k8s.io/cli-runtime v0.21.2/go.mod h1:8u/jFcM0QpoI28f6sfrAAIslLCXUYKD5SsPPMWiHYrI=
+k8s.io/client-go v0.21.2 h1:Q1j4L/iMN4pTw6Y4DWppBoUxgKO8LbffEMVEV00MUp0=
 k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA=
 k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U=
 k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc=
@@ -686,6 +731,7 @@ k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
 k8s.io/kubectl v0.21.2/go.mod h1:PgeUclpG8VVmmQIl8zpLar3IQEpFc9mrmvlwY3CK1xo=
 k8s.io/metrics v0.21.2/go.mod h1:wzlOINZMCtWq8dR9gHlyaOemmYlOpAoldEIXE82gAhI=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

acceptanceTests/setup.sh

@@ -36,15 +36,24 @@ kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests
 
 kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2
 
+echo "Creating redis deployment"
+kubectl create deployment redis --image=redis -n mizu-tests
+
 echo "Creating httpbin services"
 kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
 kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests
 
 kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2
 
+echo "Creating redis service"
+kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests
+
 echo "Starting proxy"
 kubectl proxy --port=8080 &
 
+echo "Starting tunnel"
+minikube tunnel &
+
 echo "Setting minikube docker env"
 eval $(minikube docker-env)
 

acceptanceTests/tap_test.go

@@ -110,7 +110,16 @@ func TestTapGuiPort(t *testing.T) {
 				return
 			}
 
-			runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env port=%d", guiPort))
+			proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+			for i := 0; i < defaultEntriesCount; i++ {
+				if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+					t.Errorf("failed to send proxy request, err: %v", requestErr)
+					return
+				}
+			}
+
+			runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env name=%v,namespace=%v,port=%d",
+				"httpbin", "mizu-tests", guiPort))
 		})
 	}
 }

acceptanceTests/testsUtils.go

@@ -2,13 +2,19 @@ package acceptanceTests
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/homedir"
 	"net/http"
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strings"
 	"syscall"
 	"testing"
@@ -68,10 +74,39 @@ func getApiServerUrl(port uint16) string {
 	return fmt.Sprintf("http://localhost:%v", port)
 }
 
+func getServiceExternalIp(ctx context.Context, namespace string, service string) (string, error) {
+	home := homedir.HomeDir()
+	configLoadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(home, ".kube", "config")}
+	clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		configLoadingRules,
+		&clientcmd.ConfigOverrides{
+			CurrentContext: "",
+		},
+	)
+
+	restClientConfig, err := clientConfig.ClientConfig()
+	if err != nil {
+		return "", err
+	}
+
+	clientSet, err := kubernetes.NewForConfig(restClientConfig)
+	if err != nil {
+		return "", err
+	}
+
+	serviceObj, err := clientSet.CoreV1().Services(namespace).Get(ctx, service, metav1.GetOptions{})
+	if err != nil {
+		return "", err
+	}
+
+	externalIp := serviceObj.Status.LoadBalancer.Ingress[0].IP
+	return externalIp, nil
+}
+
 func getDefaultCommandArgs() []string {
 	setFlag := "--set"
 	telemetry := "telemetry=false"
-	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
+	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0"
 	imagePullPolicy := "image-pull-policy=IfNotPresent"
 	headless := "headless=true"
 

agent/go.mod

@@ -4,7 +4,7 @@ go 1.16
 
 require (
 	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
-	github.com/chanced/openapi v0.0.6
+	github.com/chanced/openapi v0.0.7
 	github.com/djherbis/atime v1.0.0
 	github.com/elastic/go-elasticsearch/v7 v7.16.0
 	github.com/getkin/kin-openapi v0.76.0
@@ -30,6 +30,7 @@ require (
 	github.com/up9inc/mizu/tap/extensions/http v0.0.0
 	github.com/up9inc/mizu/tap/extensions/kafka v0.0.0
 	github.com/up9inc/mizu/tap/extensions/redis v0.0.0
+	github.com/wI2L/jsondiff v0.1.1
 	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
 	k8s.io/api v0.21.2
 	k8s.io/apimachinery v0.21.2

agent/go.sum

@@ -95,8 +95,8 @@ github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a h1:zG6t+4krPXcCKtL
 github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a/go.mod h1:yhcmlFk1hxuZ+5XZbupzT/cEm/eE4ZvWbmsW1+Q/aZE=
 github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44 h1:4NOJMtvZaOA6cI2gkIuXk/2b5KTOvm/R4zyPy/yLCM4=
 github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44/go.mod h1:XVNfXN5kgZST4PQ0W/oBAHJku2OteCeHxjAbvfd0ARM=
-github.com/chanced/openapi v0.0.6 h1:2giGS47+T8/7MN2hfRHSIUPx86Qksk+J/ciIWcAM7hY=
-github.com/chanced/openapi v0.0.6/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU=
+github.com/chanced/openapi v0.0.7 h1:OmOBHCg/5ViUg0gaGxXBeEFoVBE8C2pHK4BO/AiD6k8=
+github.com/chanced/openapi v0.0.7/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=

agent/keto/build-push-featurebranch.sh

@@ -7,9 +7,9 @@ SERVER_NAME=mizu-keto
 GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
 
 DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
-SEM_VER=${SEM_VER=0.0.0}
+VER=${VER=0.0}
 
-DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
+DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$VER")
 
 if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
 then
@@ -19,7 +19,7 @@ fi
 
 echo "building ${DOCKER_TAGGED_BUILDS[@]}"
 DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
-docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
+docker build $DOCKER_TAGS_ARGS --build-arg VER=${VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
 
 for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
 do

agent/kratos/build-push-featurebranch.sh

@@ -7,9 +7,9 @@ SERVER_NAME=mizu-kratos
 GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
 
 DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
-SEM_VER=${SEM_VER=0.0.0}
+VER=${VER=0.0.0}
 
-DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
+DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$VER")
 
 if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
 then
@@ -19,7 +19,7 @@ fi
 
 echo "building ${DOCKER_TAGGED_BUILDS[@]}"
 DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
-docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
+docker build $DOCKER_TAGS_ARGS --build-arg VER=${VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
 
 for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
 do

agent/pkg/api/main.go

@@ -140,7 +140,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 				mizuEntry.Rules = rules
 			}
 
-			entryWSource := oas.EntryWithSource{Entry: *harEntry, Source: mizuEntry.Source.Name}
+			entryWSource := oas.EntryWithSource{Entry: *harEntry, Source: mizuEntry.Source.Name, Id: mizuEntry.Id}
 			oas.GetOasGeneratorInstance().PushEntry(&entryWSource)
 		}
 

agent/pkg/controllers/metadata_controller.go

@@ -9,6 +9,6 @@ import (
 )
 
 func GetVersion(c *gin.Context) {
-	resp := shared.VersionResponse{SemVer: version.SemVer}
+	resp := shared.VersionResponse{Ver: version.Ver}
 	c.JSON(http.StatusOK, resp)
 }

agent/pkg/oas/counters.go

@@ -1,6 +1,11 @@
 package oas
 
-import "math"
+import (
+	"fmt"
+	"github.com/chanced/openapi"
+	"math"
+	"strings"
+)
 
 type Counter struct {
 	Entries     int     `json:"entries"`
@@ -12,6 +17,10 @@ type Counter struct {
 }
 
 func (c *Counter) addEntry(ts float64, rt float64, succ bool, dur float64) {
+	if dur < 0 {
+		panic("Duration cannot be negative")
+	}
+
 	c.Entries += 1
 	c.SumRT += rt
 	c.SumDuration += dur
@@ -55,3 +64,56 @@ func (m *CounterMap) addOther(other *CounterMap) {
 		}
 	}
 }
+
+func setCounterMsgIfOk(oldStr string, cnt *Counter) string {
+	tpl := "Mizu observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds"
+	if oldStr == "" || (strings.HasPrefix(oldStr, "Mizu ") && strings.HasSuffix(oldStr, " seconds")) {
+		return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries))
+	}
+	return oldStr
+}
+
+type CounterMaps struct {
+	counterTotal    Counter
+	counterMapTotal CounterMap
+}
+
+func (m *CounterMaps) processOp(opObj *openapi.Operation) error {
+	if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
+		counter := new(Counter)
+		err := opObj.Extensions.DecodeExtension(CountersTotal, counter)
+		if err != nil {
+			return err
+		}
+		m.counterTotal.addOther(counter)
+
+		opObj.Description = setCounterMsgIfOk(opObj.Description, counter)
+	}
+
+	if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
+		counterMap := new(CounterMap)
+		err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap)
+		if err != nil {
+			return err
+		}
+		m.counterMapTotal.addOther(counterMap)
+	}
+	return nil
+}
+
+func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error {
+	if oas.Extensions == nil {
+		oas.Extensions = openapi.Extensions{}
+	}
+
+	err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal)
+	if err != nil {
+		return err
+	}
+
+	err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal)
+	if err != nil {
+		return nil
+	}
+	return nil
+}

agent/pkg/oas/feeder_test.go

@@ -21,20 +21,33 @@ func getFiles(baseDir string) (result []string, err error) {
 	result = make([]string, 0)
 	logger.Log.Infof("Reading files from tree: %s", baseDir)
 
+	inputs := []string{baseDir}
+
 	// https://yourbasic.org/golang/list-files-in-directory/
-	err = filepath.Walk(baseDir,
-		func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-
-			ext := strings.ToLower(filepath.Ext(path))
-			if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
-				result = append(result, path)
-			}
+	visitor := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
 
+		if info.Mode()&os.ModeSymlink != 0 {
+			path, _ = os.Readlink(path)
+			inputs = append(inputs, path)
 			return nil
-		})
+		}
+
+		ext := strings.ToLower(filepath.Ext(path))
+		if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
+			result = append(result, path)
+		}
+
+		return nil
+	}
+
+	for len(inputs) > 0 {
+		path := inputs[0]
+		inputs = inputs[1:]
+		err = filepath.Walk(path, visitor)
+	}
 
 	sort.SliceStable(result, func(i, j int) bool {
 		return fileSize(result[i]) < fileSize(result[j])
@@ -110,13 +123,14 @@ func feedFromHAR(file string, isSync bool) (int, error) {
 	cnt := 0
 	for _, entry := range harDoc.Log.Entries {
 		cnt += 1
-		feedEntry(&entry, "", isSync)
+		feedEntry(&entry, "", isSync, file)
 	}
 
 	return cnt, nil
 }
 
-func feedEntry(entry *har.Entry, source string, isSync bool) {
+func feedEntry(entry *har.Entry, source string, isSync bool, file string) {
+	entry.Comment = file
 	if entry.Response.Status == 302 {
 		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
 	}
@@ -125,7 +139,7 @@ func feedEntry(entry *har.Entry, source string, isSync bool) {
 		logger.Log.Debugf("Interesting: %s", entry.Request.URL)
 	}
 
-	ews := EntryWithSource{Entry: *entry, Source: source}
+	ews := EntryWithSource{Entry: *entry, Source: source, Id: uint(0)}
 	if isSync {
 		GetOasGeneratorInstance().entriesChan <- ews // blocking variant, right?
 	} else {
@@ -176,7 +190,7 @@ func feedFromLDJSON(file string, isSync bool) (int, error) {
 				logger.Log.Warningf("Failed decoding entry: %s", line)
 			} else {
 				cnt += 1
-				feedEntry(&entry, source, isSync)
+				feedEntry(&entry, source, isSync, file)
 			}
 		}
 	}

agent/pkg/oas/gibberish.go

@@ -1,22 +1,26 @@
 package oas
 
 import (
+	"math"
 	"regexp"
 	"strings"
 	"unicode"
 )
 
 var (
-	patBase64   = regexp.MustCompile(`^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$`)
 	patUuid4    = regexp.MustCompile(`(?i)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
 	patEmail    = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
-	patHexLower = regexp.MustCompile(`(0x)?[0-9a-f]{6,}`)
-	patHexUpper = regexp.MustCompile(`(0x)?[0-9A-F]{6,}`)
-	patLongNum  = regexp.MustCompile(`\d{6,}`)
+	patLongNum  = regexp.MustCompile(`^\d{3,}$`)
+	patLongNumB = regexp.MustCompile(`[^\d]\d{3,}`)
+	patLongNumA = regexp.MustCompile(`\d{3,}[^\d]`)
 )
 
 func IsGibberish(str string) bool {
-	if patBase64.MatchString(str) && len(str) > 32 {
+	if IsVersionString(str) {
+		return false
+	}
+
+	if patEmail.MatchString(str) {
 		return true
 	}
 
@@ -24,16 +28,44 @@ func IsGibberish(str string) bool {
 		return true
 	}
 
-	if patEmail.MatchString(str) {
+	if patLongNum.MatchString(str) || patLongNumB.MatchString(str) || patLongNumA.MatchString(str) {
 		return true
 	}
 
-	if patHexLower.MatchString(str) || patHexUpper.MatchString(str) || patLongNum.MatchString(str) {
-		return true
+	//alNum := cleanStr(str, isAlNumRune)
+	//alpha := cleanStr(str, isAlphaRune)
+	// noiseAll := isNoisy(alNum)
+	//triAll := isTrigramBad(strings.ToLower(alpha))
+	// _ = noiseAll
+
+	isNotAlNum := func(r rune) bool { return !isAlNumRune(r) }
+	chunks := strings.FieldsFunc(str, isNotAlNum)
+	noisyLen := 0
+	alnumLen := 0
+	for _, chunk := range chunks {
+		alnumLen += len(chunk)
+		noise := isNoisy(chunk)
+		tri := isTrigramBad(strings.ToLower(chunk))
+		if noise || tri {
+			noisyLen += len(chunk)
+		}
 	}
 
-	noise := noiseLevel(str)
-	return noise >= 0.2
+	return float64(noisyLen) > 0
+
+	//if float64(noisyLen) > 0 {
+	//	return true
+	//}
+
+	//if len(chunks) > 0 && float64(noisyLen) >= float64(alnumLen)/3.0 {
+	//	return true
+	//}
+
+	//if triAll {
+	//return true
+	//}
+
+	// return false
 }
 
 func noiseLevel(str string) (score float64) {
@@ -51,21 +83,21 @@ func noiseLevel(str string) (score float64) {
 
 			// upper =>
 			case unicode.IsUpper(prev) && unicode.IsLower(char):
-				score += 0.25
+				score += 0.10
 			case unicode.IsUpper(prev) && unicode.IsDigit(char):
-				score += 0.25
+				score += 0.5
 
 			// lower =>
 			case unicode.IsLower(prev) && unicode.IsUpper(char):
 				score += 0.75
 			case unicode.IsLower(prev) && unicode.IsDigit(char):
-				score += 0.25
+				score += 0.5
 
 			// digit =>
 			case unicode.IsDigit(prev) && unicode.IsUpper(char):
 				score += 0.75
 			case unicode.IsDigit(prev) && unicode.IsLower(char):
-				score += 0.75
+				score += 1.0
 
 			// the rest is 100% noise
 			default:
@@ -75,8 +107,6 @@ func noiseLevel(str string) (score float64) {
 		prev = char
 	}
 
-	score /= cnt // weigh it
-
 	return score
 }
 
@@ -97,9 +127,59 @@ func IsVersionString(component string) bool {
 		}
 	}
 
-	if !hasV && strings.Contains(component, ".") {
+	if !hasV && !strings.Contains(component, ".") {
 		return false
 	}
 
 	return true
 }
+
+func trigramScore(str string) (float64, int) {
+	tgScore := 0.0
+	trigrams := ngrams(str, 3)
+	if len(trigrams) > 0 {
+		for _, trigram := range trigrams {
+			score, found := corpus_trigrams[trigram]
+			if found {
+				tgScore += score
+			}
+		}
+	}
+
+	return tgScore, len(trigrams)
+}
+
+func isTrigramBad(s string) bool {
+	tgScore, cnt := trigramScore(s)
+
+	if cnt > 0 {
+		val := math.Sqrt(tgScore) / float64(cnt)
+		val2 := tgScore / float64(cnt)
+		threshold := 0.005
+		bad := val < threshold
+		threshold2 := math.Log(float64(cnt)-2) * 0.1
+		bad2 := val2 < threshold2
+		return bad && bad2 // TODO: improve this logic to be clearer
+	}
+	return false
+}
+
+func isNoisy(s string) bool {
+	noise := noiseLevel(s)
+
+	if len(s) > 0 {
+		val := (noise * noise) / float64(len(s))
+		threshold := 0.6
+		bad := val > threshold
+		return bad
+	}
+	return false
+}
+
+func ngrams(s string, n int) []string {
+	result := make([]string, 0)
+	for i := 0; i < len(s)-n+1; i++ {
+		result = append(result, s[i:i+n])
+	}
+	return result
+}

agent/pkg/oas/gibberish_test.go

@@ -1,17 +1,95 @@
 package oas
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestNegative(t *testing.T) {
 	cases := []string{
 		"",
+		"{}",
+		"0.0.29",
+		"0.1",
+		"1.0",
+		"1.0.0",
+		"2.1.73",
+		"abTestV2",
+		"actionText,setName,setAttribute,save,ignore,onEnd,getContext,end,get",
+		"AddUserGroupLink",
+		"advert-management.adBlockerMessage.html",
+		"agents.author.1.json",
+		"animated-gif",
 		"b", // can be valid hexadecimal
-		"GetUniversalVariableUser",
+		"big-danger-coronavirus-panic-greater-crisis",
+		"breakout-box",
 		"callback",
-		"runs",
-		"tcfv2",
-		"StartUpCheckout",
+		"core.algorithm_execution.view",
+		"core.devices.view",
+		"data.json",
+		"dialog.overlay.infinity.json",
+		"domain-input",
+		"embeddable", // lul, it's a valid HEX!
+		"embeddable_blip",
+		"E PLURIBUS UNUM",
+		"etc",
+		"eu-central-1a",
+		"fcgi-bin",
+		"footer.include.html",
+		"fullHashes:find",
+		"generate-feed",
+		"GetAds",
 		"GetCart",
+		"GetUniversalVariableUser",
+		"github-audit-exports",
+		"g.js",
+		"g.pixel",
+		".html",
+		"Hugo Michiels",
+		"image.sbix",
+		"index.html",
+		"iPad",
+		"Joanna Mazewski",
+		"LibGit2Sharp",
+		"Michael_Vaughan1.png",
+		"New RSS feed has been generated",
+		"nick-clegg",
+		"opt-out",
+		"pixel_details.html",
+		"post.json",
+		"profile-method-info",
+		"project-id",
+		"publisha.1.json",
+		"publish_and_moderate",
+		"Ronna McDaniel",
+		"rtb-h",
+		"runs",
+		"sign-up",
+		"some-uuid-maybe",
+		"stable-4.0-version.json",
+		"StartUpCheckout",
+		"Steve Flunk",
+		"sync_a9",
+		"Ted Cruz",
+		"test.png",
+		"token",
+		"ToList",
+		"v2.1.3",
+		"VersionCheck.php",
+		"v Rusiji",
+		"Walnut St",
+		"web_widget",
+		"zoom_in.cur",
+		"xray",
+		"web",
+		"vipbets1",
+		"trcc",
+		"fbpixel",
+
+		// TODO below
+		// "tcfv2",
+		// "Matt-cartoon-255x206px-small.png",
+		// "TheTelegraph_portal_white-320-small.png",
+		// "testdata-10kB.js",
 	}
 
 	for _, str := range cases {
@@ -23,44 +101,92 @@ func TestNegative(t *testing.T) {
 
 func TestPositive(t *testing.T) {
 	cases := []string{
-		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
-		"952bea17-3776-11ea-9341-42010a84012a",
-		"456795af-b48f-4a8d-9b37-3e932622c2f0",
 		"0a0d0174-b338-4520-a1c3-24f7e3d5ec50.html",
-		"6120c057c7a97b03f6986f1b",
+		"1024807212418223",
+		"11ca096cbc224a67360493d44a9903",
+		"1553183382779",
+		"1554507871",
+		"19180481",
+		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
+		"456795af-b48f-4a8d-9b37-3e932622c2f0",
+		"601a2bdcc5b69137248ddbbf",
+		"60fe9aaeaefe2400012df94f",
 		"610bc3fd5a77a7fa25033fb0",
 		"610bd0315a77a7fa25034368",
 		"610bd0315a77a7fa25034368zh",
+		"6120c057c7a97b03f6986f1b",
 		"710a462e",
-		"1554507871",
-		"qwerqwerasdfqwer@protonmai.com",
-		"john.dow.1981@protonmail.com",
-		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
-		"11ca096cbc224a67360493d44a9903",
-		"c738338322370b47a79251f7510dd", // prefixed hex
-		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
-		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
-		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
-		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
 		"730970532670-compute@developer.gserviceaccount.com",
-		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
+		"819db2242a648b305395537022523d65",
+		"952bea17-3776-11ea-9341-42010a84012a",
+		"a3226860758.html",
 		"AAAA028295945",
-		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
+		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4",
+		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
+		"bnjksfd897345nl098asd53412kl98",
+		"c738338322370b47a79251f7510dd",                        // prefixed hex
+		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
+		"css/login.0f48c49a34eb53ea4623.min.css",
+		"d_fLLxlhzDilixeBEimaZ5",
+		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
+		"e8782afc112720300c049ff124434b79",
+		"fb6cjraf9cejut2a",
+		"i-0236530c66ed02200",
+		"JEHJW4BKVFDRTMTUQLHKK5WVAU",
+		"john.dow.1981@protonmail.com",
+		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
+		"MNUTGVFMGLEMFTBH0XSE5E02F6J2DS",
 		"n63nd45qsj",
 		"n9z9QGNiz",
+		"NC4WTmcy",
 		"proxy.3d2100fd7107262ecb55ce6847f01fa5.html",
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
+		"qwerqwerasdfqwer@protonmai.com",
 		"r-ext-5579e00a95c90",
 		"r-ext-5579e8b12f11e",
 		"r-v4-5c92513c9e04a",
 		"r-v4-5c92513c9e04a-0-c.monitoring.json",
 		"segments-1563566437171.639994",
+		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
+		"sp_dxJTfx11_576742227280287872",
+		"sp_NnUPB5wj_601a2bdcc5b69137248ddbbf",
+		"sp_NxITuoE4_premiumchron-article-14302157_c_ryGQBs_r_yIWvwP",
 		"t_52d94268-8810-4a7e-ba87-ffd657a6752f",
 		"timeouts-1563566437171.639994",
+		"u_YPF3GsGKMo02",
+
+		"0000000000 65535 f",
+		"0000000178 00000 n",
+		"0-10000",
+		"01526123,",
+		"0,18168,183955,3,4,1151616,5663,731,223,5104,207,3204,10,1051,175,364,1435,4,60,576,241,383,246,5,1102",
+		"05/10/2020",
+		"14336456724940333",
+		"fb6cjraf9cejut2a",
+		"JEHJW4BKVFDRTMTUQLHKK5WVAU",
 
 		// TODO
-		// "fb6cjraf9cejut2a",
-		// "Fxvd1timk", // questionable
-		// "JEHJW4BKVFDRTMTUQLHKK5WVAU",
+		/*
+			"0,20",
+			"0.001",
+			"YISAtiX1",
+			"Fxvd1timk", // questionable
+			"B4GCSkORAJs",
+			"D_4EDAqenHQ",
+			"EICJp29EGOk",
+			"Fxvd1timk",
+			"GTqMZELYfQQ",
+			"GZPTpLPEGmwHGWPC",
+			"_HChnE9NDPY",
+			"NwhjgIWHgGg",
+			"production/tsbqksph4xswqjexfbec",
+			"p/u/bguhrxupr23mw3nwxcrw",
+			"nRSNapbJZnc",
+			"zgfpbtolciznub5egzxk",
+			"zufnu7aimadua9wrgwwo",
+			"zznto1jzch9yjsbtbrul",
+		*/
 	}
 
 	for _, str := range cases {
@@ -69,3 +195,18 @@ func TestPositive(t *testing.T) {
 		}
 	}
 }
+
+func TestVersionStrings(t *testing.T) {
+	cases := []string{
+		"1.0",
+		"1.0.0",
+		"v2.1.3",
+		"2.1.73",
+	}
+
+	for _, str := range cases {
+		if !IsVersionString(str) {
+			t.Errorf("Mistakenly false: %s", str)
+		}
+	}
+}

agent/pkg/oas/ignores.go

@@ -12,9 +12,9 @@ var ignoredHeaders = []string{
 	"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-type", "cookie",
 	"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
 	"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
-	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests",
-	"server", "user-agent", "via", "warning", "strict-transport-security",
-	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data",
+	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests", "x-download-options",
+	"server", "user-agent", "via", "warning", "strict-transport-security", "x-permitted-cross-domain-policies",
+	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data", "x-dns-prefetch-control",
 	"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
 	"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
 	"x-wap-profile", "x-scheme", "status", "x-cache", "x-application-context", "retry-after",
@@ -31,7 +31,7 @@ var ignoredHeaderPrefixes = []string{
 	":", "accept-", "access-control-", "if-", "sec-", "grpc-",
 	"x-forwarded-", "x-original-", "cf-",
 	"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
-	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-",
+	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-", "x-goog-",
 }
 
 func isCtypeIgnored(ctype string) bool {

agent/pkg/oas/oas_generator.go

@@ -79,6 +79,10 @@ func (g *oasGenerator) runGeneretor() {
 	}
 }
 
+func (g *oasGenerator) Reset() {
+	g.ServiceSpecs = &sync.Map{}
+}
+
 func (g *oasGenerator) PushEntry(entryWithSource *EntryWithSource) {
 	if !g.started {
 		return
@@ -103,6 +107,7 @@ func newOasGenerator() *oasGenerator {
 type EntryWithSource struct {
 	Source string
 	Entry  har.Entry
+	Id     uint
 }
 
 type oasGenerator struct {

agent/pkg/oas/specgen.go

@@ -3,13 +3,17 @@ package oas
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"github.com/chanced/openapi"
 	"github.com/google/uuid"
 	"github.com/nav-inc/datetime"
 	"github.com/up9inc/mizu/shared/logger"
+	"io"
+	"io/ioutil"
 	"mime"
+	"mime/multipart"
+	"net/textproto"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -22,6 +26,7 @@ import (
 const LastSeenTS = "x-last-seen-ts"
 const CountersTotal = "x-counters-total"
 const CountersPerSource = "x-counters-per-source"
+const SampleId = "x-sample-entry"
 
 type reqResp struct { // hello, generics in Go
 	Req  *har.Request
@@ -126,6 +131,7 @@ func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
 
 func suggestTags(oas *openapi.OpenAPI) {
 	paths := getPathsKeys(oas.Paths.Items)
+	sort.Strings(paths) // make it stable in case of multiple candidates
 	for len(paths) > 0 {
 		group := make([]string, 0)
 		group = append(group, paths[0])
@@ -155,37 +161,9 @@ func suggestTags(oas *openapi.OpenAPI) {
 				}
 			}
 		}
-
-		//groups[common] = group
 	}
 }
 
-func getSimilarPrefix(strs []string) string {
-	chunked := make([][]string, 0)
-	for _, item := range strs {
-		chunked = append(chunked, strings.Split(item, "/"))
-	}
-
-	cmn := longestCommonXfix(chunked, true)
-	res := make([]string, 0)
-	for _, chunk := range cmn {
-		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
-			res = append(res, chunk)
-		}
-	}
-	return strings.Join(res[1:], ".")
-}
-
-func deleteFromSlice(s []string, val string) []string {
-	temp := s[:0]
-	for _, x := range s {
-		if x != val {
-			temp = append(temp, x)
-		}
-	}
-	return temp
-}
-
 func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
 	keys := make([]string, len(mymap))
 
@@ -329,7 +307,7 @@ func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *Ent
 	rt := float64(entryWithSource.Entry.Time) / 1000
 
 	dur := 0.0
-	if prevTs != 0 {
+	if prevTs != 0 && ts >= prevTs {
 		dur = ts - prevTs
 	}
 
@@ -351,19 +329,39 @@ func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *Ent
 		return err
 	}
 
+	err = opObj.Extensions.SetExtension(SampleId, entryWithSource.Id)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
 func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error {
 	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
+	urlParsed, err := url.Parse(req.URL)
+	if err != nil {
+		return err
+	}
+
+	qs := make([]har.NVP, 0)
+	for name, vals := range urlParsed.Query() {
+		for _, val := range vals {
+			qs = append(qs, har.NVP{Name: name, Value: val})
+		}
+	}
+
+	if len(qs) != len(req.QueryString) {
+		logger.Log.Warningf("QStr params in HAR do not match URL: %s", req.URL)
+	}
 
 	qstrGW := nvParams{
 		In:             openapi.InQuery,
-		Pairs:          req.QueryString,
+		Pairs:          qs,
 		IsIgnored:      func(name string) bool { return false },
 		GeneralizeName: func(name string) string { return name },
 	}
-	handleNameVals(qstrGW, &opObj.Parameters)
+	handleNameVals(qstrGW, &opObj.Parameters, false)
 
 	hdrGW := nvParams{
 		In:             openapi.InHeader,
@@ -371,22 +369,27 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e
 		IsIgnored:      isHeaderIgnored,
 		GeneralizeName: strings.ToLower,
 	}
-	handleNameVals(hdrGW, &opObj.Parameters)
+	handleNameVals(hdrGW, &opObj.Parameters, true)
 
-	if req.PostData.Text != "" && isSuccess {
-		reqBody, err := getRequestBody(req, opObj, isSuccess)
+	if isSuccess {
+		reqBody, err := getRequestBody(req, opObj)
 		if err != nil {
 			return err
 		}
 
 		if reqBody != nil {
-			reqCtype := getReqCtype(req)
-			reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype)
-			if err != nil {
-				return err
-			}
+			if req.PostData.Text == "" {
+				reqBody.Required = false
+			} else {
 
-			_ = reqMedia
+				reqCtype, _ := getReqCtype(req)
+				reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype)
+				if err != nil {
+					return err
+				}
+
+				_ = reqMedia
+			}
 		}
 	}
 	return nil
@@ -487,12 +490,123 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string) (*o
 			}
 		}
 
-		content.Example = exampleMsg
+		if ctype == "application/x-www-form-urlencoded" && reqResp.Req != nil {
+			handleFormDataUrlencoded(text, content)
+		} else if strings.HasPrefix(ctype, "multipart/form-data") && reqResp.Req != nil {
+			_, params := getReqCtype(reqResp.Req)
+			handleFormDataMultipart(text, content, params)
+		}
+
+		if content.Example == nil && len(exampleMsg) > len(content.Example) {
+			content.Example = exampleMsg
+		}
 	}
 
 	return respContent[ctype], nil
 }
 
+func handleFormDataUrlencoded(text string, content *openapi.MediaType) {
+	formData, err := url.ParseQuery(text)
+	if err != nil {
+		logger.Log.Warningf("Could not decode urlencoded: %s", err)
+		return
+	}
+
+	parts := make([]PartWithBody, 0)
+	for name, vals := range formData {
+		for _, val := range vals {
+			part := new(multipart.Part)
+			part.Header = textproto.MIMEHeader{}
+			part.Header.Add("Content-Disposition", "form-data; name=\""+name+"\";")
+			parts = append(parts, PartWithBody{part: part, body: []byte(val)})
+		}
+	}
+	handleFormData(content, parts)
+}
+
+func handleFormData(content *openapi.MediaType, parts []PartWithBody) {
+	hadSchema := true
+	if content.Schema == nil {
+		hadSchema = false // will use it for required flags
+		content.Schema = new(openapi.SchemaObj)
+		content.Schema.Type = openapi.Types{openapi.TypeObject}
+		content.Schema.Properties = openapi.Schemas{}
+	}
+
+	props := &content.Schema.Properties
+	seenNames := map[string]struct{}{} // set equivalent in Go, yikes
+	for _, pwb := range parts {
+		name := pwb.part.FormName()
+		seenNames[name] = struct{}{}
+		existing, found := (*props)[name]
+		if !found {
+			existing = new(openapi.SchemaObj)
+			existing.Type = openapi.Types{openapi.TypeString}
+			(*props)[name] = existing
+
+			ctype := pwb.part.Header.Get("content-type")
+			if ctype != "" {
+				if existing.Keywords == nil {
+					existing.Keywords = map[string]json.RawMessage{}
+				}
+				existing.Keywords["contentMediaType"], _ = json.Marshal(ctype)
+			}
+		}
+
+		addSchemaExample(existing, string(pwb.body))
+	}
+
+	// handle required flag
+	if content.Schema.Required == nil {
+		if !hadSchema {
+			content.Schema.Required = make([]string, 0)
+			for name := range seenNames {
+				content.Schema.Required = append(content.Schema.Required, name)
+			}
+			sort.Strings(content.Schema.Required)
+		} // else it's a known schema with no required fields
+	} else {
+		content.Schema.Required = intersectSliceWithMap(content.Schema.Required, seenNames)
+		sort.Strings(content.Schema.Required)
+	}
+}
+
+type PartWithBody struct {
+	part *multipart.Part
+	body []byte
+}
+
+func handleFormDataMultipart(text string, content *openapi.MediaType, ctypeParams map[string]string) {
+	boundary, ok := ctypeParams["boundary"]
+	if !ok {
+		logger.Log.Errorf("Multipart header has no boundary")
+		return
+	}
+	mpr := multipart.NewReader(strings.NewReader(text), boundary)
+
+	parts := make([]PartWithBody, 0)
+	for {
+		part, err := mpr.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			logger.Log.Errorf("Cannot parse multipart body: %v", err)
+			break
+		}
+		defer part.Close()
+
+		body, err := ioutil.ReadAll(part)
+		if err != nil {
+			logger.Log.Errorf("Error reading multipart Part %s: %v", part.Header, err)
+		}
+
+		parts = append(parts, PartWithBody{part: part, body: body})
+	}
+
+	handleFormData(content, parts)
+}
+
 func getRespCtype(resp *har.Response) string {
 	var ctype string
 	ctype = resp.Content.MimeType
@@ -509,8 +623,7 @@ func getRespCtype(resp *har.Response) string {
 	return mediaType
 }
 
-func getReqCtype(req *har.Request) string {
-	var ctype string
+func getReqCtype(req *har.Request) (ctype string, params map[string]string) {
 	ctype = req.PostData.MimeType
 	for _, hdr := range req.Headers {
 		if strings.ToLower(hdr.Name) == "content-type" {
@@ -518,11 +631,16 @@ func getReqCtype(req *har.Request) string {
 		}
 	}
 
-	mediaType, _, err := mime.ParseMediaType(ctype)
-	if err != nil {
-		return ""
+	if ctype == "" {
+		return "", map[string]string{}
 	}
-	return mediaType
+
+	mediaType, params, err := mime.ParseMediaType(ctype)
+	if err != nil {
+		logger.Log.Errorf("Cannot parse Content-Type header %q: %v", ctype, err)
+		return "", map[string]string{}
+	}
+	return mediaType, params
 }
 
 func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool) (*openapi.ResponseObj, error) {
@@ -552,9 +670,14 @@ func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool
 	return resResponse, nil
 }
 
-func getRequestBody(req *har.Request, opObj *openapi.Operation, isSuccess bool) (*openapi.RequestBodyObj, error) {
+func getRequestBody(req *har.Request, opObj *openapi.Operation) (*openapi.RequestBodyObj, error) {
 	if opObj.RequestBody == nil {
-		opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
+		// create if there is body in request
+		if req.PostData.Text != "" {
+			opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
+		} else {
+			return nil, nil
+		}
 	}
 
 	reqBody, err := opObj.RequestBody.ResolveRequestBody(reqBodyResolver)
@@ -562,9 +685,6 @@ func getRequestBody(req *har.Request, opObj *openapi.Operation, isSuccess bool)
 		return nil, err
 	}
 
-	// TODO: maintain required flag for it, but only consider successful responses
-	//reqBody.Content[]
-
 	return reqBody, nil
 }
 
@@ -607,56 +727,3 @@ func getOpObj(pathObj *openapi.PathObj, method string, createIfNone bool) (*open
 
 	return *op, isMissing, nil
 }
-
-type CounterMaps struct {
-	counterTotal    Counter
-	counterMapTotal CounterMap
-}
-
-func (m *CounterMaps) processOp(opObj *openapi.Operation) error {
-	if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
-		counter := new(Counter)
-		err := opObj.Extensions.DecodeExtension(CountersTotal, counter)
-		if err != nil {
-			return err
-		}
-		m.counterTotal.addOther(counter)
-
-		opObj.Description = setCounterMsgIfOk(opObj.Description, counter)
-	}
-
-	if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
-		counterMap := new(CounterMap)
-		err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap)
-		if err != nil {
-			return err
-		}
-		m.counterMapTotal.addOther(counterMap)
-	}
-	return nil
-}
-
-func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error {
-	if oas.Extensions == nil {
-		oas.Extensions = openapi.Extensions{}
-	}
-
-	err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal)
-	if err != nil {
-		return err
-	}
-
-	err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal)
-	if err != nil {
-		return nil
-	}
-	return nil
-}
-
-func setCounterMsgIfOk(oldStr string, cnt *Counter) string {
-	tpl := "Mizu observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds"
-	if oldStr == "" || (strings.HasPrefix(oldStr, "Mizu ") && strings.HasSuffix(oldStr, " seconds")) {
-		return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries))
-	}
-	return oldStr
-}

agent/pkg/oas/specgen_test.go

@@ -2,20 +2,22 @@ package oas
 
 import (
 	"encoding/json"
+	"github.com/chanced/openapi"
+	"github.com/op/go-logging"
+	"github.com/up9inc/mizu/shared/logger"
+	"github.com/wI2L/jsondiff"
 	"io/ioutil"
 	"os"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/chanced/openapi"
-	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 // if started via env, write file into subdir
-func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) {
+func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string {
 	content, err := json.MarshalIndent(spec, "", "\t")
 	if err != nil {
 		panic(err)
@@ -35,6 +37,7 @@ func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) {
 	} else {
 		t.Logf("%s", string(content))
 	}
+	return string(content)
 }
 
 func TestEntries(t *testing.T) {
@@ -103,6 +106,7 @@ func TestEntries(t *testing.T) {
 
 func TestFileSingle(t *testing.T) {
 	GetOasGeneratorInstance().Start()
+	GetOasGeneratorInstance().Reset()
 	// loadStartingOAS()
 	file := "test_artifacts/params.har"
 	files := []string{file}
@@ -123,7 +127,7 @@ func TestFileSingle(t *testing.T) {
 			t.FailNow()
 		}
 
-		outputSpec(svc, spec, t)
+		specText := outputSpec(svc, spec, t)
 
 		err = spec.Validate()
 		if err != nil {
@@ -131,6 +135,29 @@ func TestFileSingle(t *testing.T) {
 			t.FailNow()
 		}
 
+		expected, err := ioutil.ReadFile(file + ".spec.json")
+		if err != nil {
+			t.Errorf(err.Error())
+			t.FailNow()
+		}
+
+		patFloatPrecision := regexp.MustCompile(`(\d+\.\d{1,2})(\d*)`)
+
+		expected = []byte(patUuid4.ReplaceAllString(string(expected), "<UUID4>"))
+		specText = patUuid4.ReplaceAllString(specText, "<UUID4>")
+		expected = []byte(patFloatPrecision.ReplaceAllString(string(expected), "$1"))
+		specText = patFloatPrecision.ReplaceAllString(specText, "$1")
+
+		diff, err := jsondiff.CompareJSON(expected, []byte(specText))
+		if err != nil {
+			t.Errorf(err.Error())
+			t.FailNow()
+		}
+
+		if len(diff) > 0 {
+			t.Errorf("Generated spec does not match expected:\n%s", diff.String())
+		}
+
 		return true
 	})
 

agent/pkg/oas/test_artifacts/params.har

@@ -235,6 +235,309 @@
           "connect": 262,
           "ssl": -1
         }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:20.747122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/form-urlencoded",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "Content-Type",
+              "value": "application/x-www-form-urlencoded"
+            }
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "agent-id=ade&callback-url=&token=sometoken"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/form-urlencoded",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "Content-Type",
+              "value": "application/x-www-form-urlencoded"
+            }
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "agent-id=ade&callback-url=&token=sometoken-second-val&optional=another"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:22.747122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/form-multipart",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "Content-Type",
+              "value": "multipart/form-data; boundary=BOUNDARY"
+            }
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 62,
+            "mimeType": "",
+            "text": "{}"
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 62
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/body-optional",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/body-optional",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "Content-Type",
+              "value": "application/json"
+            }
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "{\"key\", \"val\"}"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/body-optional",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/body-required",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "body exists"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
       }
     ]
   }

agent/pkg/oas/test_artifacts/params.har.spec.json

@@ -0,0 +1,507 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "https://httpbin.org",
+    "description": "Mizu observed 13 entries (0 failed), at 0.155 hits/s, average response time is 0.251 seconds",
+    "version": "1.0"
+  },
+  "servers": [
+    {
+      "url": "https://httpbin.org"
+    }
+  ],
+  "paths": {
+    "/appears-once": {
+      "get": {
+        "summary": "/appears-once",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "89aa39f6-78d0-411b-b701-a33bd77868b0",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null
+              }
+            }
+          }
+        },
+        "x-last-seen-ts": 1567750580.0471218,
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750580.0471218,
+          "lastSeen": 1567750580.0471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750580.0471218,
+            "lastSeen": 1567750580.0471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-sample-entry": 0
+      }
+    },
+    "/appears-twice": {
+      "get": {
+        "summary": "/appears-twice",
+        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds",
+        "operationId": "f5e2b5a2-e01a-45f4-bde1-15a7e7a06d3c",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null
+              }
+            }
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750580.7471218,
+          "lastSeen": 1567750581.7471218,
+          "sumRT": 1.26,
+          "sumDuration": 1
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750580.7471218,
+            "lastSeen": 1567750581.7471218,
+            "sumRT": 1.26,
+            "sumDuration": 1
+          }
+        },
+        "x-sample-entry": 0,
+        "x-last-seen-ts": 1567750581.7471218
+      }
+    },
+    "/body-optional": {
+      "post": {
+        "summary": "/body-optional",
+        "description": "Mizu observed 3 entries (0 failed), at 0.003 hits/s, average response time is 0.001 seconds",
+        "operationId": "14d5b1c2-dc03-4ee5-baaa-5c7992acc82e",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {}
+            }
+          }
+        },
+        "x-counters-total": {
+          "entries": 3,
+          "failures": 0,
+          "firstSeen": 1567750581.7471218,
+          "lastSeen": 1567750581.757122,
+          "sumRT": 0.003,
+          "sumDuration": 0.010000228881835938
+        },
+        "x-sample-entry": 0,
+        "x-counters-per-source": {
+          "": {
+            "entries": 3,
+            "failures": 0,
+            "firstSeen": 1567750581.7471218,
+            "lastSeen": 1567750581.757122,
+            "sumRT": 0.003,
+            "sumDuration": 0.010000228881835938
+          }
+        },
+        "x-last-seen-ts": 1567750581.757122,
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "application/json": {
+              "example": "{\"key\", \"val\"}"
+            }
+          }
+        }
+      }
+    },
+    "/body-required": {
+      "post": {
+        "summary": "/body-required",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "d0958c5a-dce6-4616-99f4-201dbc51457a",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {}
+            }
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750581.757122,
+            "lastSeen": 1567750581.757122,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-sample-entry": 0,
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750581.757122,
+          "lastSeen": 1567750581.757122,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750581.757122,
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "": {
+              "example": "body exists"
+            }
+          },
+          "required": true
+        }
+      }
+    },
+    "/form-multipart": {
+      "post": {
+        "summary": "/form-multipart",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "cab5a2f3-c18a-4d5a-8f92-e40da4fd6603",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "example": {}
+              }
+            }
+          }
+        },
+        "x-sample-entry": 0,
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.7471218,
+            "lastSeen": 1567750582.7471218,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-last-seen-ts": 1567750582.7471218,
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.7471218,
+          "lastSeen": 1567750582.7471218,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "multipart/form-data": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "file",
+                  "path"
+                ],
+                "properties": {
+                  "file": {
+                    "type": "string",
+                    "contentMediaType": "application/json",
+                    "examples": [
+                      "{\"functions\": 123}"
+                    ]
+                  },
+                  "path": {
+                    "type": "string",
+                    "examples": [
+                      "/content/components"
+                    ]
+                  }
+                }
+              },
+              "example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n"
+            }
+          },
+          "required": true
+        }
+      }
+    },
+    "/form-urlencoded": {
+      "post": {
+        "summary": "/form-urlencoded",
+        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds",
+        "operationId": "7c373ad7-6ab5-422e-971b-1cf56b18a7a2",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {}
+            }
+          }
+        },
+        "x-last-seen-ts": 1567750581.7471218,
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750580.7471218,
+          "lastSeen": 1567750581.7471218,
+          "sumRT": 0.002,
+          "sumDuration": 1
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750580.7471218,
+            "lastSeen": 1567750581.7471218,
+            "sumRT": 0.002,
+            "sumDuration": 1
+          }
+        },
+        "x-sample-entry": 0,
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "application/x-www-form-urlencoded": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "agent-id",
+                  "callback-url",
+                  "token"
+                ],
+                "properties": {
+                  "agent-id": {
+                    "type": "string",
+                    "examples": [
+                      "ade"
+                    ]
+                  },
+                  "callback-url": {
+                    "type": "string",
+                    "examples": [
+                      ""
+                    ]
+                  },
+                  "optional": {
+                    "type": "string",
+                    "examples": [
+                      "another"
+                    ]
+                  },
+                  "token": {
+                    "type": "string",
+                    "examples": [
+                      "sometoken",
+                      "sometoken-second-val"
+                    ]
+                  }
+                }
+              },
+              "example": "agent-id=ade\u0026callback-url=\u0026token=sometoken"
+            }
+          },
+          "required": true
+        }
+      }
+    },
+    "/{Id}": {
+      "get": {
+        "summary": "/{Id}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "99f1d11f-29c0-48f9-8bf0-9f4b407c7c3f",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null
+              }
+            }
+          }
+        },
+        "x-sample-entry": 0,
+        "x-last-seen-ts": 1567750579.7471218,
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750579.7471218,
+          "lastSeen": 1567750579.7471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750579.7471218,
+            "lastSeen": 1567750579.7471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        }
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          }
+        }
+      ]
+    },
+    "/{Id}/sub1": {
+      "get": {
+        "summary": "/{Id}/sub1",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds",
+        "operationId": "f7e299d2-253c-4eef-975c-9a5659a7fc50",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "text/html": {}
+            }
+          }
+        },
+        "x-last-seen-ts": 1567750483.864529,
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750483.864529,
+          "lastSeen": 1567750483.864529,
+          "sumRT": 0.111,
+          "sumDuration": 0
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750483.864529,
+            "lastSeen": 1567750483.864529,
+            "sumRT": 0.111,
+            "sumDuration": 0
+          }
+        },
+        "x-sample-entry": 0
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          }
+        }
+      ]
+    },
+    "/{Id}/sub2": {
+      "get": {
+        "summary": "/{Id}/sub2",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "23a54e06-4298-4ea5-b1f0-09b0354a0598",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null
+              }
+            }
+          }
+        },
+        "x-last-seen-ts": 1567750578.7471218,
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750578.7471218,
+          "lastSeen": 1567750578.7471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750578.7471218,
+            "lastSeen": 1567750578.7471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-sample-entry": 0
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          }
+        }
+      ]
+    }
+  },
+  "x-counters-total": {
+    "entries": 13,
+    "failures": 0,
+    "firstSeen": 1567750483.864529,
+    "lastSeen": 1567750582.7471218,
+    "sumRT": 3.268,
+    "sumDuration": 2.010000228881836
+  },
+  "x-counters-per-source": {
+    "": {
+      "entries": 13,
+      "failures": 0,
+      "firstSeen": 1567750483.864529,
+      "lastSeen": 1567750582.7471218,
+      "sumRT": 3.268,
+      "sumDuration": 2.010000228881836
+    }
+  }
+}

agent/pkg/oas/tree.go

@@ -62,7 +62,6 @@ func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node *
 		if paramObj != nil {
 			node.pathParam = paramObj
 		} else if chunkIsGibberish {
-
 			newParam := n.createParam()
 			node.pathParam = newParam
 		} else {
@@ -100,11 +99,14 @@ func (n *Node) createParam() *openapi.ParameterObj {
 		} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
 			name = *n.constant
 			name = name[:len(name)-1] + "Id"
-		} else if isAlpha(*n.constant) {
+		} else {
 			name = *n.constant + "Id"
 		}
 
-		name = cleanNonAlnum([]byte(name))
+		name = cleanStr(name, isAlNumRune)
+		if !isAlphaRune(rune(name[0])) {
+			name = "_" + name
+		}
 	}
 
 	newParam := createSimpleParam(name, "path", "string")

agent/pkg/oas/tree_test.go

@@ -15,7 +15,7 @@ func TestTree(t *testing.T) {
 		{"/", 0, ""},
 		{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation", 1, "vendor,init,conversation"},
 		{"/v1.0.0/config/launcher/sp_nKNHCzsN/{f34efcae-6583-11eb-908a-00b0fcb9d4f6}/vendor,init,conversation", 0, "vendor,init,conversation"},
-		{"/getSvgs/size/small/brand/SFLY/layoutId/170943/layoutVersion/1/sizeId/742/surface/0/isLandscape/true/childSkus/%7B%7D", 1, ""},
+		{"/getSvgs/size/small/brand/SFLY/layoutId/170943/layoutVersion/1/sizeId/742/surface/0/isLandscape/true/childSkus/%7B%7D", 1, "{}"},
 	}
 
 	tree := new(Node)
@@ -23,7 +23,7 @@ func TestTree(t *testing.T) {
 		split := strings.Split(tc.inp, "/")
 		pathObj := new(openapi.PathObj)
 		node := tree.getOrSet(split, pathObj)
-		
+
 		fillPathParams(node, pathObj)
 
 		if node.constant != nil && *node.constant != tc.label {

agent/pkg/oas/utils.go

@@ -52,8 +52,7 @@ func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *op
 	}
 	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
 	schema := new(openapi.SchemaObj)
-	schema.Type = make(openapi.Types, 0)
-	schema.Type = append(schema.Type, ptype)
+	schema.Type = openapi.Types{ptype}
 
 	style := openapi.StyleSimple
 	if in == openapi.InQuery {
@@ -116,10 +115,10 @@ type nvParams struct {
 	GeneralizeName func(name string) string
 }
 
-func handleNameVals(gw nvParams, params **openapi.ParameterList) {
+func handleNameVals(gw nvParams, params **openapi.ParameterList, checkIgnore bool) {
 	visited := map[string]*openapi.ParameterObj{}
 	for _, pair := range gw.Pairs {
-		if gw.IsIgnored(pair.Name) {
+		if (checkIgnore && gw.IsIgnored(pair.Name)) || pair.Name == "" {
 			continue
 		}
 
@@ -197,7 +196,7 @@ func fillParamExample(param **openapi.Examples, exampleValue string) error {
 			continue
 		}
 
-		if value == exampleValue || cnt > 5 { // 5 examples is enough
+		if value == exampleValue || cnt >= 5 { // 5 examples is enough
 			return nil
 		}
 	}
@@ -213,6 +212,36 @@ func fillParamExample(param **openapi.Examples, exampleValue string) error {
 	return nil
 }
 
+// TODO: somehow generalize the two example setting functions, plus add body example handling
+
+func addSchemaExample(existing *openapi.SchemaObj, bodyStr string) {
+	if len(existing.Examples) < 5 {
+		found := false
+		for _, eVal := range existing.Examples {
+			existingExample := ""
+			err := json.Unmarshal(eVal, &existingExample)
+			if err != nil {
+				logger.Log.Debugf("Failed to unmarshal example: %v", eVal)
+				continue
+			}
+
+			if existingExample == bodyStr {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			example, err := json.Marshal(bodyStr)
+			if err != nil {
+				logger.Log.Debugf("Failed to marshal example: %v", bodyStr)
+				return
+			}
+			existing.Examples = append(existing.Examples, example)
+		}
+	}
+}
+
 func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.com/jpillora/longestcommon
 	empty := make([]string, 0)
 	//short-circuit empty list
@@ -261,6 +290,22 @@ func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.c
 	return xfix
 }
 
+func getSimilarPrefix(strs []string) string {
+	chunked := make([][]string, 0)
+	for _, item := range strs {
+		chunked = append(chunked, strings.Split(item, "/"))
+	}
+
+	cmn := longestCommonXfix(chunked, true)
+	res := make([]string, 0)
+	for _, chunk := range cmn {
+		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
+			res = append(res, chunk)
+		}
+	}
+	return strings.Join(res[1:], ".")
+}
+
 // returns all non-nil ops in PathObj
 func getOps(pathObj *openapi.PathObj) []*openapi.Operation {
 	ops := []**openapi.Operation{&pathObj.Get, &pathObj.Patch, &pathObj.Put, &pathObj.Options, &pathObj.Post, &pathObj.Trace, &pathObj.Head, &pathObj.Delete}
@@ -324,13 +369,11 @@ func anyJSON(text string) (anyVal interface{}, isJSON bool) {
 	return nil, false
 }
 
-func cleanNonAlnum(s []byte) string {
+func cleanStr(str string, criterion func(r rune) bool) string {
+	s := []byte(str)
 	j := 0
 	for _, b := range s {
-		if ('a' <= b && b <= 'z') ||
-			('A' <= b && b <= 'Z') ||
-			('0' <= b && b <= '9') ||
-			b == ' ' {
+		if criterion(rune(b)) {
 			s[j] = b
 			j++
 		}
@@ -338,11 +381,49 @@ func cleanNonAlnum(s []byte) string {
 	return string(s[:j])
 }
 
+/*
 func isAlpha(s string) bool {
 	for _, r := range s {
-		if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') {
+		if isAlphaRune(r) {
 			return false
 		}
 	}
 	return true
 }
+*/
+
+func isAlphaRune(r rune) bool {
+	return !((r < 'a' || r > 'z') && (r < 'A' || r > 'Z'))
+}
+
+func isAlNumRune(b rune) bool {
+	return isAlphaRune(b) || ('0' <= b && b <= '9')
+}
+
+func deleteFromSlice(s []string, val string) []string {
+	temp := s[:0]
+	for _, x := range s {
+		if x != val {
+			temp = append(temp, x)
+		}
+	}
+	return temp
+}
+
+func sliceContains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}
+
+func intersectSliceWithMap(required []string, names map[string]struct{}) []string {
+	for name := range names {
+		if !sliceContains(required, name) {
+			required = deleteFromSlice(required, name)
+		}
+	}
+	return required
+}

agent/pkg/oas/utils_test.go

@@ -33,3 +33,27 @@ func TestAnyJSON(t *testing.T) {
 		}
 	}
 }
+
+func TestStrRunes(t *testing.T) {
+	if isAlphaRune('5') {
+		t.Logf("Failed")
+	}
+	if !isAlphaRune('a') {
+		t.Logf("Failed")
+	}
+
+	if !isAlNumRune('5') {
+		t.Logf("Failed")
+	}
+	if isAlNumRune(' ') {
+		t.Logf("Failed")
+	}
+
+	if cleanStr("-abc_567", isAlphaRune) != "abc" {
+		t.Logf("Failed")
+	}
+
+	if cleanStr("-abc_567", isAlNumRune) != "abc567" {
+		t.Logf("Failed")
+	}
+}

agent/pkg/version/consts.go

@@ -1,7 +1,7 @@
 package version
 
 var (
-	SemVer         = "0.0.1"
+	Ver         = "0.0.1"
 	Branch         = "develop"
 	GitCommitHash  = "" // this var is overridden using ldflags in makefile when building
 	BuildTimestamp = "" // this var is overridden using ldflags in makefile when building

cli/Makefile

@@ -3,7 +3,7 @@ COMMIT_HASH=$(shell git rev-parse HEAD)
 GIT_BRANCH=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
 GIT_VERSION=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
 BUILD_TIMESTAMP=$(shell date +%s)
-export SEM_VER?=0.0.0
+export VER?=0.0
 
 .PHONY: help
 .DEFAULT_GOAL := help
@@ -23,13 +23,13 @@ build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables
 					   -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.Platform=$(SUFFIX)' \
-					   -X 'github.com/up9inc/mizu/cli/mizu.SemVer=$(SEM_VER)'" \
+					   -X 'github.com/up9inc/mizu/cli/mizu.Ver=$(VER)'" \
 					   -o bin/mizu_$(SUFFIX) mizu.go
 	(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
 
 build-all: ## Build for all supported platforms.
 	@echo "Compiling for every OS and Platform"
-	@mkdir -p bin && sed s/_SEM_VER_/$(SEM_VER)/g README.md.TEMPLATE >  bin/README.md
+	@mkdir -p bin && sed s/_VER_/$(VER)/g README.md.TEMPLATE >  bin/README.md
 	@$(MAKE) build GOOS=linux GOARCH=amd64
 	@$(MAKE) build GOOS=linux GOARCH=arm64
 	@$(MAKE) build GOOS=darwin GOARCH=amd64

cli/README.md.TEMPLATE

@@ -1,31 +1,31 @@
-# Mizu release _SEM_VER_
+# Mizu release _VER_
 Full changelog for stable release see in [docs](https://github.com/up9inc/mizu/blob/main/docs/CHANGELOG.md)
 
 ## Download Mizu for your platform
 
 **Mac** (x86-64/Intel)
 ```
-curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_darwin_amd64 && chmod 755 mizu
+curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_darwin_amd64 && chmod 755 mizu
 ```
 
 **Mac** (AArch64/Apple M1 silicon)
 ```
-curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_darwin_arm64 && chmod 755 mizu
+curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_darwin_arm64 && chmod 755 mizu
 ```
 
 **Linux** (x86-64)
 ```
-curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_linux_amd64 && chmod 755 mizu
+curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_linux_amd64 && chmod 755 mizu
 ```
 
 **Linux** (AArch64)
 ```
-curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_linux_arm64 && chmod 755 mizu
+curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_linux_arm64 && chmod 755 mizu
 ```
 
 **Windows** (x86-64)
 ```
-curl -LO https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu.exe
+curl -LO https://github.com/up9inc/mizu/releases/download/_VER_/mizu.exe
 ```
 
 ### Checksums

cli/apiserver/provider.go

@@ -143,5 +143,5 @@ func (provider *Provider) GetVersion() (string, error) {
 		return "", err
 	}
 
-	return versionResponse.SemVer, nil
+	return versionResponse.Ver, nil
 }

cli/cmd/version.go

@@ -22,11 +22,11 @@ var versionCmd = &cobra.Command{
 
 		if config.Config.Version.DebugInfo {
 			timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
-			logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
+			logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.Ver, mizu.Branch, mizu.GitCommitHash)
 			logger.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
 
 		} else {
-			logger.Log.Infof("Version: %s (%s)", mizu.SemVer, mizu.Branch)
+			logger.Log.Infof("Version: %s (%s)", mizu.Ver, mizu.Branch)
 		}
 		return nil
 	},

cli/config/configStruct.go

@@ -54,7 +54,7 @@ func (config *ConfigStruct) validate() error {
 func (config *ConfigStruct) SetDefaults() {
 	config.KratosImage = shared.KratosImageDefault
 	config.KetoImage = shared.KetoImageDefault
-	config.AgentImage = fmt.Sprintf("%s:%s", shared.MizuAgentImageRepo, mizu.SemVer)
+	config.AgentImage = fmt.Sprintf("%s:%s", shared.MizuAgentImageRepo, mizu.Ver)
 	config.ConfigFilePath = path.Join(mizu.GetMizuFolderPath(), "config.yaml")
 }
 

cli/mizu/consts.go

@@ -6,7 +6,7 @@ import (
 )
 
 var (
-	SemVer                                     = "0.0.1"
+	Ver                                        = "0.0"
 	Branch                                     = "develop"
 	GitCommitHash                              = "" // this var is overridden using ldflags in makefile when building
 	BuildTimestamp                             = "" // this var is overridden using ldflags in makefile when building

cli/mizu/version/versionCheck.go

@@ -12,31 +12,28 @@ import (
 
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/pkg/version"
 	"github.com/up9inc/mizu/shared/logger"
 
 	"github.com/google/go-github/v37/github"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/semver"
 )
 
 func CheckVersionCompatibility(apiServerProvider *apiserver.Provider) (bool, error) {
-	apiSemVer, err := apiServerProvider.GetVersion()
+	apiVer, err := apiServerProvider.GetVersion()
 	if err != nil {
 		return false, err
 	}
 
-	if !semver.SemVersion(apiSemVer).IsValid() {
-		logger.Log.Errorf(uiUtils.Red, fmt.Sprintf("api version (%s) is not a valid SemVer", apiSemVer))
+	if equals, err := version.AreEquals(apiVer, mizu.Ver); err != nil {
+		return false, fmt.Errorf("Failed to check version equality between mizuVer: %s and apiVer: %s, error: %w", mizu.Ver, apiVer, err)
+	} else if !equals {
+		logger.Log.Errorf(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", mizu.Ver, apiVer))
 		return false, nil
 	}
 
-	if semver.SemVersion(apiSemVer).Major() == semver.SemVersion(mizu.SemVer).Major() &&
-		semver.SemVersion(apiSemVer).Minor() == semver.SemVersion(mizu.SemVer).Minor() {
-		return true, nil
-	}
-
-	logger.Log.Errorf(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", mizu.SemVer, apiSemVer))
-	return false, nil
+	logger.Log.Debug("cli version %s is compatible with api version %s", mizu.Ver, apiVer)
+	return true, nil
 }
 
 func CheckNewerVersion(versionChan chan string) {
@@ -84,24 +81,23 @@ func CheckNewerVersion(versionChan chan string) {
 	gitHubVersion := string(data)
 	gitHubVersion = gitHubVersion[:len(gitHubVersion)-1]
 
-	gitHubVersionSemVer := semver.SemVersion(gitHubVersion)
-	currentSemVer := semver.SemVersion(mizu.SemVer)
-	if !gitHubVersionSemVer.IsValid() || !currentSemVer.IsValid() {
-		logger.Log.Debugf("[ERROR] Semver version is not valid, github version %v, current version %v", gitHubVersion, currentSemVer)
+	greater, err := version.GreaterThen(gitHubVersion, mizu.Ver)
+	if err != nil {
+		logger.Log.Debugf("[ERROR] Ver version is not valid, github version %v, current version %v", gitHubVersion, mizu.Ver)
 		versionChan <- ""
 		return
 	}
 
-	logger.Log.Debugf("Finished version validation, github version %v, current version %v, took %v", gitHubVersion, currentSemVer, time.Since(start))
+	logger.Log.Debugf("Finished version validation, github version %v, current version %v, took %v", gitHubVersion, mizu.Ver, time.Since(start))
 
-	if gitHubVersionSemVer.GreaterThan(currentSemVer) {
+	if greater {
 		var downloadMessage string
 		if runtime.GOOS == "windows" {
 			downloadMessage = fmt.Sprintf("curl -LO %v/mizu.exe", strings.Replace(*latestRelease.HTMLURL, "tag", "download", 1))
 		} else {
 			downloadMessage = fmt.Sprintf("curl -Lo mizu %v/mizu_%s_%s && chmod 755 mizu", strings.Replace(*latestRelease.HTMLURL, "tag", "download", 1), runtime.GOOS, runtime.GOARCH)
 		}
-		versionChan <- fmt.Sprintf("Update available! %v -> %v (%s)", mizu.SemVer, gitHubVersion, downloadMessage)
+		versionChan <- fmt.Sprintf("Update available! %v -> %v (%s)", mizu.Ver, gitHubVersion, downloadMessage)
 	} else {
 		versionChan <- ""
 	}

cli/pkg/version/version.go

@@ -0,0 +1,85 @@
+package version
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+)
+
+type Version struct {
+	Major       int
+	Patch       int
+	Incremental int
+}
+
+func Parse(ver string) (*Version, error) {
+	re := regexp.MustCompile(`^(\d+)\.(\d+)(?:-\w+(\d+))?$`)
+	match := re.FindStringSubmatch(ver)
+	if len(match) != 4 {
+		return nil, fmt.Errorf("invalid format expected <major>.<patch>(-<suffix><incremental>)? %s,", ver)
+	}
+	major, err := strconv.Atoi(match[1])
+	if err != nil {
+		return nil, fmt.Errorf("error parsing major int: %s, err %w", match[1], err)
+	}
+	patch, err := strconv.Atoi(match[2])
+	if err != nil {
+		return nil, fmt.Errorf("error parsing patch int: %s, err %w", match[2], err)
+	}
+
+	if match[3] == "" {
+		return &Version{Major: major, Patch: patch, Incremental: -1}, nil
+	}
+
+	inc, err := strconv.Atoi(match[3])
+	if err != nil {
+		return nil, fmt.Errorf("Error parsing incremental int: %s, err %w", match[3], err)
+	}
+	return &Version{Major: major, Patch: patch, Incremental: inc}, nil
+
+}
+
+func AreEquals(first string, second string) (bool, error) {
+	firstVer, err := Parse(first)
+	if err != nil {
+		return false, fmt.Errorf("Failed parsing fist version: %s, error: %w", first, err)
+	}
+	secondVer, err := Parse(second)
+	if err != nil {
+		return false, fmt.Errorf("Failed parsing second version: %s, error: %w", second, err)
+	}
+
+	return *firstVer == *secondVer, nil
+}
+
+func GreaterThen(first string, second string) (bool, error) {
+	firstVer, err := Parse(first)
+	if err != nil {
+		return false, fmt.Errorf("Failed parsing fist version: %s, error: %w", first, err)
+	}
+	secondVer, err := Parse(second)
+	if err != nil {
+		return false, fmt.Errorf("Failed parsing second version: %s, error: %w", second, err)
+	}
+
+	if firstVer.Major > secondVer.Major {
+		return true, nil
+	} else if firstVer.Major < secondVer.Major {
+		return false, nil
+	}
+
+	if firstVer.Patch > secondVer.Patch {
+		return true, nil
+	} else if firstVer.Patch < secondVer.Patch {
+		return false, nil
+	}
+
+	if firstVer.Incremental == -1 && secondVer.Incremental > -1 {
+		return true, nil
+	}
+
+	if firstVer.Incremental > secondVer.Incremental {
+		return true, nil
+	}
+	return false, nil
+}

cli/pkg/version/version_test.go

@@ -0,0 +1,104 @@
+package version
+
+import (
+	"testing"
+)
+
+func TestEqualsEquality(t *testing.T) {
+	tests := []struct {
+		Name   string
+		First  string
+		Second string
+	}{
+		{Name: "major", First: "1.0", Second: "1.0"},
+		{Name: "patch", First: "1.1", Second: "1.1"},
+		{Name: "incremental", First: "1.0-dev0", Second: "1.0-dev0"},
+	}
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			if equal, _ := AreEquals(test.First, test.Second); !equal {
+				t.Fatalf("Expected %s == %s", test.First, test.Second)
+			}
+		})
+	}
+}
+
+func TestEqualsInvalidVersion(t *testing.T) {
+	tests := []struct {
+		Name   string
+		First  string
+		Second string
+	}{
+		{Name: "first semver", First: "1.0.0", Second: "1.0"},
+		{Name: "second semver", First: "1.1", Second: "1.1.0"},
+		{Name: "incremental invalid", First: "1.0-dev0de", Second: "1.0-dev0"},
+	}
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			if _, err := AreEquals(test.First, test.Second); err == nil {
+				t.Fatalf("Expected error")
+			}
+		})
+	}
+}
+
+func TestEqualsNoEquality(t *testing.T) {
+	tests := []struct {
+		Name   string
+		First  string
+		Second string
+	}{
+		{Name: "major", First: "1.0", Second: "2.0"},
+		{Name: "patch", First: "1.0", Second: "1.1"},
+		{Name: "incremental", First: "1.0-dev2", Second: "1.0-dev3"},
+	}
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			if equal, _ := AreEquals(test.First, test.Second); equal {
+				t.Fatalf("Expected %s != %s", test.First, test.Second)
+			}
+		})
+	}
+}
+
+func TestGreaterThenGreater(t *testing.T) {
+	tests := []struct {
+		Name   string
+		First  string
+		Second string
+	}{
+		{Name: "major", First: "2.0", Second: "1.0"},
+		{Name: "patch", First: "1.1", Second: "1.0"},
+		{Name: "incremental", First: "1.0-dev1", Second: "1.0-dev0"},
+		{Name: "major vs incremental", First: "1.0", Second: "1.0-dev1"},
+	}
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			if greater, _ := GreaterThen(test.First, test.Second); !greater {
+				t.Fatalf("Expected %s > %s", test.First, test.Second)
+			}
+		})
+	}
+}
+
+func TestGreaterThenLessThen(t *testing.T) {
+	tests := []struct {
+		Name   string
+		First  string
+		Second string
+	}{
+		{Name: "major", First: "1.0", Second: "2.0"},
+		{Name: "major equals", First: "1.0", Second: "1.0"},
+		{Name: "patch", First: "1.0", Second: "1.1"},
+		{Name: "patch equals", First: "1.1", Second: "1.1"},
+		{Name: "incremental", First: "1.0-dev0", Second: "1.0-dev1"},
+		{Name: "incremental equals", First: "1.0-dev0", Second: "1.0-dev0"},
+	}
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			if greater, _ := GreaterThen(test.First, test.Second); greater {
+				t.Fatalf("Expected %s < %s", test.First, test.Second)
+			}
+		})
+	}
+}

cli/telemetry/telemetry.go

@@ -83,7 +83,7 @@ func sendTelemetry(argsMap map[string]interface{}) error {
 	argsMap["component"] = "mizu_cli"
 	argsMap["buildTimestamp"] = mizu.BuildTimestamp
 	argsMap["branch"] = mizu.Branch
-	argsMap["version"] = mizu.SemVer
+	argsMap["version"] = mizu.Ver
 	argsMap["platform"] = mizu.Platform
 
 	if machineId, err := machineid.ProtectedID("mizu"); err == nil {

devops/build-agent-ci.sh

@@ -7,9 +7,9 @@ SERVER_NAME=mizu
 GIT_BRANCH=ci
 
 DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
-SEM_VER=${SEM_VER=0.0.0}
+VER=${VER=0.0}
 
-DOCKER_TAGGED_BUILD="$DOCKER_REPO:$SEM_VER"
+DOCKER_TAGGED_BUILD="$DOCKER_REPO:$VER"
 
 echo "building $DOCKER_TAGGED_BUILD"
-docker build -t ${DOCKER_TAGGED_BUILD} --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
+docker build -t ${DOCKER_TAGGED_BUILD} --build-arg VER=${VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .

devops/build-push-featurebranch.sh

@@ -7,9 +7,9 @@ SERVER_NAME=mizu
 GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
 
 DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
-SEM_VER=${SEM_VER=0.0.0}
+VER=${VER=0.0}
 
-DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
+DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$VER")
 
 if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
 then
@@ -19,7 +19,7 @@ fi
 
 echo "building ${DOCKER_TAGGED_BUILDS[@]}"
 DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
-docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
+docker build $DOCKER_TAGS_ARGS --build-arg VER=${VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
 
 for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
 do

shared/Makefile

@@ -0,0 +1,2 @@
+test: ## Run shared tests.
+	@go test ./... -coverpkg=./... -race -coverprofile=coverage.out -covermode=atomic

shared/models.go

@@ -135,7 +135,7 @@ type HealthResponse struct {
 }
 
 type VersionResponse struct {
-	SemVer string `json:"semver"`
+	Ver string `json:"ver"`
 }
 
 type RulesPolicy struct {