OAS: Some minor fixes (#762)

* Commit some fixes

* Accept expected

Co-authored-by: Igor Gov <iggvrv@gmail.com>

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: PR validation
+name: Build
 
 on:
   pull_request:
@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   build-cli:
-    name: Build CLI
+    name: CLI executable build
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.16
@@ -27,35 +27,11 @@ jobs:
         run: make cli
 
   build-agent:
-    name: Build Agent
+    name: Agent docker image build
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.16'
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
-      - shell: bash
-        run: |
-          sudo apt-get install libpcap-dev
-
       - name: Build Agent
-        run: make agent
-
-  build-ui:
-    name: Build UI
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Node 16
-        uses: actions/setup-node@v2
-        with:
-          node-version: '16'
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Build UI
-        run: make ui
+        run: make agent-docker

.github/workflows/release.yml

@@ -32,14 +32,15 @@ jobs:
         id: condval
         with:
           cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "minor"
-          if_false: "patch"
+          if_true: "stable"
+          if_false: "dev"
 
-      - name: Auto increment SemVer action
-        uses: MCKanpolat/auto-semver-action@1.0.5
+      - name: Auto Increment Ver Action
+        uses: docker://igorgov/auto-inc-ver:v2.0.0
         id: versioning
         with:
-          releaseType: ${{ steps.condval.outputs.value }}
+          mode: ${{ steps.condval.outputs.value }}
+          suffix: 'dev'
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Get version parameters
@@ -77,7 +78,7 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             TARGETARCH=${{ matrix.target }}
-            SEM_VER=${{ steps.versioning.outputs.version }}
+            VER=${{ steps.versioning.outputs.version }}
             BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
             GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
             COMMIT_HASH=${{ github.sha }}
@@ -98,25 +99,28 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v2
 
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@master
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
         with:
-          service_account_key: ${{ secrets.GCR_JSON_KEY }}
-          export_default_credentials: true
+          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v0'
 
       - name: Determine versioning strategy
         uses: haya14busa/action-cond@v1
         id: condval
         with:
           cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "minor"
-          if_false: "patch"
+          if_true: "stable"
+          if_false: "dev"
 
-      - name: Auto increment SemVer action
-        uses: MCKanpolat/auto-semver-action@1.0.5
+      - name: Auto Increment Ver Action
+        uses: docker://igorgov/auto-inc-ver:v2.0.0
         id: versioning
         with:
-          releaseType: ${{ steps.condval.outputs.value }}
+          mode: ${{ steps.condval.outputs.value }}
+          suffix: 'dev'
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Get version parameters
@@ -160,7 +164,7 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             TARGETARCH=${{ matrix.target }}
-            SEM_VER=${{ steps.versioning.outputs.version }}
+            VER=${{ steps.versioning.outputs.version }}
             BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
             GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
             COMMIT_HASH=${{ github.sha }}
@@ -175,14 +179,15 @@ jobs:
         id: condval
         with:
           cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "minor"
-          if_false: "patch"
+          if_true: "stable"
+          if_false: "dev"
 
-      - name: Auto increment SemVer action
-        uses: MCKanpolat/auto-semver-action@1.0.5
+      - name: Auto Increment Ver Action
+        uses: docker://igorgov/auto-inc-ver:v2.0.0
         id: versioning
         with:
-          releaseType: ${{ steps.condval.outputs.value }}
+          mode: ${{ steps.condval.outputs.value }}
+          suffix: 'dev'
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Get version parameters
@@ -232,24 +237,28 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v2
 
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@master
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
         with:
-          service_account_key: ${{ secrets.GCR_JSON_KEY }}
-          export_default_credentials: true
+          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
 
-      - uses: haya14busa/action-cond@v1
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v0'
+
+      - name: Determine versioning strategy
+        uses: haya14busa/action-cond@v1
         id: condval
         with:
           cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "minor"
-          if_false: "patch"
+          if_true: "stable"
+          if_false: "dev"
 
-      - name: Auto Increment Semver Action
-        uses: MCKanpolat/auto-semver-action@1.0.5
+      - name: Auto Increment Ver Action
+        uses: docker://igorgov/auto-inc-ver:v2.0.0
         id: versioning
         with:
-          releaseType: ${{ steps.condval.outputs.value }}
+          mode: ${{ steps.condval.outputs.value }}
+          suffix: 'dev'
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Get version parameters
@@ -260,7 +269,7 @@ jobs:
         id: version_parameters
 
       - name: Build and Push CLI
-        run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
+        run: make push-cli VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
 
       - name: Log the version into a .txt file
         shell: bash

.github/workflows/static_code_analysis.yml

@@ -0,0 +1,90 @@
+name: Static code analysis
+
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+
+permissions:
+  contents: read
+
+jobs:
+  golangci:
+    name: Go lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+            go-version: '^1.16'
+
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y libpcap-dev
+
+      - name: Go lint - agent
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: agent
+          args: --timeout=3m
+
+      - name: Go lint - shared
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: shared
+          args: --timeout=3m
+
+      - name: Go lint - tap
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: tap
+          args: --timeout=3m
+
+      - name: Go lint - CLI
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: cli
+          args: --timeout=3m
+
+      - name: Go lint - acceptanceTests
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: acceptanceTests
+          args: --timeout=3m
+
+      - name: Go lint - tap/api
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: tap/api
+
+      - name: Go lint - tap/extensions/amqp
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: tap/extensions/amqp
+
+      - name: Go lint - tap/extensions/http
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: tap/extensions/http
+
+      - name: Go lint - tap/extensions/kafka
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: tap/extensions/kafka
+
+      - name: Go lint - tap/extensions/redis
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          working-directory: tap/extensions/redis

.github/workflows/test.yml

@@ -1,11 +1,11 @@
-name: tests validation
+name: Test
 
 on:
   pull_request:
     branches:
       - 'develop'
       - 'main'
-  push:
+  push: # needed to upload test coverage report to codecov
     branches:
       - 'develop'
       - 'main'
@@ -15,26 +15,8 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run-tests-cli:
-    name: Run CLI tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
-        with:
-          go-version: '^1.16'
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Test
-        run: make test-cli
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2
-
-  run-tests-agent:
-    name: Run Agent tests
+  run-unit-tests:
+    name: Unit Tests
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.16
@@ -49,8 +31,15 @@ jobs:
         run: |
           sudo apt-get install libpcap-dev
 
-      - name: Test
+      - name: CLI Test
+        run: make test-cli
+
+      - name: Agent Test
         run: make test-agent
+      
+      - name: Shared Test
+        run: make test-shared
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
+

Dockerfile

@@ -67,15 +67,15 @@ COPY agent .
 ARG COMMIT_HASH
 ARG GIT_BRANCH
 ARG BUILD_TIMESTAMP
-ARG SEM_VER=0.0.0
+ARG VER=0.0
 
 WORKDIR /app/agent-build
 
 RUN go build -ldflags="-extldflags=-static -s -w \
-    -X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
-    -X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
-    -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
-    -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
+    -X 'github.com/up9inc/mizu/agent/pkg/version.GitCommitHash=${COMMIT_HASH}' \
+    -X 'github.com/up9inc/mizu/agent/pkg/version.Branch=${GIT_BRANCH}' \
+    -X 'github.com/up9inc/mizu/agent/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
+    -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum
 ADD https://github.com/up9inc/basenine/releases/download/v0.4.13/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}

Makefile

@@ -19,7 +19,7 @@ help: ## This help.
 TS_SUFFIX="$(shell date '+%s')"
 GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
 BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
-export SEM_VER?=0.0.0
+export VER?=0.0
 
 ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
@@ -47,6 +47,10 @@ agent-debug: ## Build agent for debug.
 docker: ## Build and publish agent docker image.
 	$(MAKE) push-docker
 
+agent-docker: ## Build agent docker image.
+	@echo "Building agent docker image"
+	@docker build -t up9inc/mizu:devlatest .
+
 push: push-docker push-cli ## Build and publish agent docker image & CLI.
 
 push-docker: ## Build and publish agent docker image.
@@ -85,5 +89,8 @@ test-cli:
 test-agent:
 	@echo "running agent tests"; cd agent && $(MAKE) test
 
+test-shared:
+	@echo "running shared tests"; cd shared && $(MAKE) test
+
 acceptance-test:
 	@echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test

README.md

@@ -1,11 +1,17 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
 
 <p align="center">
+    <a href="https://github.com/up9inc/mizu/blob/main/LICENSE">
+        <img alt="GitHub License" src="https://img.shields.io/github/license/up9inc/mizu?logo=GitHub&style=flat-square">
+    </a>
     <a href="https://github.com/up9inc/mizu/releases/latest">
         <img alt="GitHub Latest Release" src="https://img.shields.io/github/v/release/up9inc/mizu?logo=GitHub&style=flat-square">
     </a>
-    <a href="https://github.com/up9inc/mizu/blob/main/LICENSE">
-        <img alt="GitHub License" src="https://img.shields.io/github/license/up9inc/mizu?logo=GitHub&style=flat-square">
+    <a href="https://hub.docker.com/r/up9inc/mizu">
+      <img alt="Docker pulls" src="https://img.shields.io/docker/pulls/up9inc/mizu?color=%23099cec">
+    </a>
+    <a href="https://hub.docker.com/r/up9inc/mizu">
+      <img alt="Image size" src="https://img.shields.io/docker/image-size/up9inc/mizu/latest">
     </a>
     <a href="https://join.slack.com/t/up9/shared_invite/zt-tfjnduli-QzlR8VV4Z1w3YnPIAJfhlQ">
       <img alt="Slack" src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social">

acceptanceTests/cypress.json

@@ -4,7 +4,7 @@
   "viewportHeight": 1080,
   "video": false,
   "screenshotOnRunFailure": false,
-  "defaultCommandTimeout": 5000,
+  "defaultCommandTimeout": 6000,
   "testFiles": [
     "tests/GuiPort.js",
     "tests/MultipleNamespaces.js",
@@ -20,6 +20,7 @@
     "testUrl": "http://localhost:8899/",
     "redactHeaderContent": "User-Header[REDACTED]",
     "redactBodyContent": "{ \"User\": \"[REDACTED]\" }",
-    "regexMaskingBodyContent": "[REDACTED]"
+    "regexMaskingBodyContent": "[REDACTED]",
+    "minimumEntries": 25
   }
 }

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -16,3 +16,15 @@ export function resizeToNormalMizu() {
     cy.viewport(1920, 1080);
 }
 
+export function verifyMinimumEntries() {
+    const minimumEntries = Cypress.env('minimumEntries');
+    it(`Making sure that mizu shows at least ${minimumEntries} entries`, async function () {
+        cy.get('#total-entries').then(number => {
+            const getNum = () => {
+                const numOfEntries = number.text();
+                return parseInt(numOfEntries);
+            };
+            cy.wrap({ there: getNum }).invoke('there').should('be.gte', minimumEntries);
+        });
+    });
+}

acceptanceTests/cypress/integration/tests/GuiPort.js

@@ -1,8 +1,16 @@
-it('check', function () {
-    cy.visit(`http://localhost:${Cypress.env('port')}/`);
+import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
+import {verifyMinimumEntries} from "../testHelpers/TrafficHelper";
 
-    cy.get('.header').should('be.visible');
-    cy.get('.TrafficPageHeader').should('be.visible');
-    cy.get('.TrafficPage-ListContainer').should('be.visible');
-    cy.get('.TrafficPage-Container').should('be.visible');
+it('check', function () {
+    const podName = Cypress.env('name'), namespace = Cypress.env('namespace');
+    const port = Cypress.env('port');
+    cy.intercept('GET', `http://localhost:${port}/status/tap`).as('statusTap');
+
+    cy.visit(`http://localhost:${port}`);
+    cy.wait('@statusTap').its('response.statusCode').should('match', /^2\d{2}/);
+
+    verifyMinimumEntries();
+
+    cy.get('.podsCount').trigger('mouseover');
+    findLineAndCheck(getExpectedDetailsDict(podName, namespace));
 });

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -1,24 +1,24 @@
-import {isValueExistsInElement, resizeToHugeMizu} from "../testHelpers/TrafficHelper";
+import {isValueExistsInElement, resizeToHugeMizu, verifyMinimumEntries} from "../testHelpers/TrafficHelper";
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
 });
 
-it('going through each entry', function () {
-    resizeToHugeMizu();
-    cy.get('#total-entries').then(number => {
-        const getNum = () => {
-            const numOfEntries = number.text();
-            return parseInt(numOfEntries);
-        };
-        cy.wrap({ there: getNum }).invoke('there').should('be.gte', 25);
+verifyMinimumEntries();
 
+checkEntries();
+
+function checkEntries() {
+    it('checking all entries', function () {
         checkThatAllEntriesShown();
+        resizeToHugeMizu();
 
-        const entriesNum = getNum();
-        [...Array(entriesNum).keys()].map(checkEntry);
+        cy.get('#total-entries').then(number => {
+            const numOfEntries = parseInt(number.text());
+            [...Array(numOfEntries).keys()].map(checkEntry);
+        });
     });
-});
+}
 
 function checkThatAllEntriesShown() {
     cy.get('#entries-length').then(number => {

acceptanceTests/cypress/integration/tests/NoRedact.js

@@ -1,8 +1,10 @@
-import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
+import {isValueExistsInElement, verifyMinimumEntries} from '../testHelpers/TrafficHelper';
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
-})
+});
+
+verifyMinimumEntries();
 
 isValueExistsInElement(false, Cypress.env('redactHeaderContent'), '#tbody-Headers');
 isValueExistsInElement(false, Cypress.env('redactBodyContent'), '.hljs');

acceptanceTests/cypress/integration/tests/Redact.js

@@ -1,8 +1,10 @@
-import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
+import {isValueExistsInElement, verifyMinimumEntries} from '../testHelpers/TrafficHelper';
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
-})
+});
+
+verifyMinimumEntries();
 
 isValueExistsInElement(true, Cypress.env('redactHeaderContent'), '#tbody-Headers');
 isValueExistsInElement(true, Cypress.env('redactBodyContent'), '.hljs');

acceptanceTests/cypress/integration/tests/RegexMasking.js

@@ -1,7 +1,9 @@
-import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
+import {isValueExistsInElement, verifyMinimumEntries} from "../testHelpers/TrafficHelper";
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
-})
+});
+
+verifyMinimumEntries();
 
 isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), '.hljs');

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -1,13 +1,16 @@
 import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
-import {resizeToHugeMizu, resizeToNormalMizu} from "../testHelpers/TrafficHelper";
+import {resizeToHugeMizu, resizeToNormalMizu, verifyMinimumEntries} from "../testHelpers/TrafficHelper";
 const greenFilterColor = 'rgb(210, 250, 210)';
 const redFilterColor = 'rgb(250, 214, 220)';
 const refreshWaitTimeout = 10000;
+const bodyJsonClass = '.hljs';
 
 it('opening mizu', function () {
     cy.visit(Cypress.env('testUrl'));
 });
 
+verifyMinimumEntries();
+
 it('top bar check', function () {
     const podName1 = 'httpbin', namespace1 = 'mizu-tests';
     const podName2 = 'httpbin2', namespace2 = 'mizu-tests';
@@ -25,6 +28,37 @@ it('filtering guide check', function () {
     cy.get('#modal-modal-title').should('not.exist');
 });
 
+it('right side sanity test', function () {
+    cy.get('#entryDetailedTitleBodySize').then(sizeTopLine => {
+        const sizeOnTopLine = sizeTopLine.text().replace(' B', '');
+        cy.contains('Response').click();
+        cy.contains('Body Size (bytes)').parent().next().then(size => {
+            const bodySizeByDetails = size.text();
+            expect(sizeOnTopLine).to.equal(bodySizeByDetails, 'The body size in the top line should match the details in the response');
+
+            if (parseInt(bodySizeByDetails) < 0) {
+                throw new Error(`The body size cannot be negative. got the size: ${bodySizeByDetails}`)
+            }
+
+            cy.get('#entryDetailedTitleElapsedTime').then(timeInMs => {
+                const time = timeInMs.text();
+                if (time < '0ms') {
+                    throw new Error(`The time in the top line cannot be negative ${time}`);
+                }
+
+                cy.get('#rightSideContainer [title="Status Code"]').then(status => {
+                    const statusCode = status.text();
+                    cy.contains('Status').parent().next().then(statusInDetails => {
+                        const statusCodeInDetails = statusInDetails.text();
+
+                        expect(statusCode).to.equal(statusCodeInDetails, 'The status code in the top line should match the status code in details');
+                    });
+                });
+            });
+        });
+    });
+});
+
 checkIllegalFilter('invalid filter');
 
 checkFilter({
@@ -161,11 +195,12 @@ function checkFilter(filterDetails){
             if (!applyByEnter)
                 cy.get('[type="submit"]').click();
 
-            // only one entry in DOM after filtering, checking all four checks on it
+            // only one entry in DOM after filtering, checking all checks on it
             leftTextCheck(totalEntries - 1, leftSidePath, leftSideExpectedText);
             leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
             rightTextCheck(rightSidePath, rightSideExpectedText);
             rightOnHoverCheck(rightSidePath, name);
+            checkRightSideResponseBody();
 
             cy.get('[title="Fetch old records"]').click();
             resizeToHugeMizu();
@@ -216,3 +251,86 @@ function rightOnHoverCheck(path, expectedText) {
     cy.get(`.TrafficPage-Container > :nth-child(2) ${path}`).trigger('mouseover');
     cy.get(`.TrafficPage-Container > :nth-child(2) .Queryable-Tooltip`).should('have.text', expectedText);
 }
+
+
+function checkRightSideResponseBody() {
+    cy.contains('Response').click();
+    clickCheckbox('Decode Base64');
+
+    cy.get(`${bodyJsonClass}`).then(value => {
+        const encodedBody = value.text();
+        const decodedBody = atob(encodedBody);
+        const responseBody = JSON.parse(decodedBody);
+
+        const expectdJsonBody = {
+            args: RegExp({}),
+            url: RegExp('http://.*/get'),
+            headers: {
+                "User-Agent": RegExp('[REDACTED]'),
+                "Accept-Encoding": RegExp('gzip'),
+                "X-Forwarded-Uri": RegExp('/api/v1/namespaces/.*/services/.*/proxy/get')
+            }
+        };
+
+        expect(responseBody.args).to.match(expectdJsonBody.args);
+        expect(responseBody.url).to.match(expectdJsonBody.url);
+        expect(responseBody.headers['User-Agent']).to.match(expectdJsonBody.headers['User-Agent']);
+        expect(responseBody.headers['Accept-Encoding']).to.match(expectdJsonBody.headers['Accept-Encoding']);
+        expect(responseBody.headers['X-Forwarded-Uri']).to.match(expectdJsonBody.headers['X-Forwarded-Uri']);
+
+        cy.get(`${bodyJsonClass}`).should('have.text', encodedBody);
+        clickCheckbox('Decode Base64');
+
+        cy.get(`${bodyJsonClass} > `).its('length').should('be.gt', 1).then(linesNum => {
+            cy.get(`${bodyJsonClass} > >`).its('length').should('be.gt', linesNum).then(jsonItemsNum => {
+                checkPrettyAndLineNums(jsonItemsNum, decodedBody);
+
+                clickCheckbox('Line numbers');
+                checkPrettyOrNothing(jsonItemsNum, decodedBody);
+
+                clickCheckbox('Pretty');
+                checkPrettyOrNothing(jsonItemsNum, decodedBody);
+
+                clickCheckbox('Line numbers');
+                checkOnlyLineNumberes(jsonItemsNum, decodedBody);
+            });
+        });
+    });
+}
+
+function clickCheckbox(type) {
+    cy.contains(`${type}`).prev().children().click();
+}
+
+function checkPrettyAndLineNums(jsonItemsLen, decodedBody) {
+    decodedBody = decodedBody.replaceAll(' ', '');
+    cy.get(`${bodyJsonClass} >`).then(elements => {
+        const lines = Object.values(elements);
+        lines.forEach((line, index) => {
+            if (line.getAttribute) {
+                const cleanLine = getCleanLine(line);
+                const currentLineFromDecodedText = decodedBody.substring(0, cleanLine.length);
+
+                expect(cleanLine).to.equal(currentLineFromDecodedText, `expected the text in line number ${index + 1} to match the text that generated by the base64 decoding`)
+
+                decodedBody = decodedBody.substring(cleanLine.length);
+            }
+        });
+    });
+}
+
+function getCleanLine(lineElement) {
+    return (lineElement.innerText.substring(0, lineElement.innerText.length - 1)).replaceAll(' ', '');
+}
+
+function checkPrettyOrNothing(jsonItems, decodedBody) {
+    cy.get(`${bodyJsonClass} > `).should('have.length', jsonItems).then(text => {
+        const json = text.text();
+        expect(json).to.equal(decodedBody);
+    });
+}
+
+function checkOnlyLineNumberes(jsonItems, decodedText) {
+    cy.get(`${bodyJsonClass} >`).should('have.length', 1).and('have.text', decodedText);
+    cy.get(`${bodyJsonClass} > >`).should('have.length', jsonItems)
+}

acceptanceTests/tap_test.go

@@ -110,7 +110,16 @@ func TestTapGuiPort(t *testing.T) {
 				return
 			}
 
-			runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env port=%d", guiPort))
+			proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+			for i := 0; i < defaultEntriesCount; i++ {
+				if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+					t.Errorf("failed to send proxy request, err: %v", requestErr)
+					return
+				}
+			}
+
+			runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env name=%v,namespace=%v,port=%d",
+				"httpbin", "mizu-tests", guiPort))
 		})
 	}
 }

acceptanceTests/testsUtils.go

@@ -3,7 +3,6 @@ package acceptanceTests
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -11,12 +10,10 @@ import (
 	"os/exec"
 	"path"
 	"strings"
-	"sync"
 	"syscall"
 	"testing"
 	"time"
 
-	"github.com/gorilla/websocket"
 	"github.com/up9inc/mizu/shared"
 )
 
@@ -28,7 +25,6 @@ const (
 	defaultServiceName    = "httpbin"
 	defaultEntriesCount   = 50
 	waitAfterTapPodsReady = 3 * time.Second
-	cleanCommandTimeout   = 1 * time.Minute
 )
 
 type PodDescriptor struct {
@@ -36,18 +32,6 @@ type PodDescriptor struct {
 	Namespace string
 }
 
-func isPodDescriptorInPodArray(pods []map[string]interface{}, podDescriptor PodDescriptor) bool {
-	for _, pod := range pods {
-		podNamespace := pod["namespace"].(string)
-		podName := pod["name"].(string)
-
-		if podDescriptor.Namespace == podNamespace && strings.Contains(podName, podDescriptor.Name) {
-			return true
-		}
-	}
-	return false
-}
-
 func getCliPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {
@@ -84,14 +68,10 @@ func getApiServerUrl(port uint16) string {
 	return fmt.Sprintf("http://localhost:%v", port)
 }
 
-func getWebSocketUrl(port uint16) string {
-	return fmt.Sprintf("ws://localhost:%v/ws", port)
-}
-
 func getDefaultCommandArgs() []string {
 	setFlag := "--set"
 	telemetry := "telemetry=false"
-	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
+	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0"
 	imagePullPolicy := "image-pull-policy=IfNotPresent"
 	headless := "headless=true"
 
@@ -130,20 +110,6 @@ func getDefaultConfigCommandArgs() []string {
 	return append([]string{configCommand}, defaultCmdArgs...)
 }
 
-func getDefaultCleanCommandArgs() []string {
-	cleanCommand := "clean"
-	defaultCmdArgs := getDefaultCommandArgs()
-
-	return append([]string{cleanCommand}, defaultCmdArgs...)
-}
-
-func getDefaultViewCommandArgs() []string {
-	viewCommand := "view"
-	defaultCmdArgs := getDefaultCommandArgs()
-
-	return append([]string{viewCommand}, defaultCmdArgs...)
-}
-
 func runCypressTests(t *testing.T, cypressRunCmd string) {
 	cypressCmd := exec.Command("bash", "-c", cypressRunCmd)
 	t.Logf("running command: %v", cypressCmd.String())
@@ -268,36 +234,6 @@ func executeHttpPostRequestWithHeaders(url string, headers map[string]string, bo
 	return executeHttpRequest(response, requestErr)
 }
 
-func runMizuClean() error {
-	cliPath, err := getCliPath()
-	if err != nil {
-		return err
-	}
-
-	cleanCmdArgs := getDefaultCleanCommandArgs()
-
-	cleanCmd := exec.Command(cliPath, cleanCmdArgs...)
-
-	commandDone := make(chan error)
-	go func() {
-		if err := cleanCmd.Run(); err != nil {
-			commandDone <- err
-		}
-		commandDone <- nil
-	}()
-
-	select {
-	case err = <-commandDone:
-		if err != nil {
-			return err
-		}
-	case <-time.After(cleanCommandTimeout):
-		return errors.New("clean command timed out")
-	}
-
-	return nil
-}
-
 func cleanupCommand(cmd *exec.Cmd) error {
 	if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
 		return err
@@ -310,17 +246,6 @@ func cleanupCommand(cmd *exec.Cmd) error {
 	return nil
 }
 
-func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) {
-	tapPodsInterface := tapStatusInterface.([]interface{})
-
-	var pods []map[string]interface{}
-	for _, podInterface := range tapPodsInterface {
-		pods = append(pods, podInterface.(map[string]interface{}))
-	}
-
-	return pods, nil
-}
-
 func getLogsPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {
@@ -331,77 +256,6 @@ func getLogsPath() (string, error) {
 	return logsPath, nil
 }
 
-// waitTimeout waits for the waitgroup for the specified max timeout.
-// Returns true if waiting timed out.
-func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
-	channel := make(chan struct{})
-	go func() {
-		defer close(channel)
-		wg.Wait()
-	}()
-	select {
-	case <-channel:
-		return false // completed normally
-	case <-time.After(timeout):
-		return true // timed out
-	}
-}
-
-// checkEntriesAtLeast checks whether the number of entries greater than or equal to n
-func checkEntriesAtLeast(entries []map[string]interface{}, n int) error {
-	if len(entries) < n {
-		return fmt.Errorf("Unexpected entries result - Expected more than %d entries", n-1)
-	}
-	return nil
-}
-
-// getDBEntries retrieves the entries from the database before the given timestamp.
-// Also limits the results according to the limit parameter.
-// Timeout for the WebSocket connection is defined by the timeout parameter.
-func getDBEntries(timestamp int64, limit int, timeout time.Duration) (entries []map[string]interface{}, err error) {
-	query := fmt.Sprintf("timestamp < %d and limit(%d)", timestamp, limit)
-	webSocketUrl := getWebSocketUrl(defaultApiServerPort)
-
-	var connection *websocket.Conn
-	connection, _, err = websocket.DefaultDialer.Dial(webSocketUrl, nil)
-	if err != nil {
-		return
-	}
-	defer connection.Close()
-
-	handleWSConnection := func(wg *sync.WaitGroup) {
-		defer wg.Done()
-		for {
-			_, message, err := connection.ReadMessage()
-			if err != nil {
-				return
-			}
-
-			var data map[string]interface{}
-			if err = json.Unmarshal([]byte(message), &data); err != nil {
-				return
-			}
-
-			if data["messageType"] == "entry" {
-				entries = append(entries, data)
-			}
-		}
-	}
-
-	err = connection.WriteMessage(websocket.TextMessage, []byte(query))
-	if err != nil {
-		return
-	}
-
-	var wg sync.WaitGroup
-	go handleWSConnection(&wg)
-	wg.Add(1)
-
-	waitTimeout(&wg, timeout)
-
-	return
-}
-
 func Contains(slice []string, containsValue string) bool {
 	for _, sliceValue := range slice {
 		if sliceValue == containsValue {

agent/go.mod

@@ -1,20 +1,21 @@
-module mizuserver
+module github.com/up9inc/mizu/agent
 
 go 1.16
 
 require (
 	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
-	github.com/chanced/openapi v0.0.6
+	github.com/chanced/openapi v0.0.7
 	github.com/djherbis/atime v1.0.0
+	github.com/elastic/go-elasticsearch/v7 v7.16.0
 	github.com/getkin/kin-openapi v0.76.0
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.7.7
 	github.com/go-playground/locales v0.13.0
 	github.com/go-playground/universal-translator v0.17.0
 	github.com/go-playground/validator/v10 v10.5.0
-	github.com/google/martian v2.1.0+incompatible
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/websocket v1.4.2
+	github.com/nav-inc/datetime v0.1.3
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/ory/keto-client-go v0.7.0-alpha.1
@@ -29,6 +30,7 @@ require (
 	github.com/up9inc/mizu/tap/extensions/http v0.0.0
 	github.com/up9inc/mizu/tap/extensions/kafka v0.0.0
 	github.com/up9inc/mizu/tap/extensions/redis v0.0.0
+	github.com/wI2L/jsondiff v0.1.1
 	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
 	k8s.io/api v0.21.2
 	k8s.io/apimachinery v0.21.2

agent/go.sum

@@ -95,8 +95,8 @@ github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a h1:zG6t+4krPXcCKtL
 github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a/go.mod h1:yhcmlFk1hxuZ+5XZbupzT/cEm/eE4ZvWbmsW1+Q/aZE=
 github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44 h1:4NOJMtvZaOA6cI2gkIuXk/2b5KTOvm/R4zyPy/yLCM4=
 github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44/go.mod h1:XVNfXN5kgZST4PQ0W/oBAHJku2OteCeHxjAbvfd0ARM=
-github.com/chanced/openapi v0.0.6 h1:2giGS47+T8/7MN2hfRHSIUPx86Qksk+J/ciIWcAM7hY=
-github.com/chanced/openapi v0.0.6/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU=
+github.com/chanced/openapi v0.0.7 h1:OmOBHCg/5ViUg0gaGxXBeEFoVBE8C2pHK4BO/AiD6k8=
+github.com/chanced/openapi v0.0.7/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -126,6 +126,8 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/elastic/go-elasticsearch/v7 v7.16.0 h1:GHsxDFXIAlhSleXun4kwA89P7kQFADRChqvgOPeYP5A=
+github.com/elastic/go-elasticsearch/v7 v7.16.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -513,6 +515,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/nav-inc/datetime v0.1.3 h1:PaybPUsScX+Cd3TEa1tYpfwU61deCEhMTlCO2hONm1c=
+github.com/nav-inc/datetime v0.1.3/go.mod h1:gKGf5G+cW7qkTo5TC/sieNyz6lYdrA9cf1PNV+pXIOE=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/ohler55/ojg v1.12.12 h1:hepbQFn7GHAecTPmwS3j5dCiOLsOpzPLvhiqnlAVAoE=
@@ -610,6 +614,7 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

agent/keto/build-push-featurebranch.sh

@@ -7,9 +7,9 @@ SERVER_NAME=mizu-keto
 GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
 
 DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
-SEM_VER=${SEM_VER=0.0.0}
+VER=${VER=0.0}
 
-DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
+DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$VER")
 
 if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
 then
@@ -19,7 +19,7 @@ fi
 
 echo "building ${DOCKER_TAGGED_BUILDS[@]}"
 DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
-docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
+docker build $DOCKER_TAGS_ARGS --build-arg VER=${VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
 
 for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
 do

agent/kratos/build-push-featurebranch.sh

@@ -7,9 +7,9 @@ SERVER_NAME=mizu-kratos
 GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
 
 DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
-SEM_VER=${SEM_VER=0.0.0}
+VER=${VER=0.0.0}
 
-DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER")
+DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$VER")
 
 if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
 then
@@ -19,7 +19,7 @@ fi
 
 echo "building ${DOCKER_TAGGED_BUILDS[@]}"
 DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
-docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
+docker build $DOCKER_TAGS_ARGS --build-arg VER=${VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
 
 for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
 do

agent/main.go

@@ -6,16 +6,6 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
-	"mizuserver/pkg/api"
-	"mizuserver/pkg/config"
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/middlewares"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/oas"
-	"mizuserver/pkg/routes"
-	"mizuserver/pkg/servicemap"
-	"mizuserver/pkg/up9"
-	"mizuserver/pkg/utils"
 	"net/http"
 	"os"
 	"os/signal"
@@ -25,6 +15,21 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/oas"
+	"github.com/up9inc/mizu/agent/pkg/routes"
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
+	"github.com/up9inc/mizu/agent/pkg/up9"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+
+	"github.com/up9inc/mizu/agent/pkg/elastic"
+
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+
+	"github.com/up9inc/mizu/agent/pkg/api"
+	"github.com/up9inc/mizu/agent/pkg/config"
+
 	v1 "k8s.io/api/core/v1"
 
 	"github.com/antelman107/net-wait-go/wait"
@@ -61,7 +66,6 @@ const (
 	socketConnectionRetries    = 30
 	socketConnectionRetryDelay = time.Second * 2
 	socketHandshakeTimeout     = time.Second * 2
-	uiIndexPath                = "./site/index.html"
 )
 
 func main() {
@@ -159,6 +163,7 @@ func enableExpFeatureIfNeeded() {
 	if config.Config.ServiceMap {
 		servicemap.GetInstance().SetConfig(config.Config)
 	}
+	elastic.GetInstance().Configure(config.Config.Elastic)
 }
 
 func configureBasenineServer(host string, port string) {
@@ -167,7 +172,7 @@ func configureBasenineServer(host string, port string) {
 		wait.WithWait(200*time.Millisecond),
 		wait.WithBreak(50*time.Millisecond),
 		wait.WithDeadline(5*time.Second),
-		wait.WithDebug(true),
+		wait.WithDebug(config.Config.LogLevel == logging.DEBUG),
 	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
 		logger.Log.Panicf("Basenine is not available!")
 	}
@@ -246,16 +251,23 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 
 	app.Use(DisableRootStaticCache())
 
-	if err := setUIFlags(); err != nil {
-		logger.Log.Errorf("Error setting ui mode, err: %v", err)
+	var staticFolder string
+	if config.Config.StandaloneMode {
+		staticFolder = "./site-standalone"
+	} else {
+		staticFolder = "./site"
 	}
 
-	if config.Config.StandaloneMode {
-		app.Use(static.ServeRoot("/", "./site-standalone"))
-	} else {
-		app.Use(static.ServeRoot("/", "./site"))
+	indexStaticFile := staticFolder + "/index.html"
+	if err := setUIFlags(indexStaticFile); err != nil {
+		logger.Log.Errorf("Error setting ui flags, err: %v", err)
 	}
 
+	app.Use(static.ServeRoot("/", staticFolder))
+	app.NoRoute(func(c *gin.Context) {
+		c.File(indexStaticFile)
+	})
+
 	app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
 
 	api.WebSocketRoutes(app, &eventHandlers, startTime)
@@ -276,7 +288,6 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 	routes.EntriesRoutes(app)
 	routes.MetadataRoutes(app)
 	routes.StatusRoutes(app)
-	routes.NotFoundRoute(app)
 	utils.StartServer(app)
 }
 
@@ -291,7 +302,7 @@ func DisableRootStaticCache() gin.HandlerFunc {
 	}
 }
 
-func setUIFlags() error {
+func setUIFlags(uiIndexPath string) error {
 	read, err := ioutil.ReadFile(uiIndexPath)
 	if err != nil {
 		return err

agent/pkg/api/main.go

@@ -5,24 +5,27 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/har"
-	"mizuserver/pkg/holder"
-	"mizuserver/pkg/providers"
 	"os"
 	"path"
 	"sort"
 	"strings"
 	"time"
 
-	"mizuserver/pkg/servicemap"
+	"github.com/up9inc/mizu/agent/pkg/elastic"
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/agent/pkg/holder"
+	"github.com/up9inc/mizu/agent/pkg/providers"
+
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
+
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/oas"
+	"github.com/up9inc/mizu/agent/pkg/resolver"
+	"github.com/up9inc/mizu/agent/pkg/utils"
 
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/oas"
-	"mizuserver/pkg/resolver"
-	"mizuserver/pkg/utils"
 
 	basenine "github.com/up9inc/basenine/client/go"
 )
@@ -40,10 +43,8 @@ func StartResolving(namespace string) {
 	res.Start(ctx)
 	go func() {
 		for {
-			select {
-			case err := <-errOut:
-				logger.Log.Infof("name resolving error %s", err)
-			}
+			err := <-errOut
+			logger.Log.Infof("name resolving error %s", err)
 		}
 	}()
 
@@ -65,7 +66,7 @@ func startReadingFiles(workingDir string) {
 		return
 	}
 
-	for true {
+	for {
 		dir, _ := os.Open(workingDir)
 		dirFiles, _ := dir.Readdir(-1)
 
@@ -122,7 +123,9 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
 				var httpPair tapApi.HTTPRequestResponsePair
-				json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair)
+				if err := json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair); err != nil {
+					logger.Log.Error(err)
+				}
 
 				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
 				mizuEntry.ContractStatus = contract.Status
@@ -137,7 +140,8 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 				mizuEntry.Rules = rules
 			}
 
-			oas.GetOasGeneratorInstance().PushEntry(harEntry)
+			entryWSource := oas.EntryWithSource{Entry: *harEntry, Source: mizuEntry.Source.Name, Id: mizuEntry.Id}
+			oas.GetOasGeneratorInstance().PushEntry(&entryWSource)
 		}
 
 		data, err := json.Marshal(mizuEntry)
@@ -150,6 +154,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		connection.SendText(string(data))
 
 		servicemap.GetInstance().NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
+		elastic.GetInstance().PushEntry(mizuEntry)
 	}
 }
 

agent/pkg/api/socket_routes.go

@@ -2,18 +2,17 @@ package api
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
-	"mizuserver/pkg/models"
 	"net/http"
 	"sync"
 	"time"
 
+	"github.com/up9inc/mizu/agent/pkg/models"
+
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
@@ -42,7 +41,7 @@ var connectedWebsocketIdCounter = 0
 
 func init() {
 	websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
-	connectedWebsockets = make(map[int]*SocketConnection, 0)
+	connectedWebsockets = make(map[int]*SocketConnection)
 }
 
 func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
@@ -94,7 +93,10 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 	eventHandlers.WebSocketConnect(socketId, isTapper)
 
 	startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime)
-	SendToSocket(socketId, startTimeBytes)
+
+	if err = SendToSocket(socketId, startTimeBytes); err != nil {
+		logger.Log.Error(err)
+	}
 
 	for {
 		_, msg, err := ws.ReadMessage()
@@ -117,7 +119,9 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 					AutoClose: 5000,
 					Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
 				})
-				SendToSocket(socketId, toastBytes)
+				if err := SendToSocket(socketId, toastBytes); err != nil {
+					logger.Log.Error(err)
+				}
 				break
 			}
 
@@ -137,7 +141,9 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 					base := tapApi.Summarize(entry)
 
 					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
-					SendToSocket(socketId, baseEntryBytes)
+					if err := SendToSocket(socketId, baseEntryBytes); err != nil {
+						logger.Log.Error(err)
+					}
 				}
 			}
 
@@ -156,7 +162,9 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 					}
 
 					metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
-					SendToSocket(socketId, metadataBytes)
+					if err := SendToSocket(socketId, metadataBytes); err != nil {
+						logger.Log.Error(err)
+					}
 				}
 			}
 
@@ -183,14 +191,10 @@ func socketCleanup(socketId int, socketConnection *SocketConnection) {
 	socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
 }
 
-var db = debounce.NewDebouncer(time.Second*5, func() {
-	logger.Log.Error("Successfully sent to socket")
-})
-
 func SendToSocket(socketId int, message []byte) error {
 	socketObj := connectedWebsockets[socketId]
 	if socketObj == nil {
-		return errors.New("Socket is disconnected")
+		return fmt.Errorf("Socket %v is disconnected", socketId)
 	}
 
 	var sent = false
@@ -204,7 +208,10 @@ func SendToSocket(socketId int, message []byte) error {
 	socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
 	err := socketObj.connection.WriteMessage(1, message)
 	socketObj.lock.Unlock()
-
 	sent = true
-	return err
+
+	if err != nil {
+		return fmt.Errorf("Failed to write message to socket %v, err: %w", socketId, err)
+	}
+	return nil
 }

agent/pkg/api/socket_server_handlers.go

@@ -3,12 +3,13 @@ package api
 import (
 	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/providers"
-	"mizuserver/pkg/providers/tappers"
-	"mizuserver/pkg/up9"
 	"sync"
 
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
+	"github.com/up9inc/mizu/agent/pkg/up9"
+
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	"github.com/up9inc/mizu/shared"
@@ -54,9 +55,8 @@ func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 func BroadcastToBrowserClients(message []byte) {
 	for _, socketId := range browserClientSocketUUIDs {
 		go func(socketId int) {
-			err := SendToSocket(socketId, message)
-			if err != nil {
-				logger.Log.Errorf("error sending message to socket ID %d: %v", socketId, err)
+			if err := SendToSocket(socketId, message); err != nil {
+				logger.Log.Error(err)
 			}
 		}(socketId)
 	}

agent/pkg/controllers/config_controller.go

@@ -2,21 +2,22 @@ package controllers
 
 import (
 	"context"
+	"net/http"
+	"regexp"
+	"time"
+
 	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/config"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/providers/tapConfig"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/kubernetes"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 	v1 "k8s.io/api/core/v1"
-	"mizuserver/pkg/config"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/providers"
-	"mizuserver/pkg/providers/tapConfig"
-	"mizuserver/pkg/providers/tappedPods"
-	"mizuserver/pkg/providers/tappers"
-	"net/http"
-	"regexp"
-	"time"
 )
 
 var cancelTapperSyncer context.CancelFunc

agent/pkg/controllers/entries_controller.go

@@ -2,13 +2,14 @@ package controllers
 
 import (
 	"encoding/json"
-	"mizuserver/pkg/har"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/validation"
 	"net/http"
 	"strconv"
 	"time"
 
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/validation"
+
 	"github.com/gin-gonic/gin"
 
 	basenine "github.com/up9inc/basenine/client/go"
@@ -26,7 +27,7 @@ func InitExtensionsMap(ref map[string]*tapApi.Extension) {
 func Error(c *gin.Context, err error) bool {
 	if err != nil {
 		logger.Log.Errorf("Error getting entry: %v", err)
-		c.Error(err)
+		_ = c.Error(err)
 		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
 			"error":     true,
 			"type":      "error",
@@ -131,7 +132,9 @@ func GetEntry(c *gin.Context) {
 		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
 		isRulesEnabled = _isRulesEnabled
 		inrec, _ := json.Marshal(rulesMatched)
-		json.Unmarshal(inrec, &rules)
+		if err := json.Unmarshal(inrec, &rules); err != nil {
+			logger.Log.Error(err)
+		}
 	}
 
 	c.JSON(http.StatusOK, tapApi.EntryWrapper{

agent/pkg/controllers/install_controller.go

@@ -1,9 +1,10 @@
 package controllers
 
 import (
-	"mizuserver/pkg/providers"
 	"net/http"
 
+	"github.com/up9inc/mizu/agent/pkg/providers"
+
 	"github.com/gin-gonic/gin"
 	"github.com/up9inc/mizu/shared/logger"
 )

agent/pkg/controllers/metadata_controller.go

@@ -1,13 +1,14 @@
 package controllers
 
 import (
-	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/shared"
-	"mizuserver/pkg/version"
 	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/version"
+	"github.com/up9inc/mizu/shared"
 )
 
 func GetVersion(c *gin.Context) {
-	resp := shared.VersionResponse{SemVer: version.SemVer}
+	resp := shared.VersionResponse{Ver: version.Ver}
 	c.JSON(http.StatusOK, resp)
 }

agent/pkg/controllers/oas_controller.go

@@ -1,11 +1,12 @@
 package controllers
 
 import (
+	"net/http"
+
 	"github.com/chanced/openapi"
 	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/oas"
 	"github.com/up9inc/mizu/shared/logger"
-	"mizuserver/pkg/oas"
-	"net/http"
 )
 
 func GetOASServers(c *gin.Context) {

agent/pkg/controllers/oas_controller_test.go

@@ -1,10 +1,12 @@
 package controllers
 
 import (
-	"github.com/gin-gonic/gin"
-	"mizuserver/pkg/oas"
 	"net/http/httptest"
 	"testing"
+
+	"github.com/up9inc/mizu/agent/pkg/oas"
+
+	"github.com/gin-gonic/gin"
 )
 
 func TestGetOASServers(t *testing.T) {
@@ -15,7 +17,6 @@ func TestGetOASServers(t *testing.T) {
 
 	GetOASServers(c)
 	t.Logf("Written body: %s", recorder.Body.String())
-	return
 }
 
 func TestGetOASAllSpecs(t *testing.T) {
@@ -26,7 +27,6 @@ func TestGetOASAllSpecs(t *testing.T) {
 
 	GetOASAllSpecs(c)
 	t.Logf("Written body: %s", recorder.Body.String())
-	return
 }
 
 func TestGetOASSpec(t *testing.T) {
@@ -39,5 +39,4 @@ func TestGetOASSpec(t *testing.T) {
 
 	GetOASSpec(c)
 	t.Logf("Written body: %s", recorder.Body.String())
-	return
 }

agent/pkg/controllers/service_map_controller.go

@@ -1,9 +1,10 @@
 package controllers
 
 import (
-	"mizuserver/pkg/servicemap"
 	"net/http"
 
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
+
 	"github.com/gin-gonic/gin"
 )
 

agent/pkg/controllers/service_map_controller_test.go

@@ -7,7 +7,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"mizuserver/pkg/servicemap"
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"

agent/pkg/controllers/status_controller.go

@@ -2,17 +2,18 @@ package controllers
 
 import (
 	"encoding/json"
+	"net/http"
+
 	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/api"
+	"github.com/up9inc/mizu/agent/pkg/holder"
+	"github.com/up9inc/mizu/agent/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
+	"github.com/up9inc/mizu/agent/pkg/up9"
+	"github.com/up9inc/mizu/agent/pkg/validation"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
-	"mizuserver/pkg/api"
-	"mizuserver/pkg/holder"
-	"mizuserver/pkg/providers"
-	"mizuserver/pkg/providers/tappedPods"
-	"mizuserver/pkg/providers/tappers"
-	"mizuserver/pkg/up9"
-	"mizuserver/pkg/validation"
-	"net/http"
 )
 
 func HealthCheck(c *gin.Context) {

agent/pkg/controllers/user_controller.go

@@ -1,7 +1,7 @@
 package controllers
 
 import (
-	"mizuserver/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/providers"
 
 	"github.com/gin-gonic/gin"
 	"github.com/up9inc/mizu/shared/logger"

agent/pkg/elastic/esClient.go

@@ -0,0 +1,120 @@
+package elastic
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/tap/api"
+	"net/http"
+	"sync"
+	"time"
+)
+
+type client struct {
+	es            *elasticsearch.Client
+	index         string
+	insertedCount int
+}
+
+var instance *client
+var once sync.Once
+
+func GetInstance() *client {
+	once.Do(func() {
+		instance = newClient()
+	})
+	return instance
+}
+
+func (client *client) Configure(config shared.ElasticConfig) {
+	if config.Url == "" || config.User == "" || config.Password == "" {
+		logger.Log.Infof("No elastic configuration was supplied, elastic exporter disabled")
+		return
+	}
+	transport := http.DefaultTransport
+	tlsClientConfig := &tls.Config{InsecureSkipVerify: true}
+	transport.(*http.Transport).TLSClientConfig = tlsClientConfig
+	cfg := elasticsearch.Config{
+		Addresses: []string{config.Url},
+		Username:  config.User,
+		Password:  config.Password,
+		Transport: transport,
+	}
+
+	es, err := elasticsearch.NewClient(cfg)
+	if err != nil {
+		logger.Log.Fatalf("Failed to initialize elastic client %v", err)
+	}
+
+	// Have the client instance return a response
+	res, err := es.Info()
+	if err != nil {
+		logger.Log.Fatalf("Elastic client.Info() ERROR: %v", err)
+	} else {
+		client.es = es
+		client.index = "mizu_traffic_http_" + time.Now().Format("2006_01_02_15_04")
+		client.insertedCount = 0
+		logger.Log.Infof("Elastic client configured, index: %s, cluster info: %v", client.index, res)
+	}
+	defer res.Body.Close()
+}
+
+func newClient() *client {
+	return &client{
+		es:    nil,
+		index: "",
+	}
+}
+
+type httpEntry struct {
+	Source      *api.TCP               `json:"src"`
+	Destination *api.TCP               `json:"dst"`
+	Outgoing    bool                   `json:"outgoing"`
+	CreatedAt   time.Time              `json:"createdAt"`
+	Request     map[string]interface{} `json:"request"`
+	Response    map[string]interface{} `json:"response"`
+	Summary     string                 `json:"summary"`
+	Method      string                 `json:"method"`
+	Status      int                    `json:"status"`
+	ElapsedTime int64                  `json:"elapsedTime"`
+	Path        string                 `json:"path"`
+}
+
+func (client *client) PushEntry(entry *api.Entry) {
+	if client.es == nil {
+		return
+	}
+
+	if entry.Protocol.Name != "http" {
+		return
+	}
+
+	entryToPush := httpEntry{
+		Source:      entry.Source,
+		Destination: entry.Destination,
+		Outgoing:    entry.Outgoing,
+		CreatedAt:   entry.StartTime,
+		Request:     entry.Request,
+		Response:    entry.Response,
+		Summary:     entry.Summary,
+		Method:      entry.Method,
+		Status:      entry.Status,
+		ElapsedTime: entry.ElapsedTime,
+		Path:        entry.Path,
+	}
+
+	entryJson, err := json.Marshal(entryToPush)
+	if err != nil {
+		logger.Log.Errorf("json.Marshal ERROR: %v", err)
+		return
+	}
+	var buffer bytes.Buffer
+	buffer.WriteString(string(entryJson))
+	res, _ := client.es.Index(client.index, &buffer)
+	if res.StatusCode == 201 {
+		client.insertedCount += 1
+	}
+}

agent/pkg/har/utils.go

@@ -123,11 +123,11 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
 	cookies := make([]Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
 
 	postData, _ := request["postData"].(map[string]interface{})
-	mimeType, _ := postData["mimeType"]
+	mimeType := postData["mimeType"]
 	if mimeType == nil || len(mimeType.(string)) == 0 {
 		mimeType = "text/html"
 	}
-	text, _ := postData["text"]
+	text := postData["text"]
 	postDataText := ""
 	if text != nil {
 		postDataText = text.(string)
@@ -176,12 +176,12 @@ func NewResponse(response map[string]interface{}) (harResponse *Response, err er
 	cookies := make([]Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
 
 	content, _ := response["content"].(map[string]interface{})
-	mimeType, _ := content["mimeType"]
+	mimeType := content["mimeType"]
 	if mimeType == nil || len(mimeType.(string)) == 0 {
 		mimeType = "text/html"
 	}
-	encoding, _ := content["encoding"]
-	text, _ := content["text"]
+	encoding := content["encoding"]
+	text := content["text"]
 	bodyText := ""
 	if text != nil {
 		bodyText = text.(string)

agent/pkg/holder/main.go

@@ -1,6 +1,6 @@
 package holder
 
-import "mizuserver/pkg/resolver"
+import "github.com/up9inc/mizu/agent/pkg/resolver"
 
 var k8sResolver *resolver.Resolver
 
@@ -11,4 +11,3 @@ func SetResolver(param *resolver.Resolver) {
 func GetResolver() *resolver.Resolver {
 	return k8sResolver
 }
-

agent/pkg/middlewares/auth.go

@@ -1,8 +1,8 @@
 package middlewares
 
 import (
-	"mizuserver/pkg/config"
-	"mizuserver/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/config"
+	"github.com/up9inc/mizu/agent/pkg/providers"
 
 	"github.com/gin-gonic/gin"
 	ory "github.com/ory/kratos-client-go"

agent/pkg/models/models.go

@@ -2,9 +2,10 @@ package models
 
 import (
 	"encoding/json"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/agent/pkg/rules"
 	tapApi "github.com/up9inc/mizu/tap/api"
-	"mizuserver/pkg/har"
-	"mizuserver/pkg/rules"
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"

agent/pkg/oas/counters.go

@@ -0,0 +1,119 @@
+package oas
+
+import (
+	"fmt"
+	"github.com/chanced/openapi"
+	"math"
+	"strings"
+)
+
+type Counter struct {
+	Entries     int     `json:"entries"`
+	Failures    int     `json:"failures"`
+	FirstSeen   float64 `json:"firstSeen"`
+	LastSeen    float64 `json:"lastSeen"`
+	SumRT       float64 `json:"sumRT"`
+	SumDuration float64 `json:"sumDuration"`
+}
+
+func (c *Counter) addEntry(ts float64, rt float64, succ bool, dur float64) {
+	if dur < 0 {
+		panic("Duration cannot be negative")
+	}
+
+	c.Entries += 1
+	c.SumRT += rt
+	c.SumDuration += dur
+	if !succ {
+		c.Failures += 1
+	}
+
+	if c.FirstSeen == 0 {
+		c.FirstSeen = ts
+	} else {
+		c.FirstSeen = math.Min(c.FirstSeen, ts)
+	}
+
+	c.LastSeen = math.Max(c.LastSeen, ts)
+}
+
+func (c *Counter) addOther(other *Counter) {
+	c.Entries += other.Entries
+	c.SumRT += other.SumRT
+	c.Failures += other.Failures
+	c.SumDuration += other.SumDuration
+
+	if c.FirstSeen == 0 {
+		c.FirstSeen = other.FirstSeen
+	} else {
+		c.FirstSeen = math.Min(c.FirstSeen, other.FirstSeen)
+	}
+
+	c.LastSeen = math.Max(c.LastSeen, other.LastSeen)
+}
+
+type CounterMap map[string]*Counter
+
+func (m *CounterMap) addOther(other *CounterMap) {
+	for src, cmap := range *other {
+		if existing, ok := (*m)[src]; ok {
+			existing.addOther(cmap)
+		} else {
+			copied := *cmap
+			(*m)[src] = &copied
+		}
+	}
+}
+
+func setCounterMsgIfOk(oldStr string, cnt *Counter) string {
+	tpl := "Mizu observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds"
+	if oldStr == "" || (strings.HasPrefix(oldStr, "Mizu ") && strings.HasSuffix(oldStr, " seconds")) {
+		return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries))
+	}
+	return oldStr
+}
+
+type CounterMaps struct {
+	counterTotal    Counter
+	counterMapTotal CounterMap
+}
+
+func (m *CounterMaps) processOp(opObj *openapi.Operation) error {
+	if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
+		counter := new(Counter)
+		err := opObj.Extensions.DecodeExtension(CountersTotal, counter)
+		if err != nil {
+			return err
+		}
+		m.counterTotal.addOther(counter)
+
+		opObj.Description = setCounterMsgIfOk(opObj.Description, counter)
+	}
+
+	if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
+		counterMap := new(CounterMap)
+		err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap)
+		if err != nil {
+			return err
+		}
+		m.counterMapTotal.addOther(counterMap)
+	}
+	return nil
+}
+
+func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error {
+	if oas.Extensions == nil {
+		oas.Extensions = openapi.Extensions{}
+	}
+
+	err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal)
+	if err != nil {
+		return err
+	}
+
+	err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal)
+	if err != nil {
+		return nil
+	}
+	return nil
+}

agent/pkg/oas/feeder_test.go

@@ -4,35 +4,50 @@ import (
 	"bufio"
 	"encoding/json"
 	"errors"
-	"github.com/up9inc/mizu/shared/logger"
 	"io"
 	"io/ioutil"
-	"mizuserver/pkg/har"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"testing"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
+
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func getFiles(baseDir string) (result []string, err error) {
-	result = make([]string, 0, 0)
+	result = make([]string, 0)
 	logger.Log.Infof("Reading files from tree: %s", baseDir)
 
+	inputs := []string{baseDir}
+
 	// https://yourbasic.org/golang/list-files-in-directory/
-	err = filepath.Walk(baseDir,
-		func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-
-			ext := strings.ToLower(filepath.Ext(path))
-			if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
-				result = append(result, path)
-			}
+	visitor := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
 
+		if info.Mode()&os.ModeSymlink != 0 {
+			path, _ = os.Readlink(path)
+			inputs = append(inputs, path)
 			return nil
-		})
+		}
+
+		ext := strings.ToLower(filepath.Ext(path))
+		if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
+			result = append(result, path)
+		}
+
+		return nil
+	}
+
+	for len(inputs) > 0 {
+		path := inputs[0]
+		inputs = inputs[1:]
+		err = filepath.Walk(path, visitor)
+	}
 
 	sort.SliceStable(result, func(i, j int) bool {
 		return fileSize(result[i]) < fileSize(result[j])
@@ -51,32 +66,42 @@ func fileSize(fname string) int64 {
 	return fi.Size()
 }
 
-func feedEntries(fromFiles []string) (err error) {
+func feedEntries(fromFiles []string, isSync bool) (count int, err error) {
+	badFiles := make([]string, 0)
+	cnt := 0
 	for _, file := range fromFiles {
 		logger.Log.Info("Processing file: " + file)
 		ext := strings.ToLower(filepath.Ext(file))
+		eCnt := 0
 		switch ext {
 		case ".har":
-			err = feedFromHAR(file)
+			eCnt, err = feedFromHAR(file, isSync)
 			if err != nil {
 				logger.Log.Warning("Failed processing file: " + err.Error())
+				badFiles = append(badFiles, file)
 				continue
 			}
 		case ".ldjson":
-			err = feedFromLDJSON(file)
+			eCnt, err = feedFromLDJSON(file, isSync)
 			if err != nil {
 				logger.Log.Warning("Failed processing file: " + err.Error())
+				badFiles = append(badFiles, file)
 				continue
 			}
 		default:
-			return errors.New("Unsupported file extension: " + ext)
+			return 0, errors.New("Unsupported file extension: " + ext)
 		}
+		cnt += eCnt
 	}
 
-	return nil
+	for _, f := range badFiles {
+		logger.Log.Infof("Bad file: %s", f)
+	}
+
+	return cnt, nil
 }
 
-func feedFromHAR(file string) error {
+func feedFromHAR(file string, isSync bool) (int, error) {
 	fd, err := os.Open(file)
 	if err != nil {
 		panic(err)
@@ -86,23 +111,43 @@ func feedFromHAR(file string) error {
 
 	data, err := ioutil.ReadAll(fd)
 	if err != nil {
-		return err
+		return 0, err
 	}
 
 	var harDoc har.HAR
 	err = json.Unmarshal(data, &harDoc)
 	if err != nil {
-		return err
+		return 0, err
 	}
 
+	cnt := 0
 	for _, entry := range harDoc.Log.Entries {
-		GetOasGeneratorInstance().PushEntry(&entry)
+		cnt += 1
+		feedEntry(&entry, "", isSync, file)
 	}
 
-	return nil
+	return cnt, nil
 }
 
-func feedFromLDJSON(file string) error {
+func feedEntry(entry *har.Entry, source string, isSync bool, file string) {
+	entry.Comment = file
+	if entry.Response.Status == 302 {
+		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
+	}
+
+	if strings.Contains(entry.Request.URL, "some") { // for debugging
+		logger.Log.Debugf("Interesting: %s", entry.Request.URL)
+	}
+
+	ews := EntryWithSource{Entry: *entry, Source: source, Id: uint(0)}
+	if isSync {
+		GetOasGeneratorInstance().entriesChan <- ews // blocking variant, right?
+	} else {
+		GetOasGeneratorInstance().PushEntry(&ews)
+	}
+}
+
+func feedFromLDJSON(file string, isSync bool) (int, error) {
 	fd, err := os.Open(file)
 	if err != nil {
 		panic(err)
@@ -113,8 +158,9 @@ func feedFromLDJSON(file string) error {
 	reader := bufio.NewReader(fd)
 
 	var meta map[string]interface{}
-
 	buf := strings.Builder{}
+	cnt := 0
+	source := ""
 	for {
 		substr, isPrefix, err := reader.ReadLine()
 		if err == io.EOF {
@@ -132,26 +178,31 @@ func feedFromLDJSON(file string) error {
 		if meta == nil {
 			err := json.Unmarshal([]byte(line), &meta)
 			if err != nil {
-				return err
+				return 0, err
+			}
+			if s, ok := meta["_source"]; ok && s != nil {
+				source = s.(string)
 			}
 		} else {
 			var entry har.Entry
 			err := json.Unmarshal([]byte(line), &entry)
 			if err != nil {
 				logger.Log.Warningf("Failed decoding entry: %s", line)
+			} else {
+				cnt += 1
+				feedEntry(&entry, source, isSync, file)
 			}
-			GetOasGeneratorInstance().PushEntry(&entry)
 		}
 	}
 
-	return nil
+	return cnt, nil
 }
 
 func TestFilesList(t *testing.T) {
 	res, err := getFiles("./test_artifacts/")
 	t.Log(len(res))
 	t.Log(res)
-	if err != nil || len(res) != 2 {
+	if err != nil || len(res) != 3 {
 		t.Logf("Should return 2 files but returned %d", len(res))
 		t.FailNow()
 	}

agent/pkg/oas/gibberish.go

@@ -1,21 +1,26 @@
 package oas
 
 import (
+	"math"
 	"regexp"
 	"strings"
 	"unicode"
 )
 
 var (
-	patBase64   = regexp.MustCompile(`^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$`)
 	patUuid4    = regexp.MustCompile(`(?i)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
 	patEmail    = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
-	patHexLower = regexp.MustCompile(`(0x)?[0-9a-f]{6,}`)
-	patHexUpper = regexp.MustCompile(`(0x)?[0-9A-F]{6,}`)
+	patLongNum  = regexp.MustCompile(`^\d{3,}$`)
+	patLongNumB = regexp.MustCompile(`[^\d]\d{3,}`)
+	patLongNumA = regexp.MustCompile(`\d{3,}[^\d]`)
 )
 
 func IsGibberish(str string) bool {
-	if patBase64.MatchString(str) && len(str) > 32 {
+	if IsVersionString(str) {
+		return false
+	}
+
+	if patEmail.MatchString(str) {
 		return true
 	}
 
@@ -23,20 +28,44 @@ func IsGibberish(str string) bool {
 		return true
 	}
 
-	if patEmail.MatchString(str) {
+	if patLongNum.MatchString(str) || patLongNumB.MatchString(str) || patLongNumA.MatchString(str) {
 		return true
 	}
 
-	if patHexLower.MatchString(str) || patHexUpper.MatchString(str) {
-		return true
+	//alNum := cleanStr(str, isAlNumRune)
+	//alpha := cleanStr(str, isAlphaRune)
+	// noiseAll := isNoisy(alNum)
+	//triAll := isTrigramBad(strings.ToLower(alpha))
+	// _ = noiseAll
+
+	isNotAlNum := func(r rune) bool { return !isAlNumRune(r) }
+	chunks := strings.FieldsFunc(str, isNotAlNum)
+	noisyLen := 0
+	alnumLen := 0
+	for _, chunk := range chunks {
+		alnumLen += len(chunk)
+		noise := isNoisy(chunk)
+		tri := isTrigramBad(strings.ToLower(chunk))
+		if noise || tri {
+			noisyLen += len(chunk)
+		}
 	}
 
-	noise := noiseLevel(str)
-	if noise >= 0.2 {
-		return true
-	}
+	return float64(noisyLen) > 0
 
-	return false
+	//if float64(noisyLen) > 0 {
+	//	return true
+	//}
+
+	//if len(chunks) > 0 && float64(noisyLen) >= float64(alnumLen)/3.0 {
+	//	return true
+	//}
+
+	//if triAll {
+	//return true
+	//}
+
+	// return false
 }
 
 func noiseLevel(str string) (score float64) {
@@ -54,21 +83,21 @@ func noiseLevel(str string) (score float64) {
 
 			// upper =>
 			case unicode.IsUpper(prev) && unicode.IsLower(char):
-				score += 0.25
+				score += 0.10
 			case unicode.IsUpper(prev) && unicode.IsDigit(char):
-				score += 0.25
+				score += 0.5
 
 			// lower =>
 			case unicode.IsLower(prev) && unicode.IsUpper(char):
 				score += 0.75
 			case unicode.IsLower(prev) && unicode.IsDigit(char):
-				score += 0.25
+				score += 0.5
 
 			// digit =>
 			case unicode.IsDigit(prev) && unicode.IsUpper(char):
 				score += 0.75
 			case unicode.IsDigit(prev) && unicode.IsLower(char):
-				score += 0.75
+				score += 1.0
 
 			// the rest is 100% noise
 			default:
@@ -78,8 +107,6 @@ func noiseLevel(str string) (score float64) {
 		prev = char
 	}
 
-	score /= cnt // weigh it
-
 	return score
 }
 
@@ -100,9 +127,59 @@ func IsVersionString(component string) bool {
 		}
 	}
 
-	if !hasV && strings.Contains(component, ".") {
+	if !hasV && !strings.Contains(component, ".") {
 		return false
 	}
 
 	return true
 }
+
+func trigramScore(str string) (float64, int) {
+	tgScore := 0.0
+	trigrams := ngrams(str, 3)
+	if len(trigrams) > 0 {
+		for _, trigram := range trigrams {
+			score, found := corpus_trigrams[trigram]
+			if found {
+				tgScore += score
+			}
+		}
+	}
+
+	return tgScore, len(trigrams)
+}
+
+func isTrigramBad(s string) bool {
+	tgScore, cnt := trigramScore(s)
+
+	if cnt > 0 {
+		val := math.Sqrt(tgScore) / float64(cnt)
+		val2 := tgScore / float64(cnt)
+		threshold := 0.005
+		bad := val < threshold
+		threshold2 := math.Log(float64(cnt)-2) * 0.1
+		bad2 := val2 < threshold2
+		return bad && bad2 // TODO: improve this logic to be clearer
+	}
+	return false
+}
+
+func isNoisy(s string) bool {
+	noise := noiseLevel(s)
+
+	if len(s) > 0 {
+		val := (noise * noise) / float64(len(s))
+		threshold := 0.6
+		bad := val > threshold
+		return bad
+	}
+	return false
+}
+
+func ngrams(s string, n int) []string {
+	result := make([]string, 0)
+	for i := 0; i < len(s)-n+1; i++ {
+		result = append(result, s[i:i+n])
+	}
+	return result
+}

agent/pkg/oas/gibberish_test.go

@@ -1,17 +1,95 @@
 package oas
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestNegative(t *testing.T) {
 	cases := []string{
 		"",
+		"{}",
+		"0.0.29",
+		"0.1",
+		"1.0",
+		"1.0.0",
+		"2.1.73",
+		"abTestV2",
+		"actionText,setName,setAttribute,save,ignore,onEnd,getContext,end,get",
+		"AddUserGroupLink",
+		"advert-management.adBlockerMessage.html",
+		"agents.author.1.json",
+		"animated-gif",
 		"b", // can be valid hexadecimal
-		"GetUniversalVariableUser",
+		"big-danger-coronavirus-panic-greater-crisis",
+		"breakout-box",
 		"callback",
-		"runs",
-		"tcfv2",
-		"StartUpCheckout",
+		"core.algorithm_execution.view",
+		"core.devices.view",
+		"data.json",
+		"dialog.overlay.infinity.json",
+		"domain-input",
+		"embeddable", // lul, it's a valid HEX!
+		"embeddable_blip",
+		"E PLURIBUS UNUM",
+		"etc",
+		"eu-central-1a",
+		"fcgi-bin",
+		"footer.include.html",
+		"fullHashes:find",
+		"generate-feed",
+		"GetAds",
 		"GetCart",
+		"GetUniversalVariableUser",
+		"github-audit-exports",
+		"g.js",
+		"g.pixel",
+		".html",
+		"Hugo Michiels",
+		"image.sbix",
+		"index.html",
+		"iPad",
+		"Joanna Mazewski",
+		"LibGit2Sharp",
+		"Michael_Vaughan1.png",
+		"New RSS feed has been generated",
+		"nick-clegg",
+		"opt-out",
+		"pixel_details.html",
+		"post.json",
+		"profile-method-info",
+		"project-id",
+		"publisha.1.json",
+		"publish_and_moderate",
+		"Ronna McDaniel",
+		"rtb-h",
+		"runs",
+		"sign-up",
+		"some-uuid-maybe",
+		"stable-4.0-version.json",
+		"StartUpCheckout",
+		"Steve Flunk",
+		"sync_a9",
+		"Ted Cruz",
+		"test.png",
+		"token",
+		"ToList",
+		"v2.1.3",
+		"VersionCheck.php",
+		"v Rusiji",
+		"Walnut St",
+		"web_widget",
+		"zoom_in.cur",
+		"xray",
+		"web",
+		"vipbets1",
+		"trcc",
+		"fbpixel",
+
+		// TODO below
+		// "tcfv2",
+		// "Matt-cartoon-255x206px-small.png",
+		// "TheTelegraph_portal_white-320-small.png",
+		// "testdata-10kB.js",
 	}
 
 	for _, str := range cases {
@@ -23,44 +101,92 @@ func TestNegative(t *testing.T) {
 
 func TestPositive(t *testing.T) {
 	cases := []string{
-		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
-		"952bea17-3776-11ea-9341-42010a84012a",
-		"456795af-b48f-4a8d-9b37-3e932622c2f0",
 		"0a0d0174-b338-4520-a1c3-24f7e3d5ec50.html",
-		"6120c057c7a97b03f6986f1b",
+		"1024807212418223",
+		"11ca096cbc224a67360493d44a9903",
+		"1553183382779",
+		"1554507871",
+		"19180481",
+		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
+		"456795af-b48f-4a8d-9b37-3e932622c2f0",
+		"601a2bdcc5b69137248ddbbf",
+		"60fe9aaeaefe2400012df94f",
 		"610bc3fd5a77a7fa25033fb0",
 		"610bd0315a77a7fa25034368",
 		"610bd0315a77a7fa25034368zh",
+		"6120c057c7a97b03f6986f1b",
 		"710a462e",
-		"1554507871",
-		"qwerqwerasdfqwer@protonmai.com",
-		"john.dow.1981@protonmail.com",
-		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
-		"11ca096cbc224a67360493d44a9903",
-		"c738338322370b47a79251f7510dd", // prefixed hex
-		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
-		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
-		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
-		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
 		"730970532670-compute@developer.gserviceaccount.com",
-		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
+		"819db2242a648b305395537022523d65",
+		"952bea17-3776-11ea-9341-42010a84012a",
+		"a3226860758.html",
 		"AAAA028295945",
-		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
+		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4",
+		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
+		"bnjksfd897345nl098asd53412kl98",
+		"c738338322370b47a79251f7510dd",                        // prefixed hex
+		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
+		"css/login.0f48c49a34eb53ea4623.min.css",
+		"d_fLLxlhzDilixeBEimaZ5",
+		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
+		"e8782afc112720300c049ff124434b79",
+		"fb6cjraf9cejut2a",
+		"i-0236530c66ed02200",
+		"JEHJW4BKVFDRTMTUQLHKK5WVAU",
+		"john.dow.1981@protonmail.com",
+		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
+		"MNUTGVFMGLEMFTBH0XSE5E02F6J2DS",
 		"n63nd45qsj",
 		"n9z9QGNiz",
+		"NC4WTmcy",
 		"proxy.3d2100fd7107262ecb55ce6847f01fa5.html",
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
+		"qwerqwerasdfqwer@protonmai.com",
 		"r-ext-5579e00a95c90",
 		"r-ext-5579e8b12f11e",
 		"r-v4-5c92513c9e04a",
 		"r-v4-5c92513c9e04a-0-c.monitoring.json",
 		"segments-1563566437171.639994",
+		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
+		"sp_dxJTfx11_576742227280287872",
+		"sp_NnUPB5wj_601a2bdcc5b69137248ddbbf",
+		"sp_NxITuoE4_premiumchron-article-14302157_c_ryGQBs_r_yIWvwP",
 		"t_52d94268-8810-4a7e-ba87-ffd657a6752f",
 		"timeouts-1563566437171.639994",
+		"u_YPF3GsGKMo02",
+
+		"0000000000 65535 f",
+		"0000000178 00000 n",
+		"0-10000",
+		"01526123,",
+		"0,18168,183955,3,4,1151616,5663,731,223,5104,207,3204,10,1051,175,364,1435,4,60,576,241,383,246,5,1102",
+		"05/10/2020",
+		"14336456724940333",
+		"fb6cjraf9cejut2a",
+		"JEHJW4BKVFDRTMTUQLHKK5WVAU",
 
 		// TODO
-		// "fb6cjraf9cejut2a",
-		// "Fxvd1timk", // questionable
-		// "JEHJW4BKVFDRTMTUQLHKK5WVAU",
+		/*
+			"0,20",
+			"0.001",
+			"YISAtiX1",
+			"Fxvd1timk", // questionable
+			"B4GCSkORAJs",
+			"D_4EDAqenHQ",
+			"EICJp29EGOk",
+			"Fxvd1timk",
+			"GTqMZELYfQQ",
+			"GZPTpLPEGmwHGWPC",
+			"_HChnE9NDPY",
+			"NwhjgIWHgGg",
+			"production/tsbqksph4xswqjexfbec",
+			"p/u/bguhrxupr23mw3nwxcrw",
+			"nRSNapbJZnc",
+			"zgfpbtolciznub5egzxk",
+			"zufnu7aimadua9wrgwwo",
+			"zznto1jzch9yjsbtbrul",
+		*/
 	}
 
 	for _, str := range cases {
@@ -69,3 +195,18 @@ func TestPositive(t *testing.T) {
 		}
 	}
 }
+
+func TestVersionStrings(t *testing.T) {
+	cases := []string{
+		"1.0",
+		"1.0.0",
+		"v2.1.3",
+		"2.1.73",
+	}
+
+	for _, str := range cases {
+		if !IsVersionString(str) {
+			t.Errorf("Mistakenly false: %s", str)
+		}
+	}
+}

agent/pkg/oas/ignores.go

@@ -12,24 +12,26 @@ var ignoredHeaders = []string{
 	"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-type", "cookie",
 	"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
 	"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
-	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests",
-	"server", "user-agent", "via", "warning", "strict-transport-security",
-	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data",
+	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests", "x-download-options",
+	"server", "user-agent", "via", "warning", "strict-transport-security", "x-permitted-cross-domain-policies",
+	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data", "x-dns-prefetch-control",
 	"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
 	"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
-	"x-wap-profile", "x-scheme",
-	"newrelic", "x-cloud-trace-context", "sentry-trace",
-	"expires", "set-cookie", "p3p", "location", "content-security-policy", "content-security-policy-report-only",
-	"last-modified", "content-language",
+	"x-wap-profile", "x-scheme", "status", "x-cache", "x-application-context", "retry-after",
+	"newrelic", "x-cloud-trace-context", "sentry-trace", "x-cache-hits", "x-served-by", "x-span-name",
+	"expires", "set-cookie", "p3p", "content-security-policy", "content-security-policy-report-only",
+	"last-modified", "content-language", "x-varnish", "true-client-ip", "akamai-origin-hop",
 	"keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by",
-	"age", "allow", "www-authenticate",
+	"age", "allow", "www-authenticate", "expect-ct", "timing-allow-origin", "referrer-policy",
+	"x-aspnet-version", "x-aspnetmvc-version", "x-timer", "x-abuse-info", "x-mod-pagespeed",
+	"duration_ms", // UP9 custom
 }
 
 var ignoredHeaderPrefixes = []string{
 	":", "accept-", "access-control-", "if-", "sec-", "grpc-",
-	"x-forwarded-", "x-original-",
+	"x-forwarded-", "x-original-", "cf-",
 	"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
-	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-",
+	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-", "x-goog-",
 }
 
 func isCtypeIgnored(ctype string) bool {

agent/pkg/oas/oas_generator.go

@@ -3,10 +3,11 @@ package oas
 import (
 	"context"
 	"encoding/json"
-	"github.com/up9inc/mizu/shared/logger"
-	"mizuserver/pkg/har"
 	"net/url"
 	"sync"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var (
@@ -29,7 +30,7 @@ func (g *oasGenerator) Start() {
 	ctx, cancel := context.WithCancel(context.Background())
 	g.cancel = cancel
 	g.ctx = ctx
-	g.entriesChan = make(chan har.Entry, 100) // buffer up to 100 entries for OAS processing
+	g.entriesChan = make(chan EntryWithSource, 100) // buffer up to 100 entries for OAS processing
 	g.ServiceSpecs = &sync.Map{}
 	g.started = true
 	go instance.runGeneretor()
@@ -42,11 +43,12 @@ func (g *oasGenerator) runGeneretor() {
 			logger.Log.Infof("OAS Generator was canceled")
 			return
 
-		case entry, ok := <-g.entriesChan:
+		case entryWithSource, ok := <-g.entriesChan:
 			if !ok {
 				logger.Log.Infof("OAS Generator - entries channel closed")
 				break
 			}
+			entry := entryWithSource.Entry
 			u, err := url.Parse(entry.Request.URL)
 			if err != nil {
 				logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
@@ -61,7 +63,7 @@ func (g *oasGenerator) runGeneretor() {
 				gen = val.(*SpecGen)
 			}
 
-			opId, err := gen.feedEntry(entry)
+			opId, err := gen.feedEntry(entryWithSource)
 			if err != nil {
 				txt, suberr := json.Marshal(entry)
 				if suberr == nil {
@@ -77,12 +79,16 @@ func (g *oasGenerator) runGeneretor() {
 	}
 }
 
-func (g *oasGenerator) PushEntry(entry *har.Entry) {
+func (g *oasGenerator) Reset() {
+	g.ServiceSpecs = &sync.Map{}
+}
+
+func (g *oasGenerator) PushEntry(entryWithSource *EntryWithSource) {
 	if !g.started {
 		return
 	}
 	select {
-	case g.entriesChan <- *entry:
+	case g.entriesChan <- *entryWithSource:
 	default:
 		logger.Log.Warningf("OAS Generator - entry wasn't sent to channel because the channel has no buffer or there is no receiver")
 	}
@@ -98,10 +104,16 @@ func newOasGenerator() *oasGenerator {
 	}
 }
 
+type EntryWithSource struct {
+	Source string
+	Entry  har.Entry
+	Id     uint
+}
+
 type oasGenerator struct {
 	started      bool
 	ctx          context.Context
 	cancel       context.CancelFunc
 	ServiceSpecs *sync.Map
-	entriesChan  chan har.Entry
+	entriesChan  chan EntryWithSource
 }

agent/pkg/oas/specgen.go

@@ -5,15 +5,29 @@ import (
 	"errors"
 	"github.com/chanced/openapi"
 	"github.com/google/uuid"
+	"github.com/nav-inc/datetime"
 	"github.com/up9inc/mizu/shared/logger"
+	"io"
+	"io/ioutil"
 	"mime"
-	"mizuserver/pkg/har"
+	"mime/multipart"
+	"net/textproto"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
+
+	"time"
 )
 
+const LastSeenTS = "x-last-seen-ts"
+const CountersTotal = "x-counters-total"
+const CountersPerSource = "x-counters-per-source"
+const SampleId = "x-sample-entry"
+
 type reqResp struct { // hello, generics in Go
 	Req  *har.Request
 	Resp *har.Response
@@ -30,7 +44,7 @@ func NewGen(server string) *SpecGen {
 	spec.Version = "3.1.0"
 
 	info := openapi.Info{Title: server}
-	info.Version = "0.0"
+	info.Version = "1.0"
 	spec.Info = &info
 	spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
 
@@ -43,17 +57,23 @@ func NewGen(server string) *SpecGen {
 
 func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
 	g.oas = oas
+	g.tree = new(Node)
 	for pathStr, pathObj := range oas.Paths.Items {
 		pathSplit := strings.Split(string(pathStr), "/")
 		g.tree.getOrSet(pathSplit, pathObj)
+
+		// clean "last entry timestamp" markers from the past
+		for _, pathAndOp := range g.tree.listOps() {
+			delete(pathAndOp.op.Extensions, LastSeenTS)
+		}
 	}
 }
 
-func (g *SpecGen) feedEntry(entry har.Entry) (string, error) {
+func (g *SpecGen) feedEntry(entryWithSource EntryWithSource) (string, error) {
 	g.lock.Lock()
 	defer g.lock.Unlock()
 
-	opId, err := g.handlePathObj(&entry)
+	opId, err := g.handlePathObj(&entryWithSource)
 	if err != nil {
 		return "", err
 	}
@@ -68,10 +88,23 @@ func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
 
 	g.tree.compact()
 
-	for _, pathop := range g.tree.listOps() {
-		if pathop.op.Summary == "" {
-			pathop.op.Summary = pathop.path
+	counters := CounterMaps{counterTotal: Counter{}, counterMapTotal: CounterMap{}}
+
+	for _, pathAndOp := range g.tree.listOps() {
+		opObj := pathAndOp.op
+		if opObj.Summary == "" {
+			opObj.Summary = pathAndOp.path
 		}
+
+		err := counters.processOp(opObj)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	err := counters.processOas(g.oas)
+	if err != nil {
+		return nil, err
 	}
 
 	// put paths back from tree into OAS
@@ -79,6 +112,8 @@ func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
 
 	suggestTags(g.oas)
 
+	g.oas.Info.Description = setCounterMsgIfOk(g.oas.Info.Description, &counters.counterTotal)
+
 	// to make a deep copy, no better idea than marshal+unmarshal
 	specText, err := json.MarshalIndent(g.oas, "", "\t")
 	if err != nil {
@@ -96,6 +131,7 @@ func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
 
 func suggestTags(oas *openapi.OpenAPI) {
 	paths := getPathsKeys(oas.Paths.Items)
+	sort.Strings(paths) // make it stable in case of multiple candidates
 	for len(paths) > 0 {
 		group := make([]string, 0)
 		group = append(group, paths[0])
@@ -125,37 +161,9 @@ func suggestTags(oas *openapi.OpenAPI) {
 				}
 			}
 		}
-
-		//groups[common] = group
 	}
 }
 
-func getSimilarPrefix(strs []string) string {
-	chunked := make([][]string, 0)
-	for _, item := range strs {
-		chunked = append(chunked, strings.Split(item, "/"))
-	}
-
-	cmn := longestCommonXfix(chunked, true)
-	res := make([]string, 0)
-	for _, chunk := range cmn {
-		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
-			res = append(res, chunk)
-		}
-	}
-	return strings.Join(res[1:], ".")
-}
-
-func deleteFromSlice(s []string, val string) []string {
-	temp := s[:0]
-	for _, x := range s {
-		if x != val {
-			temp = append(temp, x)
-		}
-	}
-	return temp
-}
-
 func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
 	keys := make([]string, len(mymap))
 
@@ -167,7 +175,8 @@ func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
 	return keys
 }
 
-func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
+func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error) {
+	entry := entryWithSource.Entry
 	urlParsed, err := url.Parse(entry.Request.URL)
 	if err != nil {
 		return "", err
@@ -175,11 +184,18 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
 
 	if isExtIgnored(urlParsed.Path) {
 		logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
+		return "", nil
+	}
+
+	if entry.Request.Method == "OPTIONS" {
+		logger.Log.Debugf("Dropped traffic entry due to its method: %s", urlParsed.Path)
+		return "", nil
 	}
 
 	ctype := getRespCtype(&entry.Response)
 	if isCtypeIgnored(ctype) {
 		logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
+		return "", nil
 	}
 
 	if entry.Response.Status < 100 {
@@ -192,9 +208,19 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
 		return "", nil
 	}
 
-	split := strings.Split(urlParsed.Path, "/")
+	if entry.Response.Status == 502 || entry.Response.Status == 503 || entry.Response.Status == 504 {
+		logger.Log.Debugf("Dropped traffic entry due to temporary server error: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	var split []string
+	if urlParsed.RawPath != "" {
+		split = strings.Split(urlParsed.RawPath, "/")
+	} else {
+		split = strings.Split(urlParsed.Path, "/")
+	}
 	node := g.tree.getOrSet(split, new(openapi.PathObj))
-	opObj, err := handleOpObj(entry, node.ops)
+	opObj, err := handleOpObj(entryWithSource, node.pathObj)
 
 	if opObj != nil {
 		return opObj.OperationID, err
@@ -203,7 +229,8 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
 	return "", err
 }
 
-func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation, error) {
+func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj) (*openapi.Operation, error) {
+	entry := entryWithSource.Entry
 	isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
 	opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
 	if err != nil {
@@ -225,46 +252,144 @@ func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation
 		return nil, err
 	}
 
+	err = handleCounters(opObj, isSuccess, entryWithSource)
+	if err != nil {
+		return nil, err
+	}
+
 	return opObj, nil
 }
 
+func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *EntryWithSource) error {
+	// TODO: if performance around DecodeExtension+SetExtension is bad, store counters as separate maps
+	counter := Counter{}
+	counterMap := CounterMap{}
+	prevTs := 0.0
+	if opObj.Extensions == nil {
+		opObj.Extensions = openapi.Extensions{}
+	} else {
+		if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
+			err := opObj.Extensions.DecodeExtension(CountersTotal, &counter)
+			if err != nil {
+				return err
+			}
+		}
+
+		if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
+			err := opObj.Extensions.DecodeExtension(CountersPerSource, &counterMap)
+			if err != nil {
+				return err
+			}
+		}
+
+		if _, ok := opObj.Extensions.Extension(LastSeenTS); ok {
+			err := opObj.Extensions.DecodeExtension(LastSeenTS, &prevTs)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	var counterPerSource *Counter
+	if existing, ok := counterMap[entryWithSource.Source]; ok {
+		counterPerSource = existing
+	} else {
+		counterPerSource = new(Counter)
+		counterMap[entryWithSource.Source] = counterPerSource
+	}
+
+	started, err := datetime.Parse(entryWithSource.Entry.StartedDateTime, time.UTC)
+	if err != nil {
+		return err
+	}
+
+	ts := float64(started.UnixNano()) / float64(time.Millisecond) / 1000
+	rt := float64(entryWithSource.Entry.Time) / 1000
+
+	dur := 0.0
+	if prevTs != 0 && ts >= prevTs {
+		dur = ts - prevTs
+	}
+
+	counter.addEntry(ts, rt, success, dur)
+	counterPerSource.addEntry(ts, rt, success, dur)
+
+	err = opObj.Extensions.SetExtension(LastSeenTS, ts)
+	if err != nil {
+		return err
+	}
+
+	err = opObj.Extensions.SetExtension(CountersTotal, counter)
+	if err != nil {
+		return err
+	}
+
+	err = opObj.Extensions.SetExtension(CountersPerSource, counterMap)
+	if err != nil {
+		return err
+	}
+
+	err = opObj.Extensions.SetExtension(SampleId, entryWithSource.Id)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error {
 	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
+	urlParsed, err := url.Parse(req.URL)
+	if err != nil {
+		return err
+	}
+
+	qs := make([]har.NVP, 0)
+	for name, vals := range urlParsed.Query() {
+		for _, val := range vals {
+			qs = append(qs, har.NVP{Name: name, Value: val})
+		}
+	}
+
+	if len(qs) != len(req.QueryString) {
+		logger.Log.Warningf("QStr params in HAR do not match URL: %s", req.URL)
+	}
 
 	qstrGW := nvParams{
-		In: openapi.InQuery,
-		Pairs: func() []NVPair {
-			return qstrToNVP(req.QueryString)
-		},
+		In:             openapi.InQuery,
+		Pairs:          qs,
 		IsIgnored:      func(name string) bool { return false },
 		GeneralizeName: func(name string) string { return name },
 	}
-	handleNameVals(qstrGW, &opObj.Parameters)
+	handleNameVals(qstrGW, &opObj.Parameters, false)
 
 	hdrGW := nvParams{
-		In: openapi.InHeader,
-		Pairs: func() []NVPair {
-			return hdrToNVP(req.Headers)
-		},
+		In:             openapi.InHeader,
+		Pairs:          req.Headers,
 		IsIgnored:      isHeaderIgnored,
 		GeneralizeName: strings.ToLower,
 	}
-	handleNameVals(hdrGW, &opObj.Parameters)
+	handleNameVals(hdrGW, &opObj.Parameters, true)
 
-	if req.PostData.Text != "" && isSuccess {
-		reqBody, err := getRequestBody(req, opObj, isSuccess)
+	if isSuccess {
+		reqBody, err := getRequestBody(req, opObj)
 		if err != nil {
 			return err
 		}
 
 		if reqBody != nil {
-			reqCtype := getReqCtype(req)
-			reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, err)
-			if err != nil {
-				return err
-			}
+			if req.PostData.Text == "" {
+				reqBody.Required = false
+			} else {
 
-			_ = reqMedia
+				reqCtype, _ := getReqCtype(req)
+				reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype)
+				if err != nil {
+					return err
+				}
+
+				_ = reqMedia
+			}
 		}
 	}
 	return nil
@@ -281,7 +406,7 @@ func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool
 
 	respCtype := getRespCtype(resp)
 	respContent := respObj.Content
-	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, err)
+	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype)
 	if err != nil {
 		return err
 	}
@@ -329,11 +454,9 @@ func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj) {
 			}
 		}
 	}
-
-	return
 }
 
-func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err error) (*openapi.MediaType, error) {
+func fillContent(reqResp reqResp, respContent openapi.Content, ctype string) (*openapi.MediaType, error) {
 	content, found := respContent[ctype]
 	if !found {
 		respContent[ctype] = &openapi.MediaType{}
@@ -348,29 +471,142 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err
 		isBinary, _, text = reqResp.Resp.Content.B64Decoded()
 	}
 
-	if !isBinary {
+	if !isBinary && text != "" {
 		var exampleMsg []byte
 		// try treating it as json
 		any, isJSON := anyJSON(text)
 		if isJSON {
 			// re-marshal with forced indent
-			exampleMsg, err = json.MarshalIndent(any, "", "\t")
-			if err != nil {
+			if msg, err := json.MarshalIndent(any, "", "\t"); err != nil {
 				panic("Failed to re-marshal value, super-strange")
+			} else {
+				exampleMsg = msg
 			}
 		} else {
-			exampleMsg, err = json.Marshal(text)
-			if err != nil {
+			if msg, err := json.Marshal(text); err != nil {
 				return nil, err
+			} else {
+				exampleMsg = msg
 			}
 		}
 
-		content.Example = exampleMsg
+		if ctype == "application/x-www-form-urlencoded" && reqResp.Req != nil {
+			handleFormDataUrlencoded(text, content)
+		} else if strings.HasPrefix(ctype, "multipart/form-data") && reqResp.Req != nil {
+			_, params := getReqCtype(reqResp.Req)
+			handleFormDataMultipart(text, content, params)
+		}
+
+		if content.Example == nil && len(exampleMsg) > len(content.Example) {
+			content.Example = exampleMsg
+		}
 	}
 
 	return respContent[ctype], nil
 }
 
+func handleFormDataUrlencoded(text string, content *openapi.MediaType) {
+	formData, err := url.ParseQuery(text)
+	if err != nil {
+		logger.Log.Warningf("Could not decode urlencoded: %s", err)
+		return
+	}
+
+	parts := make([]PartWithBody, 0)
+	for name, vals := range formData {
+		for _, val := range vals {
+			part := new(multipart.Part)
+			part.Header = textproto.MIMEHeader{}
+			part.Header.Add("Content-Disposition", "form-data; name=\""+name+"\";")
+			parts = append(parts, PartWithBody{part: part, body: []byte(val)})
+		}
+	}
+	handleFormData(content, parts)
+}
+
+func handleFormData(content *openapi.MediaType, parts []PartWithBody) {
+	hadSchema := true
+	if content.Schema == nil {
+		hadSchema = false // will use it for required flags
+		content.Schema = new(openapi.SchemaObj)
+		content.Schema.Type = openapi.Types{openapi.TypeObject}
+		content.Schema.Properties = openapi.Schemas{}
+	}
+
+	props := &content.Schema.Properties
+	seenNames := map[string]struct{}{} // set equivalent in Go, yikes
+	for _, pwb := range parts {
+		name := pwb.part.FormName()
+		seenNames[name] = struct{}{}
+		existing, found := (*props)[name]
+		if !found {
+			existing = new(openapi.SchemaObj)
+			existing.Type = openapi.Types{openapi.TypeString}
+			(*props)[name] = existing
+
+			ctype := pwb.part.Header.Get("content-type")
+			if ctype != "" {
+				if existing.Keywords == nil {
+					existing.Keywords = map[string]json.RawMessage{}
+				}
+				existing.Keywords["contentMediaType"], _ = json.Marshal(ctype)
+			}
+		}
+
+		addSchemaExample(existing, string(pwb.body))
+	}
+
+	// handle required flag
+	if content.Schema.Required == nil {
+		if !hadSchema {
+			content.Schema.Required = make([]string, 0)
+			for name := range seenNames {
+				content.Schema.Required = append(content.Schema.Required, name)
+			}
+			sort.Strings(content.Schema.Required)
+		} // else it's a known schema with no required fields
+	} else {
+		content.Schema.Required = intersectSliceWithMap(content.Schema.Required, seenNames)
+		sort.Strings(content.Schema.Required)
+	}
+}
+
+type PartWithBody struct {
+	part *multipart.Part
+	body []byte
+}
+
+func handleFormDataMultipart(text string, content *openapi.MediaType, ctypeParams map[string]string) {
+	boundary, ok := ctypeParams["boundary"]
+	if !ok {
+		logger.Log.Errorf("Multipart header has no boundary")
+		return
+	}
+	mpr := multipart.NewReader(strings.NewReader(text), boundary)
+
+	parts := make([]PartWithBody, 0)
+	for {
+		part, err := mpr.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			logger.Log.Errorf("Cannot parse multipart body: %v", err)
+			break
+		}
+		defer part.Close()
+
+		body, err := ioutil.ReadAll(part)
+		if err != nil {
+			logger.Log.Errorf("Error reading multipart Part %s: %v", part.Header, err)
+		}
+
+		parts = append(parts, PartWithBody{part: part, body: body})
+	}
+
+	handleFormData(content, parts)
+}
+
 func getRespCtype(resp *har.Response) string {
 	var ctype string
 	ctype = resp.Content.MimeType
@@ -387,8 +623,7 @@ func getRespCtype(resp *har.Response) string {
 	return mediaType
 }
 
-func getReqCtype(req *har.Request) string {
-	var ctype string
+func getReqCtype(req *har.Request) (ctype string, params map[string]string) {
 	ctype = req.PostData.MimeType
 	for _, hdr := range req.Headers {
 		if strings.ToLower(hdr.Name) == "content-type" {
@@ -396,11 +631,16 @@ func getReqCtype(req *har.Request) string {
 		}
 	}
 
-	mediaType, _, err := mime.ParseMediaType(ctype)
-	if err != nil {
-		return ""
+	if ctype == "" {
+		return "", map[string]string{}
 	}
-	return mediaType
+
+	mediaType, params, err := mime.ParseMediaType(ctype)
+	if err != nil {
+		logger.Log.Errorf("Cannot parse Content-Type header %q: %v", ctype, err)
+		return "", map[string]string{}
+	}
+	return mediaType, params
 }
 
 func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool) (*openapi.ResponseObj, error) {
@@ -430,9 +670,14 @@ func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool
 	return resResponse, nil
 }
 
-func getRequestBody(req *har.Request, opObj *openapi.Operation, isSuccess bool) (*openapi.RequestBodyObj, error) {
+func getRequestBody(req *har.Request, opObj *openapi.Operation) (*openapi.RequestBodyObj, error) {
 	if opObj.RequestBody == nil {
-		opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
+		// create if there is body in request
+		if req.PostData.Text != "" {
+			opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
+		} else {
+			return nil, nil
+		}
 	}
 
 	reqBody, err := opObj.RequestBody.ResolveRequestBody(reqBodyResolver)
@@ -440,9 +685,6 @@ func getRequestBody(req *har.Request, opObj *openapi.Operation, isSuccess bool)
 		return nil, err
 	}
 
-	// TODO: maintain required flag for it, but only consider successful responses
-	//reqBody.Content[]
-
 	return reqBody, nil
 }
 

agent/pkg/oas/specgen_test.go

@@ -3,48 +3,60 @@ package oas
 import (
 	"encoding/json"
 	"github.com/chanced/openapi"
+	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/shared/logger"
+	"github.com/wI2L/jsondiff"
 	"io/ioutil"
-	"mizuserver/pkg/har"
 	"os"
+	"regexp"
 	"strings"
 	"testing"
+	"time"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
 )
 
 // if started via env, write file into subdir
-func writeFiles(label string, spec *openapi.OpenAPI) {
+func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string {
+	content, err := json.MarshalIndent(spec, "", "\t")
+	if err != nil {
+		panic(err)
+	}
+
 	if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
 		path := "./oas-samples"
 		err := os.MkdirAll(path, 0o755)
 		if err != nil {
 			panic(err)
 		}
-		content, err := json.MarshalIndent(spec, "", "\t")
-		if err != nil {
-			panic(err)
-		}
 		err = ioutil.WriteFile(path+"/"+label+".json", content, 0644)
 		if err != nil {
 			panic(err)
 		}
+		t.Logf("Written: %s", label)
+	} else {
+		t.Logf("%s", string(content))
 	}
+	return string(content)
 }
 
 func TestEntries(t *testing.T) {
+	logger.InitLoggerStderrOnly(logging.INFO)
 	files, err := getFiles("./test_artifacts/")
-	// files, err = getFiles("/media/bigdisk/UP9")
 	if err != nil {
 		t.Log(err)
 		t.FailNow()
 	}
 	GetOasGeneratorInstance().Start()
+	loadStartingOAS()
 
-	if err := feedEntries(files); err != nil {
+	cnt, err := feedEntries(files, true)
+	if err != nil {
 		t.Log(err)
 		t.Fail()
 	}
 
-	loadStartingOAS()
+	waitQueueProcessed()
 
 	svcs := strings.Builder{}
 	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
@@ -78,33 +90,36 @@ func TestEntries(t *testing.T) {
 			t.FailNow()
 		}
 
-		specText, _ := json.MarshalIndent(spec, "", "\t")
-		t.Logf("%s", string(specText))
+		outputSpec(svc, spec, t)
 
 		err = spec.Validate()
 		if err != nil {
 			t.Log(err)
 			t.FailNow()
 		}
-		writeFiles(svc, spec)
 
 		return true
 	})
 
+	logger.Log.Infof("Total entries: %d", cnt)
 }
 
-func TestFileLDJSON(t *testing.T) {
+func TestFileSingle(t *testing.T) {
 	GetOasGeneratorInstance().Start()
-	file := "test_artifacts/output_rdwtyeoyrj.har.ldjson"
-	err := feedFromLDJSON(file)
+	GetOasGeneratorInstance().Reset()
+	// loadStartingOAS()
+	file := "test_artifacts/params.har"
+	files := []string{file}
+	cnt, err := feedEntries(files, true)
 	if err != nil {
 		logger.Log.Warning("Failed processing file: " + err.Error())
 		t.Fail()
 	}
 
-	loadStartingOAS()
+	waitQueueProcessed()
 
-	GetOasGeneratorInstance().ServiceSpecs.Range(func(_, val interface{}) bool {
+	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+		svc := key.(string)
 		gen := val.(*SpecGen)
 		spec, err := gen.GetSpec()
 		if err != nil {
@@ -112,8 +127,7 @@ func TestFileLDJSON(t *testing.T) {
 			t.FailNow()
 		}
 
-		specText, _ := json.MarshalIndent(spec, "", "\t")
-		t.Logf("%s", string(specText))
+		specText := outputSpec(svc, spec, t)
 
 		err = spec.Validate()
 		if err != nil {
@@ -121,8 +135,44 @@ func TestFileLDJSON(t *testing.T) {
 			t.FailNow()
 		}
 
+		expected, err := ioutil.ReadFile(file + ".spec.json")
+		if err != nil {
+			t.Errorf(err.Error())
+			t.FailNow()
+		}
+
+		patFloatPrecision := regexp.MustCompile(`(\d+\.\d{1,2})(\d*)`)
+
+		expected = []byte(patUuid4.ReplaceAllString(string(expected), "<UUID4>"))
+		specText = patUuid4.ReplaceAllString(specText, "<UUID4>")
+		expected = []byte(patFloatPrecision.ReplaceAllString(string(expected), "$1"))
+		specText = patFloatPrecision.ReplaceAllString(specText, "$1")
+
+		diff, err := jsondiff.CompareJSON(expected, []byte(specText))
+		if err != nil {
+			t.Errorf(err.Error())
+			t.FailNow()
+		}
+
+		if len(diff) > 0 {
+			t.Errorf("Generated spec does not match expected:\n%s", diff.String())
+		}
+
 		return true
 	})
+
+	logger.Log.Infof("Processed entries: %d", cnt)
+}
+
+func waitQueueProcessed() {
+	for {
+		time.Sleep(100 * time.Millisecond)
+		queue := len(GetOasGeneratorInstance().entriesChan)
+		logger.Log.Infof("Queue: %d", queue)
+		if queue < 1 {
+			break
+		}
+	}
 }
 
 func loadStartingOAS() {
@@ -149,26 +199,33 @@ func loadStartingOAS() {
 	gen.StartFromSpec(doc)
 
 	GetOasGeneratorInstance().ServiceSpecs.Store("catalogue", gen)
-
-	return
 }
 
 func TestEntriesNegative(t *testing.T) {
 	files := []string{"invalid"}
-	err := feedEntries(files)
+	_, err := feedEntries(files, false)
 	if err == nil {
 		t.Logf("Should have failed")
 		t.Fail()
 	}
 }
 
+func TestEntriesPositive(t *testing.T) {
+	files := []string{"test_artifacts/params.har"}
+	_, err := feedEntries(files, false)
+	if err != nil {
+		t.Logf("Failed")
+		t.Fail()
+	}
+}
+
 func TestLoadValidHAR(t *testing.T) {
 	inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}`
 	var entry *har.Entry
 	var err = json.Unmarshal([]byte(inp), &entry)
 	if err != nil {
 		t.Logf("Failed to decode entry: %s", err)
-		// t.FailNow() demonstrates the problem of library
+		t.FailNow() // demonstrates the problem of `martian` HAR library
 	}
 }
 
@@ -193,5 +250,4 @@ func TestLoadValid3_1(t *testing.T) {
 		t.Log(err)
 		t.FailNow()
 	}
-	return
 }

agent/pkg/oas/test_artifacts/example.har

@@ -13,10 +13,14 @@
         "time": 1022,
         "request": {
           "method": "GET",
-          "url": "http://trcc-api-service/proxies/",
+          "url": "http://trcc-api-service/proxies;matrixparam=example/",
           "httpVersion": "",
           "cookies": [],
           "headers": [
+            {
+              "name": "X-Custom-Demo-Header",
+              "value": "demo"
+            },
             {
               "name": "Sec-Fetch-Dest",
               "value": "empty"
@@ -124,6 +128,10 @@
           "httpVersion": "",
           "cookies": [],
           "headers": [
+            {
+              "name": "X-Custom-Demo-Header2",
+              "value": "demo2"
+            },
             {
               "name": "Vary",
               "value": "Origin"
@@ -24568,7 +24576,7 @@
           "bodySize": -1
         },
         "response": {
-          "status": 200,
+          "status": 308,
           "statusText": "OK",
           "httpVersion": "",
           "cookies": [],
@@ -24635,7 +24643,7 @@
         "time": 1,
         "request": {
           "method": "GET",
-          "url": "http://trcc-api-service/models/aws_kong5/suites/all/runs",
+          "url": "http://trcc-api-service/models/aws_kong5/suites/all/runs.png",
           "httpVersion": "",
           "cookies": [],
           "headers": [
@@ -24894,7 +24902,7 @@
           "bodySize": -1
         },
         "response": {
-          "status": 200,
+          "status": 0,
           "statusText": "OK",
           "httpVersion": "",
           "cookies": [],

agent/pkg/oas/test_artifacts/example.ldjson

@@ -1,4 +1,4 @@
-{"messageType": "http", "_source": null, "firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
+{"messageType": "http", "_source": "some-source", ",firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
 {"_id": "", "startedDateTime": "2021-07-26T11:14:17.78415179Z", "time": 13, "request": {"method": "GET", "url": "http://catalogue/catalogue/size?tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "x-some", "value": "demo val"},{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [{"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "22"}], "content": {"size": 22, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInNpemUiOjl9", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 22}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 13}}
 {"_id": "", "startedDateTime": "2021-07-26T11:14:17.784918698Z", "time": 19, "request": {"method": "GET", "url": "http://catalogue/catalogue?page=1&size=6&tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "page", "value": "1"}, {"name": "size", "value": "6"}, {"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "1927"}], "content": {"size": 1927, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19LHsiaWQiOiI4MzdhYjE0MS0zOTllLTRjMWYtOWFiYy1iYWNlNDAyOTZiYWMiLCJuYW1lIjoiQ2F0IHNvY2tzIiwiZGVzY3JpcHRpb24iOiJjb25zZXF1YXQgYW1ldCBjdXBpZGF0YXQgbWluaW0gbGFib3J1bSB0ZW1wb3IgZWxpdCBleCBjb25zZXF1YXQgaW4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2NrczIuanBnIl0sInByaWNlIjoxNSwiY291bnQiOjE3NSwidGFnIjpbImJyb3duIiwiZm9ybWFsIiwiZ3JlZW4iXX1dCg==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1927}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 19}}
 {"_id": "", "startedDateTime": "2021-07-26T11:14:17.78418182Z", "time": 7, "request": {"method": "GET", "url": "http://catalogue/tags", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "107"}], "content": {"size": 107, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInRhZ3MiOlsiYnJvd24iLCJnZWVrIiwiZm9ybWFsIiwiYmx1ZSIsInNraW4iLCJyZWQiLCJhY3Rpb24iLCJzcG9ydCIsImJsYWNrIiwibWFnaWMiLCJncmVlbiJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 107}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 7}}

agent/pkg/oas/test_artifacts/params.har

@@ -0,0 +1,544 @@
+{
+  "log": {
+    "version": "1.2",
+    "creator": {
+      "name": "mitmproxy har_dump",
+      "version": "0.1",
+      "comment": "mitmproxy version mitmproxy 4.0.4"
+    },
+    "entries": [
+      {
+        "startedDateTime": "2019-09-06T06:14:43.864529+00:00",
+        "time": 111,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/e21f7112-3d3b-4632-9da3-a4af2e0e9166/sub1",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "headersSize": 1542,
+          "bodySize": 0,
+          "queryString": []
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "mimeType": "text/html",
+            "text": "",
+            "size": 0
+          },
+          "redirectURL": "",
+          "headersSize": 245,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 22,
+          "receive": 2,
+          "wait": 87,
+          "connect": -1,
+          "ssl": -1
+        },
+        "serverIPAddress": "54.210.29.33"
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:18.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a/sub2",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:19.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a;mparam=matrixparam",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:20.047122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/appears-once",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:20.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/appears-twice",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/appears-twice",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:20.747122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/form-urlencoded",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "Content-Type",
+              "value": "application/x-www-form-urlencoded"
+            }
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "agent-id=ade&callback-url=&token=sometoken"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/form-urlencoded",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "Content-Type",
+              "value": "application/x-www-form-urlencoded"
+            }
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "agent-id=ade&callback-url=&token=sometoken-second-val&optional=another"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:22.747122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/form-multipart",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "Content-Type",
+              "value": "multipart/form-data; boundary=BOUNDARY"
+            }
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 62,
+            "mimeType": "",
+            "text": "{}"
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 62
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/body-optional",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/body-optional",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "Content-Type",
+              "value": "application/json"
+            }
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "{\"key\", \"val\"}"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/body-optional",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/body-required",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "body exists"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      }
+    ]
+  }
+}

agent/pkg/oas/test_artifacts/params.har.spec.json

@@ -0,0 +1,507 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "https://httpbin.org",
+    "description": "Mizu observed 13 entries (0 failed), at 0.155 hits/s, average response time is 0.251 seconds",
+    "version": "1.0"
+  },
+  "servers": [
+    {
+      "url": "https://httpbin.org"
+    }
+  ],
+  "paths": {
+    "/appears-once": {
+      "get": {
+        "summary": "/appears-once",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "89aa39f6-78d0-411b-b701-a33bd77868b0",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null
+              }
+            }
+          }
+        },
+        "x-last-seen-ts": 1567750580.0471218,
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750580.0471218,
+          "lastSeen": 1567750580.0471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750580.0471218,
+            "lastSeen": 1567750580.0471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-sample-entry": 0
+      }
+    },
+    "/appears-twice": {
+      "get": {
+        "summary": "/appears-twice",
+        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds",
+        "operationId": "f5e2b5a2-e01a-45f4-bde1-15a7e7a06d3c",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null
+              }
+            }
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750580.7471218,
+          "lastSeen": 1567750581.7471218,
+          "sumRT": 1.26,
+          "sumDuration": 1
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750580.7471218,
+            "lastSeen": 1567750581.7471218,
+            "sumRT": 1.26,
+            "sumDuration": 1
+          }
+        },
+        "x-sample-entry": 0,
+        "x-last-seen-ts": 1567750581.7471218
+      }
+    },
+    "/body-optional": {
+      "post": {
+        "summary": "/body-optional",
+        "description": "Mizu observed 3 entries (0 failed), at 0.003 hits/s, average response time is 0.001 seconds",
+        "operationId": "14d5b1c2-dc03-4ee5-baaa-5c7992acc82e",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {}
+            }
+          }
+        },
+        "x-counters-total": {
+          "entries": 3,
+          "failures": 0,
+          "firstSeen": 1567750581.7471218,
+          "lastSeen": 1567750581.757122,
+          "sumRT": 0.003,
+          "sumDuration": 0.010000228881835938
+        },
+        "x-sample-entry": 0,
+        "x-counters-per-source": {
+          "": {
+            "entries": 3,
+            "failures": 0,
+            "firstSeen": 1567750581.7471218,
+            "lastSeen": 1567750581.757122,
+            "sumRT": 0.003,
+            "sumDuration": 0.010000228881835938
+          }
+        },
+        "x-last-seen-ts": 1567750581.757122,
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "application/json": {
+              "example": "{\"key\", \"val\"}"
+            }
+          }
+        }
+      }
+    },
+    "/body-required": {
+      "post": {
+        "summary": "/body-required",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "d0958c5a-dce6-4616-99f4-201dbc51457a",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {}
+            }
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750581.757122,
+            "lastSeen": 1567750581.757122,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-sample-entry": 0,
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750581.757122,
+          "lastSeen": 1567750581.757122,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750581.757122,
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "": {
+              "example": "body exists"
+            }
+          },
+          "required": true
+        }
+      }
+    },
+    "/form-multipart": {
+      "post": {
+        "summary": "/form-multipart",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "cab5a2f3-c18a-4d5a-8f92-e40da4fd6603",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "example": {}
+              }
+            }
+          }
+        },
+        "x-sample-entry": 0,
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.7471218,
+            "lastSeen": 1567750582.7471218,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-last-seen-ts": 1567750582.7471218,
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.7471218,
+          "lastSeen": 1567750582.7471218,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "multipart/form-data": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "file",
+                  "path"
+                ],
+                "properties": {
+                  "file": {
+                    "type": "string",
+                    "contentMediaType": "application/json",
+                    "examples": [
+                      "{\"functions\": 123}"
+                    ]
+                  },
+                  "path": {
+                    "type": "string",
+                    "examples": [
+                      "/content/components"
+                    ]
+                  }
+                }
+              },
+              "example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n"
+            }
+          },
+          "required": true
+        }
+      }
+    },
+    "/form-urlencoded": {
+      "post": {
+        "summary": "/form-urlencoded",
+        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds",
+        "operationId": "7c373ad7-6ab5-422e-971b-1cf56b18a7a2",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {}
+            }
+          }
+        },
+        "x-last-seen-ts": 1567750581.7471218,
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750580.7471218,
+          "lastSeen": 1567750581.7471218,
+          "sumRT": 0.002,
+          "sumDuration": 1
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750580.7471218,
+            "lastSeen": 1567750581.7471218,
+            "sumRT": 0.002,
+            "sumDuration": 1
+          }
+        },
+        "x-sample-entry": 0,
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "application/x-www-form-urlencoded": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "agent-id",
+                  "callback-url",
+                  "token"
+                ],
+                "properties": {
+                  "agent-id": {
+                    "type": "string",
+                    "examples": [
+                      "ade"
+                    ]
+                  },
+                  "callback-url": {
+                    "type": "string",
+                    "examples": [
+                      ""
+                    ]
+                  },
+                  "optional": {
+                    "type": "string",
+                    "examples": [
+                      "another"
+                    ]
+                  },
+                  "token": {
+                    "type": "string",
+                    "examples": [
+                      "sometoken",
+                      "sometoken-second-val"
+                    ]
+                  }
+                }
+              },
+              "example": "agent-id=ade\u0026callback-url=\u0026token=sometoken"
+            }
+          },
+          "required": true
+        }
+      }
+    },
+    "/{Id}": {
+      "get": {
+        "summary": "/{Id}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "99f1d11f-29c0-48f9-8bf0-9f4b407c7c3f",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null
+              }
+            }
+          }
+        },
+        "x-sample-entry": 0,
+        "x-last-seen-ts": 1567750579.7471218,
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750579.7471218,
+          "lastSeen": 1567750579.7471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750579.7471218,
+            "lastSeen": 1567750579.7471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        }
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          }
+        }
+      ]
+    },
+    "/{Id}/sub1": {
+      "get": {
+        "summary": "/{Id}/sub1",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds",
+        "operationId": "f7e299d2-253c-4eef-975c-9a5659a7fc50",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "text/html": {}
+            }
+          }
+        },
+        "x-last-seen-ts": 1567750483.864529,
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750483.864529,
+          "lastSeen": 1567750483.864529,
+          "sumRT": 0.111,
+          "sumDuration": 0
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750483.864529,
+            "lastSeen": 1567750483.864529,
+            "sumRT": 0.111,
+            "sumDuration": 0
+          }
+        },
+        "x-sample-entry": 0
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          }
+        }
+      ]
+    },
+    "/{Id}/sub2": {
+      "get": {
+        "summary": "/{Id}/sub2",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "23a54e06-4298-4ea5-b1f0-09b0354a0598",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null
+              }
+            }
+          }
+        },
+        "x-last-seen-ts": 1567750578.7471218,
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750578.7471218,
+          "lastSeen": 1567750578.7471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750578.7471218,
+            "lastSeen": 1567750578.7471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-sample-entry": 0
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          }
+        }
+      ]
+    }
+  },
+  "x-counters-total": {
+    "entries": 13,
+    "failures": 0,
+    "firstSeen": 1567750483.864529,
+    "lastSeen": 1567750582.7471218,
+    "sumRT": 3.268,
+    "sumDuration": 2.010000228881836
+  },
+  "x-counters-per-source": {
+    "": {
+      "entries": 13,
+      "failures": 0,
+      "firstSeen": 1567750483.864529,
+      "lastSeen": 1567750582.7471218,
+      "sumRT": 3.268,
+      "sumDuration": 2.010000228881836
+    }
+  }
+}

agent/pkg/oas/tree.go

@@ -1,34 +1,48 @@
 package oas
 
 import (
-	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/shared/logger"
+	"net/url"
 	"strconv"
 	"strings"
+
+	"github.com/chanced/openapi"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 type NodePath = []string
 
 type Node struct {
-	constant *string
-	param    *openapi.ParameterObj
-	ops      *openapi.PathObj
-	parent   *Node
-	children []*Node
+	constant  *string
+	pathParam *openapi.ParameterObj
+	pathObj   *openapi.PathObj
+	parent    *Node
+	children  []*Node
 }
 
-func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Node) {
-	if pathObjToSet == nil {
+func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node *Node) {
+	if existingPathObj == nil {
 		panic("Invalid function call")
 	}
 
 	pathChunk := path[0]
+	potentialMatrix := strings.SplitN(pathChunk, ";", 2)
+	if len(potentialMatrix) > 1 {
+		pathChunk = potentialMatrix[0]
+		logger.Log.Warningf("URI matrix params are not supported: %s", potentialMatrix[1])
+	}
+
 	chunkIsParam := strings.HasPrefix(pathChunk, "{") && strings.HasSuffix(pathChunk, "}")
+	pathChunk, err := url.PathUnescape(pathChunk)
+	if err != nil {
+		logger.Log.Warningf("URI segment is not correctly encoded: %s", pathChunk)
+		// any side effects on continuing?
+	}
+
 	chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
 
 	var paramObj *openapi.ParameterObj
-	if chunkIsParam && pathObjToSet != nil && pathObjToSet.Parameters != nil {
-		paramObj = findParamByName(pathObjToSet.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
+	if chunkIsParam && existingPathObj != nil && existingPathObj.Parameters != nil {
+		_, paramObj = findParamByName(existingPathObj.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
 	}
 
 	if paramObj == nil {
@@ -46,34 +60,29 @@ func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Nod
 		n.children = append(n.children, node)
 
 		if paramObj != nil {
-			node.param = paramObj
+			node.pathParam = paramObj
 		} else if chunkIsGibberish {
-			initParams(&pathObjToSet.Parameters)
-
 			newParam := n.createParam()
-			node.param = newParam
-
-			appended := append(*pathObjToSet.Parameters, newParam)
-			pathObjToSet.Parameters = &appended
+			node.pathParam = newParam
 		} else {
 			node.constant = &pathChunk
 		}
 	}
 
-	// add example if it's a param
-	if node.param != nil && !chunkIsParam {
-		exmp := &node.param.Examples
+	// add example if it's a gibberish chunk
+	if node.pathParam != nil && !chunkIsParam {
+		exmp := &node.pathParam.Examples
 		err := fillParamExample(&exmp, pathChunk)
 		if err != nil {
 			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
 		}
 	}
 
-	// TODO: eat up trailing slash, in a smart way: node.ops!=nil && path[1]==""
+	// TODO: eat up trailing slash, in a smart way: node.pathObj!=nil && path[1]==""
 	if len(path) > 1 {
-		return node.getOrSet(path[1:], pathObjToSet)
-	} else if node.ops == nil {
-		node.ops = pathObjToSet
+		return node.getOrSet(path[1:], existingPathObj)
+	} else if node.pathObj == nil {
+		node.pathObj = existingPathObj
 	}
 
 	return node
@@ -90,12 +99,19 @@ func (n *Node) createParam() *openapi.ParameterObj {
 		} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
 			name = *n.constant
 			name = name[:len(name)-1] + "Id"
+		} else {
+			name = *n.constant + "Id"
+		}
+
+		name = cleanStr(name, isAlNumRune)
+		if !isAlphaRune(rune(name[0])) {
+			name = "_" + name
 		}
 	}
 
 	newParam := createSimpleParam(name, "path", "string")
 	x := n.countParentParams()
-	if x > 1 {
+	if x > 0 {
 		newParam.Name = newParam.Name + strconv.Itoa(x)
 	}
 
@@ -113,7 +129,7 @@ func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunkIsGibberish b
 			// TODO: check the regex pattern of param? for exceptions etc
 
 			if paramObj != nil {
-				// TODO: mergeParam(subnode.param, paramObj)
+				// TODO: mergeParam(subnode.pathParam, paramObj)
 				return subnode
 			} else {
 				return subnode
@@ -147,15 +163,14 @@ func (n *Node) listPaths() *openapi.Paths {
 	var strChunk string
 	if n.constant != nil {
 		strChunk = *n.constant
-	} else if n.param != nil {
-		strChunk = "{" + n.param.Name + "}"
-	} else {
-		// this is the root node
-	}
+	} else if n.pathParam != nil {
+		strChunk = "{" + n.pathParam.Name + "}"
+	} // else -> this is the root node
 
 	// add self
-	if n.ops != nil {
-		paths.Items[openapi.PathValue(strChunk)] = n.ops
+	if n.pathObj != nil {
+		fillPathParams(n, n.pathObj)
+		paths.Items[openapi.PathValue(strChunk)] = n.pathObj
 	}
 
 	// recurse into children
@@ -175,6 +190,29 @@ func (n *Node) listPaths() *openapi.Paths {
 	return paths
 }
 
+func fillPathParams(n *Node, pathObj *openapi.PathObj) {
+	// collect all path parameters from parent hierarchy
+	node := n
+	for {
+		if node.pathParam != nil {
+			initParams(&pathObj.Parameters)
+
+			idx, paramObj := findParamByName(pathObj.Parameters, openapi.InPath, node.pathParam.Name)
+			if paramObj == nil {
+				appended := append(*pathObj.Parameters, node.pathParam)
+				pathObj.Parameters = &appended
+			} else {
+				(*pathObj.Parameters)[idx] = paramObj
+			}
+		}
+
+		node = node.parent
+		if node == nil {
+			break
+		}
+	}
+}
+
 type PathAndOp struct {
 	path string
 	op   *openapi.Operation
@@ -194,7 +232,7 @@ func (n *Node) countParentParams() int {
 	res := 0
 	node := n
 	for {
-		if node.param != nil {
+		if node.pathParam != nil {
 			res++
 		}
 

agent/pkg/oas/tree_test.go

@@ -8,19 +8,30 @@ import (
 
 func TestTree(t *testing.T) {
 	testCases := []struct {
-		inp string
+		inp       string
+		numParams int
+		label     string
 	}{
-		{"/"},
-		{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation"},
+		{"/", 0, ""},
+		{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation", 1, "vendor,init,conversation"},
+		{"/v1.0.0/config/launcher/sp_nKNHCzsN/{f34efcae-6583-11eb-908a-00b0fcb9d4f6}/vendor,init,conversation", 0, "vendor,init,conversation"},
+		{"/getSvgs/size/small/brand/SFLY/layoutId/170943/layoutVersion/1/sizeId/742/surface/0/isLandscape/true/childSkus/%7B%7D", 1, "{}"},
 	}
 
 	tree := new(Node)
 	for _, tc := range testCases {
 		split := strings.Split(tc.inp, "/")
-		node := tree.getOrSet(split, new(openapi.PathObj))
+		pathObj := new(openapi.PathObj)
+		node := tree.getOrSet(split, pathObj)
 
-		if node.constant == nil {
-			t.Errorf("nil constant: %s", tc.inp)
+		fillPathParams(node, pathObj)
+
+		if node.constant != nil && *node.constant != tc.label {
+			t.Errorf("Constant does not match: %s != %s", *node.constant, tc.label)
+		}
+
+		if tc.numParams > 0 && (pathObj.Parameters == nil || len(*pathObj.Parameters) < tc.numParams) {
+			t.Errorf("Wrong num of params, expected: %d", tc.numParams)
 		}
 	}
 }

agent/pkg/oas/utils.go

@@ -3,11 +3,13 @@ package oas
 import (
 	"encoding/json"
 	"errors"
-	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/shared/logger"
-	"mizuserver/pkg/har"
 	"strconv"
 	"strings"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
+
+	"github.com/chanced/openapi"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 func exampleResolver(ref string) (*openapi.ExampleObj, error) {
@@ -32,16 +34,14 @@ func headerResolver(ref string) (*openapi.HeaderObj, error) {
 
 func initParams(obj **openapi.ParameterList) {
 	if *obj == nil {
-		var params openapi.ParameterList
-		params = make([]openapi.Parameter, 0)
+		var params openapi.ParameterList = make([]openapi.Parameter, 0)
 		*obj = &params
 	}
 }
 
 func initHeaders(respObj *openapi.ResponseObj) {
 	if respObj.Headers == nil {
-		var created openapi.Headers
-		created = map[string]openapi.Header{}
+		var created openapi.Headers = map[string]openapi.Header{}
 		respObj.Headers = created
 	}
 }
@@ -52,8 +52,7 @@ func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *op
 	}
 	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
 	schema := new(openapi.SchemaObj)
-	schema.Type = make(openapi.Types, 0)
-	schema.Type = append(schema.Type, ptype)
+	schema.Type = openapi.Types{ptype}
 
 	style := openapi.StyleSimple
 	if in == openapi.InQuery {
@@ -71,9 +70,10 @@ func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *op
 	return &newParam
 }
 
-func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (pathParam *openapi.ParameterObj) {
+func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (idx int, pathParam *openapi.ParameterObj) {
 	caseInsensitive := in == openapi.InHeader
-	for _, param := range *params {
+	for i, param := range *params {
+		idx = i
 		paramObj, err := param.ResolveParameter(paramResolver)
 		if err != nil {
 			logger.Log.Warningf("Failed to resolve reference: %s", err)
@@ -84,12 +84,13 @@ func findParamByName(params *openapi.ParameterList, in openapi.In, name string)
 			continue
 		}
 
-		if paramObj.Name == name || (caseInsensitive && strings.ToLower(paramObj.Name) == strings.ToLower(name)) {
+		if paramObj.Name == name || (caseInsensitive && strings.EqualFold(paramObj.Name, name)) {
 			pathParam = paramObj
 			break
 		}
 	}
-	return pathParam
+
+	return idx, pathParam
 }
 
 func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj {
@@ -100,52 +101,31 @@ func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj
 			continue
 		}
 
-		if strings.ToLower(hname) == strings.ToLower(name) {
+		if strings.EqualFold(hname, name) {
 			return hdrObj
 		}
 	}
 	return nil
 }
 
-type NVPair struct {
-	Name  string
-	Value string
-}
-
 type nvParams struct {
 	In             openapi.In
-	Pairs          func() []NVPair
+	Pairs          []har.NVP
 	IsIgnored      func(name string) bool
 	GeneralizeName func(name string) string
 }
 
-func qstrToNVP(list []har.QueryString) []NVPair {
-	res := make([]NVPair, len(list))
-	for idx, val := range list {
-		res[idx] = NVPair{Name: val.Name, Value: val.Value}
-	}
-	return res
-}
-
-func hdrToNVP(list []har.Header) []NVPair {
-	res := make([]NVPair, len(list))
-	for idx, val := range list {
-		res[idx] = NVPair{Name: val.Name, Value: val.Value}
-	}
-	return res
-}
-
-func handleNameVals(gw nvParams, params **openapi.ParameterList) {
+func handleNameVals(gw nvParams, params **openapi.ParameterList, checkIgnore bool) {
 	visited := map[string]*openapi.ParameterObj{}
-	for _, pair := range gw.Pairs() {
-		if gw.IsIgnored(pair.Name) {
+	for _, pair := range gw.Pairs {
+		if (checkIgnore && gw.IsIgnored(pair.Name)) || pair.Name == "" {
 			continue
 		}
 
 		nameGeneral := gw.GeneralizeName(pair.Name)
 
 		initParams(params)
-		param := findParamByName(*params, gw.In, pair.Name)
+		_, param := findParamByName(*params, gw.In, pair.Name)
 		if param == nil {
 			param = createSimpleParam(nameGeneral, gw.In, openapi.TypeString)
 			appended := append(**params, param)
@@ -216,7 +196,7 @@ func fillParamExample(param **openapi.Examples, exampleValue string) error {
 			continue
 		}
 
-		if value == exampleValue || cnt > 5 { // 5 examples is enough
+		if value == exampleValue || cnt >= 5 { // 5 examples is enough
 			return nil
 		}
 	}
@@ -232,6 +212,36 @@ func fillParamExample(param **openapi.Examples, exampleValue string) error {
 	return nil
 }
 
+// TODO: somehow generalize the two example setting functions, plus add body example handling
+
+func addSchemaExample(existing *openapi.SchemaObj, bodyStr string) {
+	if len(existing.Examples) < 5 {
+		found := false
+		for _, eVal := range existing.Examples {
+			existingExample := ""
+			err := json.Unmarshal(eVal, &existingExample)
+			if err != nil {
+				logger.Log.Debugf("Failed to unmarshal example: %v", eVal)
+				continue
+			}
+
+			if existingExample == bodyStr {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			example, err := json.Marshal(bodyStr)
+			if err != nil {
+				logger.Log.Debugf("Failed to marshal example: %v", bodyStr)
+				return
+			}
+			existing.Examples = append(existing.Examples, example)
+		}
+	}
+}
+
 func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.com/jpillora/longestcommon
 	empty := make([]string, 0)
 	//short-circuit empty list
@@ -280,6 +290,22 @@ func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.c
 	return xfix
 }
 
+func getSimilarPrefix(strs []string) string {
+	chunked := make([][]string, 0)
+	for _, item := range strs {
+		chunked = append(chunked, strings.Split(item, "/"))
+	}
+
+	cmn := longestCommonXfix(chunked, true)
+	res := make([]string, 0)
+	for _, chunk := range cmn {
+		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
+			res = append(res, chunk)
+		}
+	}
+	return strings.Join(res[1:], ".")
+}
+
 // returns all non-nil ops in PathObj
 func getOps(pathObj *openapi.PathObj) []*openapi.Operation {
 	ops := []**openapi.Operation{&pathObj.Get, &pathObj.Patch, &pathObj.Put, &pathObj.Options, &pathObj.Post, &pathObj.Trace, &pathObj.Head, &pathObj.Delete}
@@ -342,3 +368,62 @@ func anyJSON(text string) (anyVal interface{}, isJSON bool) {
 
 	return nil, false
 }
+
+func cleanStr(str string, criterion func(r rune) bool) string {
+	s := []byte(str)
+	j := 0
+	for _, b := range s {
+		if criterion(rune(b)) {
+			s[j] = b
+			j++
+		}
+	}
+	return string(s[:j])
+}
+
+/*
+func isAlpha(s string) bool {
+	for _, r := range s {
+		if isAlphaRune(r) {
+			return false
+		}
+	}
+	return true
+}
+*/
+
+func isAlphaRune(r rune) bool {
+	return !((r < 'a' || r > 'z') && (r < 'A' || r > 'Z'))
+}
+
+func isAlNumRune(b rune) bool {
+	return isAlphaRune(b) || ('0' <= b && b <= '9')
+}
+
+func deleteFromSlice(s []string, val string) []string {
+	temp := s[:0]
+	for _, x := range s {
+		if x != val {
+			temp = append(temp, x)
+		}
+	}
+	return temp
+}
+
+func sliceContains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}
+
+func intersectSliceWithMap(required []string, names map[string]struct{}) []string {
+	for name := range names {
+		if !sliceContains(required, name) {
+			required = deleteFromSlice(required, name)
+		}
+	}
+	return required
+}

agent/pkg/oas/utils_test.go

@@ -33,3 +33,27 @@ func TestAnyJSON(t *testing.T) {
 		}
 	}
 }
+
+func TestStrRunes(t *testing.T) {
+	if isAlphaRune('5') {
+		t.Logf("Failed")
+	}
+	if !isAlphaRune('a') {
+		t.Logf("Failed")
+	}
+
+	if !isAlNumRune('5') {
+		t.Logf("Failed")
+	}
+	if isAlNumRune(' ') {
+		t.Logf("Failed")
+	}
+
+	if cleanStr("-abc_567", isAlphaRune) != "abc" {
+		t.Logf("Failed")
+	}
+
+	if cleanStr("-abc_567", isAlNumRune) != "abc567" {
+		t.Logf("Failed")
+	}
+}

agent/pkg/providers/install_provider.go

@@ -3,7 +3,10 @@ package providers
 import (
 	"context"
 	"errors"
-	"mizuserver/pkg/config"
+
+	"github.com/up9inc/mizu/agent/pkg/config"
+
+	"github.com/up9inc/mizu/shared/logger"
 
 	ory "github.com/ory/kratos-client-go"
 )
@@ -38,7 +41,9 @@ func CreateAdminUser(password string, ctx context.Context) (token *string, err e
 
 	if err != nil {
 		//Delete the user to prevent a half-setup situation where admin user is created without admin privileges
-		DeleteUser(identityId, ctx)
+		if err := DeleteUser(identityId, ctx); err != nil {
+			logger.Log.Error(err)
+		}
 
 		return nil, err, nil
 	}

agent/pkg/providers/stats_provider_test.go

@@ -2,8 +2,9 @@ package providers_test
 
 import (
 	"fmt"
-	"mizuserver/pkg/providers"
 	"testing"
+
+	"github.com/up9inc/mizu/agent/pkg/providers"
 )
 
 func TestNoEntryAddedCount(t *testing.T) {

agent/pkg/providers/status_provider.go

@@ -3,12 +3,13 @@ package providers
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/patrickmn/go-cache"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap"
-	"mizuserver/pkg/models"
 	"os"
 	"time"
+
+	"github.com/patrickmn/go-cache"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap"
 )
 
 const tlsLinkRetainmentTime = time.Minute * 15

agent/pkg/providers/tapConfig/tap_config_provider.go

@@ -1,12 +1,13 @@
 package tapConfig
 
 import (
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/utils"
 	"os"
 	"sync"
+
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 const FilePath = shared.DataDirPath + "tap-config.json"

agent/pkg/providers/tappedPods/tapped_pods_provider.go

@@ -1,13 +1,14 @@
 package tappedPods
 
 import (
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	"mizuserver/pkg/providers/tappers"
-	"mizuserver/pkg/utils"
 	"os"
 	"strings"
 	"sync"
+
+	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 const FilePath = shared.DataDirPath + "tapped-pods.json"

agent/pkg/providers/tappers/tappers_provider.go

@@ -1,11 +1,12 @@
 package tappers
 
 import (
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	"mizuserver/pkg/utils"
 	"os"
 	"sync"
+
+	"github.com/up9inc/mizu/agent/pkg/utils"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 const FilePath = shared.DataDirPath + "tappers-status.json"

agent/pkg/providers/user_provider.go

@@ -85,11 +85,11 @@ func DeleteUser(identityId string, ctx context.Context) error {
 		return err
 	}
 	if result == nil {
-		return errors.New("unknown error occured during user deletion")
+		return fmt.Errorf("unknown error occured during user deletion %v", identityId)
 	}
 
 	if result.StatusCode < 200 || result.StatusCode > 299 {
-		return errors.New(fmt.Sprintf("user deletion returned bad status %d", result.StatusCode))
+		return fmt.Errorf("user deletion %v returned bad status %d", identityId, result.StatusCode)
 	} else {
 		return nil
 	}

agent/pkg/providers/user_role_provider.go

@@ -15,7 +15,8 @@ more on keto here: https://www.ory.sh/keto/docs/
 
 import (
 	"fmt"
-	"mizuserver/pkg/utils"
+
+	"github.com/up9inc/mizu/agent/pkg/utils"
 
 	ketoClient "github.com/ory/keto-client-go/client"
 	ketoRead "github.com/ory/keto-client-go/client/read"

agent/pkg/routes/config_routes.go

@@ -1,8 +1,8 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/entries_routes.go

@@ -1,8 +1,8 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/install_routes.go

@@ -1,7 +1,7 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/metadata_routes.go

@@ -1,7 +1,7 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/not_found_route.go

@@ -1,18 +0,0 @@
-package routes
-
-import (
-	"github.com/gin-gonic/gin"
-	"net/http"
-)
-
-// NotFoundRoute defines the 404 Error route.
-func NotFoundRoute(app *gin.Engine) {
-	app.Use(
-		func(c *gin.Context) {
-			c.JSON(http.StatusNotFound, map[string]interface{}{
-				"error": true,
-				"msg":   "sorry, endpoint is not found",
-			})
-		},
-	)
-}

agent/pkg/routes/oas_routes.go

@@ -1,8 +1,8 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/query_routes.go

@@ -1,8 +1,8 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/service_map_routes.go

@@ -1,8 +1,8 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/status_routes.go

@@ -1,8 +1,8 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/user_routes.go

@@ -1,7 +1,7 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/rules/rulesHTTP.go

@@ -4,11 +4,12 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/har"
 	"reflect"
 	"regexp"
 	"strings"
 
+	"github.com/up9inc/mizu/agent/pkg/har"
+
 	"github.com/up9inc/mizu/shared/logger"
 
 	"github.com/up9inc/mizu/shared"
@@ -108,7 +109,7 @@ func PassedValidationRules(rulesMatched []RulesMatched) (bool, int64, int) {
 	}
 
 	for _, rule := range rulesMatched {
-		if rule.Matched == false {
+		if !rule.Matched {
 			return false, responseTime, numberOfRulesMatched
 		} else {
 			if strings.ToLower(rule.Rule.Type) == "slo" {

agent/pkg/up9/main.go

@@ -6,8 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"mizuserver/pkg/har"
-	"mizuserver/pkg/utils"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -15,6 +13,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"

agent/pkg/utils/utils.go

@@ -31,10 +31,14 @@ func StartServer(app *gin.Engine) {
 	}
 
 	go func() {
-		_ = <-signals
+		<-signals
 		logger.Log.Infof("Shutting down...")
-		ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
-		_ = srv.Shutdown(ctx)
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			logger.Log.Errorf("%v", err)
+		}
 		os.Exit(0)
 	}()
 
@@ -91,7 +95,7 @@ func UniqueStringSlice(s []string) []string {
 	uniqueMap := map[string]bool{}
 
 	for _, val := range s {
-		if uniqueMap[val] == true {
+		if uniqueMap[val] {
 			continue
 		}
 		uniqueMap[val] = true

agent/pkg/version/consts.go

@@ -1,7 +1,7 @@
 package version
 
 var (
-	SemVer         = "0.0.1"
+	Ver         = "0.0.1"
 	Branch         = "develop"
 	GitCommitHash  = "" // this var is overridden using ldflags in makefile when building
 	BuildTimestamp = "" // this var is overridden using ldflags in makefile when building

cli/Makefile

@@ -3,7 +3,7 @@ COMMIT_HASH=$(shell git rev-parse HEAD)
 GIT_BRANCH=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
 GIT_VERSION=$(shell git branch --show-current | tr '[:upper:]' '[:lower:]')
 BUILD_TIMESTAMP=$(shell date +%s)
-export SEM_VER?=0.0.0
+export VER?=0.0
 
 .PHONY: help
 .DEFAULT_GOAL := help
@@ -23,13 +23,13 @@ build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables
 					   -X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
 					   -X 'github.com/up9inc/mizu/cli/mizu.Platform=$(SUFFIX)' \
-					   -X 'github.com/up9inc/mizu/cli/mizu.SemVer=$(SEM_VER)'" \
+					   -X 'github.com/up9inc/mizu/cli/mizu.Ver=$(VER)'" \
 					   -o bin/mizu_$(SUFFIX) mizu.go
 	(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
 
 build-all: ## Build for all supported platforms.
 	@echo "Compiling for every OS and Platform"
-	@mkdir -p bin && sed s/_SEM_VER_/$(SEM_VER)/g README.md.TEMPLATE >  bin/README.md
+	@mkdir -p bin && sed s/_VER_/$(VER)/g README.md.TEMPLATE >  bin/README.md
 	@$(MAKE) build GOOS=linux GOARCH=amd64
 	@$(MAKE) build GOOS=linux GOARCH=arm64
 	@$(MAKE) build GOOS=darwin GOARCH=amd64

cli/README.md.TEMPLATE

@@ -1,31 +1,31 @@
-# Mizu release _SEM_VER_
+# Mizu release _VER_
 Full changelog for stable release see in [docs](https://github.com/up9inc/mizu/blob/main/docs/CHANGELOG.md)
 
 ## Download Mizu for your platform
 
 **Mac** (x86-64/Intel)
 ```
-curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_darwin_amd64 && chmod 755 mizu
+curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_darwin_amd64 && chmod 755 mizu
 ```
 
 **Mac** (AArch64/Apple M1 silicon)
 ```
-curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_darwin_arm64 && chmod 755 mizu
+curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_darwin_arm64 && chmod 755 mizu
 ```
 
 **Linux** (x86-64)
 ```
-curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_linux_amd64 && chmod 755 mizu
+curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_linux_amd64 && chmod 755 mizu
 ```
 
 **Linux** (AArch64)
 ```
-curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu_linux_arm64 && chmod 755 mizu
+curl -Lo mizu https://github.com/up9inc/mizu/releases/download/_VER_/mizu_linux_arm64 && chmod 755 mizu
 ```
 
 **Windows** (x86-64)
 ```
-curl -LO https://github.com/up9inc/mizu/releases/download/_SEM_VER_/mizu.exe
+curl -LO https://github.com/up9inc/mizu/releases/download/_VER_/mizu.exe
 ```
 
 ### Checksums

cli/apiserver/provider.go

@@ -23,8 +23,8 @@ type Provider struct {
 	client  *http.Client
 }
 
-const DefaultRetries = 20
-const DefaultTimeout = 5 * time.Second
+const DefaultRetries = 3
+const DefaultTimeout = 2 * time.Second
 
 func NewProvider(url string, retries int, timeout time.Duration) *Provider {
 	return &Provider{
@@ -36,16 +36,6 @@ func NewProvider(url string, retries int, timeout time.Duration) *Provider {
 	}
 }
 
-func NewProviderWithoutRetries(url string, timeout time.Duration) *Provider {
-	return &Provider{
-		url:     url,
-		retries: 1,
-		client: &http.Client{
-			Timeout: timeout,
-		},
-	}
-}
-
 func (provider *Provider) TestConnection() error {
 	retriesLeft := provider.retries
 	for retriesLeft > 0 {
@@ -153,5 +143,5 @@ func (provider *Provider) GetVersion() (string, error) {
 		return "", err
 	}
 
-	return versionResponse.SemVer, nil
+	return versionResponse.Ver, nil
 }

cli/cmd/checkRunner.go

@@ -3,13 +3,14 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"regexp"
+
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared/kubernetes"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/semver"
-	"regexp"
 )
 
 func runMizuCheck() {
@@ -96,7 +97,7 @@ func checkServerConnection(kubernetesProvider *kubernetes.Provider) bool {
 
 	serverUrl := GetApiServerUrl()
 
-	apiServerProvider := apiserver.NewProviderWithoutRetries(serverUrl, apiserver.DefaultTimeout)
+	apiServerProvider := apiserver.NewProvider(serverUrl, 1, apiserver.DefaultTimeout)
 	if err := apiServerProvider.TestConnection(); err == nil {
 		logger.Log.Infof("%v found Mizu server tunnel available and connected successfully to API server", fmt.Sprintf(uiUtils.Green, "√"))
 		return true
@@ -169,32 +170,32 @@ func checkAllResourcesExist(ctx context.Context, kubernetesProvider *kubernetes.
 	allResourcesExist := checkResourceExist(config.Config.MizuResourcesNamespace, "namespace", exist, err)
 
 	exist, err = kubernetesProvider.DoesConfigMapExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ConfigMapName)
-	allResourcesExist = checkResourceExist(kubernetes.ConfigMapName, "config map", exist, err)
+	allResourcesExist = checkResourceExist(kubernetes.ConfigMapName, "config map", exist, err) && allResourcesExist
 
 	exist, err = kubernetesProvider.DoesServiceAccountExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName)
-	allResourcesExist = checkResourceExist(kubernetes.ServiceAccountName, "service account", exist, err)
+	allResourcesExist = checkResourceExist(kubernetes.ServiceAccountName, "service account", exist, err) && allResourcesExist
 
 	if config.Config.IsNsRestrictedMode() {
 		exist, err = kubernetesProvider.DoesRoleExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleName)
-		allResourcesExist = checkResourceExist(kubernetes.RoleName, "role", exist, err)
+		allResourcesExist = checkResourceExist(kubernetes.RoleName, "role", exist, err) && allResourcesExist
 
 		exist, err = kubernetesProvider.DoesRoleBindingExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleBindingName)
-		allResourcesExist = checkResourceExist(kubernetes.RoleBindingName, "role binding", exist, err)
+		allResourcesExist = checkResourceExist(kubernetes.RoleBindingName, "role binding", exist, err) && allResourcesExist
 	} else {
 		exist, err = kubernetesProvider.DoesClusterRoleExist(ctx, kubernetes.ClusterRoleName)
-		allResourcesExist = checkResourceExist(kubernetes.ClusterRoleName, "cluster role", exist, err)
+		allResourcesExist = checkResourceExist(kubernetes.ClusterRoleName, "cluster role", exist, err) && allResourcesExist
 
 		exist, err = kubernetesProvider.DoesClusterRoleBindingExist(ctx, kubernetes.ClusterRoleBindingName)
-		allResourcesExist = checkResourceExist(kubernetes.ClusterRoleBindingName, "cluster role binding", exist, err)
+		allResourcesExist = checkResourceExist(kubernetes.ClusterRoleBindingName, "cluster role binding", exist, err) && allResourcesExist
 	}
 
 	exist, err = kubernetesProvider.DoesServiceExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
-	allResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "service", exist, err)
+	allResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "service", exist, err) && allResourcesExist
 
 	if isInstallCommand {
-		allResourcesExist = checkInstallResourcesExist(ctx, kubernetesProvider)
+		allResourcesExist = checkInstallResourcesExist(ctx, kubernetesProvider) && allResourcesExist
 	} else {
-		allResourcesExist = checkTapResourcesExist(ctx, kubernetesProvider)
+		allResourcesExist = checkTapResourcesExist(ctx, kubernetesProvider) && allResourcesExist
 	}
 
 	return allResourcesExist
@@ -205,13 +206,13 @@ func checkInstallResourcesExist(ctx context.Context, kubernetesProvider *kuberne
 	installResourcesExist := checkResourceExist(kubernetes.DaemonRoleName, "role", exist, err)
 
 	exist, err = kubernetesProvider.DoesRoleBindingExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleBindingName)
-	installResourcesExist = checkResourceExist(kubernetes.DaemonRoleBindingName, "role binding", exist, err)
+	installResourcesExist = checkResourceExist(kubernetes.DaemonRoleBindingName, "role binding", exist, err) && installResourcesExist
 
 	exist, err = kubernetesProvider.DoesPersistentVolumeClaimExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName)
-	installResourcesExist = checkResourceExist(kubernetes.PersistentVolumeClaimName, "persistent volume claim", exist, err)
+	installResourcesExist = checkResourceExist(kubernetes.PersistentVolumeClaimName, "persistent volume claim", exist, err) && installResourcesExist
 
 	exist, err = kubernetesProvider.DoesDeploymentExist(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName)
-	installResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "deployment", exist, err)
+	installResourcesExist = checkResourceExist(kubernetes.ApiServerPodName, "deployment", exist, err) && installResourcesExist
 
 	return installResourcesExist
 }

cli/cmd/common.go

@@ -5,6 +5,10 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"path"
+	"regexp"
+	"time"
+
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
@@ -13,9 +17,6 @@ import (
 	"github.com/up9inc/mizu/cli/resources"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
-	"path"
-	"regexp"
-	"time"
 
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/shared/kubernetes"
@@ -35,10 +36,10 @@ func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, ctx con
 		return
 	}
 
-	apiProvider = apiserver.NewProviderWithoutRetries(GetApiServerUrl(), time.Second) // short check for proxy
+	apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
 	if err := apiProvider.TestConnection(); err != nil {
 		logger.Log.Debugf("Couldn't connect using proxy, stopping proxy and trying to create port-forward")
-		if err := httpServer.Shutdown(context.Background()); err != nil {
+		if err := httpServer.Shutdown(ctx); err != nil {
 			logger.Log.Debugf("Error occurred while stopping proxy %v", errormessage.FormatError(err))
 		}
 
@@ -50,7 +51,7 @@ func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, ctx con
 			return
 		}
 
-		apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout) // long check for port-forward
+		apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
 		if err := apiProvider.TestConnection(); err != nil {
 			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
 			cancel()

cli/cmd/config.go

@@ -50,7 +50,9 @@ func init() {
 	rootCmd.AddCommand(configCmd)
 
 	defaultConfig := config.ConfigStruct{}
-	defaults.Set(&defaultConfig)
+	if err := defaults.Set(&defaultConfig); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	configCmd.Flags().BoolP(configStructs.RegenerateConfigName, "r", defaultConfig.Config.Regenerate, fmt.Sprintf("Regenerate the config file with default values to path %s or to chosen path using --%s", defaultConfig.ConfigFilePath, config.ConfigFilePathCommandName))
 }

cli/cmd/installRunner.go

@@ -31,7 +31,9 @@ func runMizuInstall() {
 	var defaultMaxEntriesDBSizeBytes int64 = 200 * 1000 * 1000
 
 	defaultResources := shared.Resources{}
-	defaults.Set(&defaultResources)
+	if err := defaults.Set(&defaultResources); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	mizuAgentConfig := getInstallMizuAgentConfig(defaultMaxEntriesDBSizeBytes, defaultResources)
 	serializedMizuConfig, err := getSerializedMizuAgentConfig(mizuAgentConfig)
@@ -74,6 +76,7 @@ func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Reso
 		StandaloneMode:         true,
 		ServiceMap:             config.Config.ServiceMap,
 		OAS:                    config.Config.OAS,
+		Elastic:                config.Config.Elastic,
 	}
 
 	return &mizuAgentConfig

cli/cmd/logs.go

@@ -23,7 +23,7 @@ var logsCmd = &cobra.Command{
 		if err != nil {
 			return nil
 		}
-		ctx, _ := context.WithCancel(context.Background())
+		ctx := context.Background()
 
 		if validationErr := config.Config.Logs.Validate(); validationErr != nil {
 			return errormessage.FormatError(validationErr)
@@ -43,7 +43,9 @@ func init() {
 	rootCmd.AddCommand(logsCmd)
 
 	defaultLogsConfig := configStructs.LogsConfig{}
-	defaults.Set(&defaultLogsConfig)
+	if err := defaults.Set(&defaultLogsConfig); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	logsCmd.Flags().StringP(configStructs.FileLogsName, "f", defaultLogsConfig.FileStr, "Path for zip file (default current <pwd>\\mizu_logs.zip)")
 }

cli/cmd/root.go

@@ -30,7 +30,9 @@ Further info is available at https://github.com/up9inc/mizu`,
 
 func init() {
 	defaultConfig := config.ConfigStruct{}
-	defaults.Set(&defaultConfig)
+	if err := defaults.Set(&defaultConfig); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	rootCmd.PersistentFlags().StringSlice(config.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", config.SetCommandName))
 	rootCmd.PersistentFlags().String(config.ConfigFilePathCommandName, defaultConfig.ConfigFilePath, fmt.Sprintf("Override config file path using --%s", config.ConfigFilePathCommandName))

cli/cmd/tap.go

@@ -104,7 +104,9 @@ func init() {
 	rootCmd.AddCommand(tapCmd)
 
 	defaultTapConfig := configStructs.TapConfig{}
-	defaults.Set(&defaultTapConfig)
+	if err := defaults.Set(&defaultTapConfig); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
 	tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")

cli/cmd/tapRunner.go

@@ -164,6 +164,7 @@ func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 		AgentDatabasePath:      shared.DataDirPath,
 		ServiceMap:             config.Config.ServiceMap,
 		OAS:                    config.Config.OAS,
+		Elastic:                config.Config.Elastic,
 	}
 
 	return &mizuAgentConfig
@@ -342,7 +343,7 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 
 				if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
 					isPodReady = true
-					postApiServerStarted(ctx, kubernetesProvider, cancel, err)
+					postApiServerStarted(ctx, kubernetesProvider, cancel)
 				}
 			case kubernetes.EventBookmark:
 				break
@@ -405,7 +406,7 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
 			case "FailedScheduling", "Failed":
 				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Mizu API Server status: %s - %s", event.Reason, event.Note))
 				cancel()
-				break
+				
 			}
 		case err, ok := <-errorChan:
 			if !ok {
@@ -421,11 +422,11 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
 	}
 }
 
-func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, err error) {
+func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	startProxyReportErrorIfAny(kubernetesProvider, ctx, cancel)
 
 	options, _ := getMizuApiFilteringOptions()
-	if err = startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
+	if err := startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", err))
 		cancel()
 	}

cli/cmd/version.go

@@ -22,11 +22,11 @@ var versionCmd = &cobra.Command{
 
 		if config.Config.Version.DebugInfo {
 			timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
-			logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
+			logger.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.Ver, mizu.Branch, mizu.GitCommitHash)
 			logger.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
 
 		} else {
-			logger.Log.Infof("Version: %s (%s)", mizu.SemVer, mizu.Branch)
+			logger.Log.Infof("Version: %s (%s)", mizu.Ver, mizu.Branch)
 		}
 		return nil
 	},
@@ -36,7 +36,9 @@ func init() {
 	rootCmd.AddCommand(versionCmd)
 
 	defaultVersionConfig := configStructs.VersionConfig{}
-	defaults.Set(&defaultVersionConfig)
+	if err := defaults.Set(&defaultVersionConfig); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	versionCmd.Flags().BoolP(configStructs.DebugInfoVersionName, "d", defaultVersionConfig.DebugInfo, "Provide all information about version")
 

cli/cmd/view.go

@@ -6,6 +6,7 @@ import (
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/telemetry"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var viewCmd = &cobra.Command{
@@ -22,10 +23,14 @@ func init() {
 	rootCmd.AddCommand(viewCmd)
 
 	defaultViewConfig := configStructs.ViewConfig{}
-	defaults.Set(&defaultViewConfig)
+	if err := defaults.Set(&defaultViewConfig); err != nil {
+		logger.Log.Debug(err)
+	}
 
 	viewCmd.Flags().Uint16P(configStructs.GuiPortViewName, "p", defaultViewConfig.GuiPort, "Provide a custom port for the web interface webserver")
 	viewCmd.Flags().StringP(configStructs.UrlViewName, "u", defaultViewConfig.Url, "Provide a custom host")
 
-	viewCmd.Flags().MarkHidden(configStructs.UrlViewName)
+	if err := viewCmd.Flags().MarkHidden(configStructs.UrlViewName); err != nil {
+		logger.Log.Debug(err)
+	}
 }

cli/config/configStruct.go

@@ -38,8 +38,9 @@ type ConfigStruct struct {
 	ConfigFilePath         string                      `yaml:"config-path,omitempty" readonly:""`
 	HeadlessMode           bool                        `yaml:"headless" default:"false"`
 	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
-	ServiceMap             bool                        `yaml:"service-map,omitempty" default:"false" readonly:""`
+	ServiceMap             bool                        `yaml:"service-map" default:"false"`
 	OAS                    bool                        `yaml:"oas,omitempty" default:"false" readonly:""`
+	Elastic                shared.ElasticConfig        `yaml:"elastic"`
 }
 
 func (config *ConfigStruct) validate() error {
@@ -53,7 +54,7 @@ func (config *ConfigStruct) validate() error {
 func (config *ConfigStruct) SetDefaults() {
 	config.KratosImage = shared.KratosImageDefault
 	config.KetoImage = shared.KetoImageDefault
-	config.AgentImage = fmt.Sprintf("%s:%s", shared.MizuAgentImageRepo, mizu.SemVer)
+	config.AgentImage = fmt.Sprintf("%s:%s", shared.MizuAgentImageRepo, mizu.Ver)
 	config.ConfigFilePath = path.Join(mizu.GetMizuFolderPath(), "config.yaml")
 }
 

cli/config/configStructs/tapConfig.go

@@ -60,12 +60,12 @@ func (config *TapConfig) MaxEntriesDBSizeBytes() int64 {
 func (config *TapConfig) Validate() error {
 	_, compileErr := regexp.Compile(config.PodRegexStr)
 	if compileErr != nil {
-		return errors.New(fmt.Sprintf("%s is not a valid regex %s", config.PodRegexStr, compileErr))
+		return fmt.Errorf("%s is not a valid regex %s", config.PodRegexStr, compileErr)
 	}
 
 	_, parseHumanDataSizeErr := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
 	if parseHumanDataSizeErr != nil {
-		return errors.New(fmt.Sprintf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize))
+		return fmt.Errorf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize)
 	}
 
 	if config.Workspace != "" {
@@ -76,7 +76,7 @@ func (config *TapConfig) Validate() error {
 	}
 
 	if config.Analysis && config.Workspace != "" {
-		return errors.New(fmt.Sprintf("Can't run with both --%s and --%s flags", AnalysisTapName, WorkspaceTapName))
+		return fmt.Errorf("Can't run with both --%s and --%s flags", AnalysisTapName, WorkspaceTapName)
 	}
 
 	return nil

cli/mizu/consts.go

@@ -6,7 +6,7 @@ import (
 )
 
 var (
-	SemVer                                     = "0.0.1"
+	Ver                                        = "0.0"
 	Branch                                     = "develop"
 	GitCommitHash                              = "" // this var is overridden using ldflags in makefile when building
 	BuildTimestamp                             = "" // this var is overridden using ldflags in makefile when building

cli/mizu/fsUtils/dirUtils.go

@@ -1,7 +1,6 @@
 package fsUtils
 
 import (
-	"errors"
 	"fmt"
 	"os"
 )
@@ -18,7 +17,7 @@ func EnsureDir(dirName string) error {
 			return err
 		}
 		if !info.IsDir() {
-			return errors.New(fmt.Sprintf("path exists but is not a directory: %s", dirName))
+			return fmt.Errorf("path exists but is not a directory: %s", dirName)
 		}
 		return nil
 	}

cli/mizu/version/versionCheck.go

@@ -12,31 +12,28 @@ import (
 
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/mizu"
+	"github.com/up9inc/mizu/cli/pkg/version"
 	"github.com/up9inc/mizu/shared/logger"
 
 	"github.com/google/go-github/v37/github"
 	"github.com/up9inc/mizu/cli/uiUtils"
-	"github.com/up9inc/mizu/shared/semver"
 )
 
 func CheckVersionCompatibility(apiServerProvider *apiserver.Provider) (bool, error) {
-	apiSemVer, err := apiServerProvider.GetVersion()
+	apiVer, err := apiServerProvider.GetVersion()
 	if err != nil {
 		return false, err
 	}
 
-	if !semver.SemVersion(apiSemVer).IsValid() {
-		logger.Log.Errorf(uiUtils.Red, fmt.Sprintf("api version (%s) is not a valid SemVer", apiSemVer))
+	if equals, err := version.AreEquals(apiVer, mizu.Ver); err != nil {
+		return false, fmt.Errorf("Failed to check version equality between mizuVer: %s and apiVer: %s, error: %w", mizu.Ver, apiVer, err)
+	} else if !equals {
+		logger.Log.Errorf(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", mizu.Ver, apiVer))
 		return false, nil
 	}
 
-	if semver.SemVersion(apiSemVer).Major() == semver.SemVersion(mizu.SemVer).Major() &&
-		semver.SemVersion(apiSemVer).Minor() == semver.SemVersion(mizu.SemVer).Minor() {
-		return true, nil
-	}
-
-	logger.Log.Errorf(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", mizu.SemVer, apiSemVer))
-	return false, nil
+	logger.Log.Debug("cli version %s is compatible with api version %s", mizu.Ver, apiVer)
+	return true, nil
 }
 
 func CheckNewerVersion(versionChan chan string) {
@@ -84,24 +81,23 @@ func CheckNewerVersion(versionChan chan string) {
 	gitHubVersion := string(data)
 	gitHubVersion = gitHubVersion[:len(gitHubVersion)-1]
 
-	gitHubVersionSemVer := semver.SemVersion(gitHubVersion)
-	currentSemVer := semver.SemVersion(mizu.SemVer)
-	if !gitHubVersionSemVer.IsValid() || !currentSemVer.IsValid() {
-		logger.Log.Debugf("[ERROR] Semver version is not valid, github version %v, current version %v", gitHubVersion, currentSemVer)
+	greater, err := version.GreaterThen(gitHubVersion, mizu.Ver)
+	if err != nil {
+		logger.Log.Debugf("[ERROR] Ver version is not valid, github version %v, current version %v", gitHubVersion, mizu.Ver)
 		versionChan <- ""
 		return
 	}
 
-	logger.Log.Debugf("Finished version validation, github version %v, current version %v, took %v", gitHubVersion, currentSemVer, time.Since(start))
+	logger.Log.Debugf("Finished version validation, github version %v, current version %v, took %v", gitHubVersion, mizu.Ver, time.Since(start))
 
-	if gitHubVersionSemVer.GreaterThan(currentSemVer) {
+	if greater {
 		var downloadMessage string
 		if runtime.GOOS == "windows" {
 			downloadMessage = fmt.Sprintf("curl -LO %v/mizu.exe", strings.Replace(*latestRelease.HTMLURL, "tag", "download", 1))
 		} else {
 			downloadMessage = fmt.Sprintf("curl -Lo mizu %v/mizu_%s_%s && chmod 755 mizu", strings.Replace(*latestRelease.HTMLURL, "tag", "download", 1), runtime.GOOS, runtime.GOARCH)
 		}
-		versionChan <- fmt.Sprintf("Update available! %v -> %v (%s)", mizu.SemVer, gitHubVersion, downloadMessage)
+		versionChan <- fmt.Sprintf("Update available! %v -> %v (%s)", mizu.Ver, gitHubVersion, downloadMessage)
 	} else {
 		versionChan <- ""
 	}

cli/pkg/version/version.go

@@ -0,0 +1,85 @@
+package version
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+)
+
+type Version struct {
+	Major       int
+	Patch       int
+	Incremental int
+}
+
+func Parse(ver string) (*Version, error) {
+	re := regexp.MustCompile(`^(\d+)\.(\d+)(?:-\w+(\d+))?$`)
+	match := re.FindStringSubmatch(ver)
+	if len(match) != 4 {
+		return nil, fmt.Errorf("invalid format expected <major>.<patch>(-<suffix><incremental>)? %s,", ver)
+	}
+	major, err := strconv.Atoi(match[1])
+	if err != nil {
+		return nil, fmt.Errorf("error parsing major int: %s, err %w", match[1], err)
+	}
+	patch, err := strconv.Atoi(match[2])
+	if err != nil {
+		return nil, fmt.Errorf("error parsing patch int: %s, err %w", match[2], err)
+	}
+
+	if match[3] == "" {
+		return &Version{Major: major, Patch: patch, Incremental: -1}, nil
+	}
+
+	inc, err := strconv.Atoi(match[3])
+	if err != nil {
+		return nil, fmt.Errorf("Error parsing incremental int: %s, err %w", match[3], err)
+	}
+	return &Version{Major: major, Patch: patch, Incremental: inc}, nil
+
+}
+
+func AreEquals(first string, second string) (bool, error) {
+	firstVer, err := Parse(first)
+	if err != nil {
+		return false, fmt.Errorf("Failed parsing fist version: %s, error: %w", first, err)
+	}
+	secondVer, err := Parse(second)
+	if err != nil {
+		return false, fmt.Errorf("Failed parsing second version: %s, error: %w", second, err)
+	}
+
+	return *firstVer == *secondVer, nil
+}
+
+func GreaterThen(first string, second string) (bool, error) {
+	firstVer, err := Parse(first)
+	if err != nil {
+		return false, fmt.Errorf("Failed parsing fist version: %s, error: %w", first, err)
+	}
+	secondVer, err := Parse(second)
+	if err != nil {
+		return false, fmt.Errorf("Failed parsing second version: %s, error: %w", second, err)
+	}
+
+	if firstVer.Major > secondVer.Major {
+		return true, nil
+	} else if firstVer.Major < secondVer.Major {
+		return false, nil
+	}
+
+	if firstVer.Patch > secondVer.Patch {
+		return true, nil
+	} else if firstVer.Patch < secondVer.Patch {
+		return false, nil
+	}
+
+	if firstVer.Incremental == -1 && secondVer.Incremental > -1 {
+		return true, nil
+	}
+
+	if firstVer.Incremental > secondVer.Incremental {
+		return true, nil
+	}
+	return false, nil
+}

cli/pkg/version/version_test.go

@@ -0,0 +1,104 @@
+package version
+
+import (
+	"testing"
+)
+
+func TestEqualsEquality(t *testing.T) {
+	tests := []struct {
+		Name   string
+		First  string
+		Second string
+	}{
+		{Name: "major", First: "1.0", Second: "1.0"},
+		{Name: "patch", First: "1.1", Second: "1.1"},
+		{Name: "incremental", First: "1.0-dev0", Second: "1.0-dev0"},
+	}
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			if equal, _ := AreEquals(test.First, test.Second); !equal {
+				t.Fatalf("Expected %s == %s", test.First, test.Second)
+			}
+		})
+	}
+}
+
+func TestEqualsInvalidVersion(t *testing.T) {
+	tests := []struct {
+		Name   string
+		First  string
+		Second string
+	}{
+		{Name: "first semver", First: "1.0.0", Second: "1.0"},
+		{Name: "second semver", First: "1.1", Second: "1.1.0"},
+		{Name: "incremental invalid", First: "1.0-dev0de", Second: "1.0-dev0"},
+	}
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			if _, err := AreEquals(test.First, test.Second); err == nil {
+				t.Fatalf("Expected error")
+			}
+		})
+	}
+}
+
+func TestEqualsNoEquality(t *testing.T) {
+	tests := []struct {
+		Name   string
+		First  string
+		Second string
+	}{
+		{Name: "major", First: "1.0", Second: "2.0"},
+		{Name: "patch", First: "1.0", Second: "1.1"},
+		{Name: "incremental", First: "1.0-dev2", Second: "1.0-dev3"},
+	}
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			if equal, _ := AreEquals(test.First, test.Second); equal {
+				t.Fatalf("Expected %s != %s", test.First, test.Second)
+			}
+		})
+	}
+}
+
+func TestGreaterThenGreater(t *testing.T) {
+	tests := []struct {
+		Name   string
+		First  string
+		Second string
+	}{
+		{Name: "major", First: "2.0", Second: "1.0"},
+		{Name: "patch", First: "1.1", Second: "1.0"},
+		{Name: "incremental", First: "1.0-dev1", Second: "1.0-dev0"},
+		{Name: "major vs incremental", First: "1.0", Second: "1.0-dev1"},
+	}
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			if greater, _ := GreaterThen(test.First, test.Second); !greater {
+				t.Fatalf("Expected %s > %s", test.First, test.Second)
+			}
+		})
+	}
+}
+
+func TestGreaterThenLessThen(t *testing.T) {
+	tests := []struct {
+		Name   string
+		First  string
+		Second string
+	}{
+		{Name: "major", First: "1.0", Second: "2.0"},
+		{Name: "major equals", First: "1.0", Second: "1.0"},
+		{Name: "patch", First: "1.0", Second: "1.1"},
+		{Name: "patch equals", First: "1.1", Second: "1.1"},
+		{Name: "incremental", First: "1.0-dev0", Second: "1.0-dev1"},
+		{Name: "incremental equals", First: "1.0-dev0", Second: "1.0-dev0"},
+	}
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			if greater, _ := GreaterThen(test.First, test.Second); greater {
+				t.Fatalf("Expected %s < %s", test.First, test.Second)
+			}
+		})
+	}
+}

cli/telemetry/telemetry.go

@@ -83,7 +83,7 @@ func sendTelemetry(argsMap map[string]interface{}) error {
 	argsMap["component"] = "mizu_cli"
 	argsMap["buildTimestamp"] = mizu.BuildTimestamp
 	argsMap["branch"] = mizu.Branch
-	argsMap["version"] = mizu.SemVer
+	argsMap["version"] = mizu.Ver
 	argsMap["platform"] = mizu.Platform
 
 	if machineId, err := machineid.ProtectedID("mizu"); err == nil {