Implement AMQP request-response matcher (#1091)

* Implement the basis of AMQP request-response matching

* Fix `package.json`

* Add `ExchangeDeclareOk`

* Add `ConnectionCloseOk`

* Add `BasicConsumeOk`

* Add `QueueBindOk`

* Add `representEmptyResponse` and fix `BasicPublish` and `BasicDeliver`

* Fix ident and matcher, add `connectionOpen`, `channelOpen`, `connectionTune`, `basicCancel`

* Fix linter

* Fix the unit tests

* #run_acceptance_tests

* #run_acceptance_tests

* Fix the tests #run_acceptance_tests

* Log don't panic

* Don't skip AMQP acceptance tests #run_acceptance_tests

* Revert "Don't skip AMQP acceptance tests #run_acceptance_tests"

This reverts commit c60e9cf747.

* Remove `Details` section from `representEmpty`

* Add `This request or response has no data.` text

.dockerignore

@@ -2,7 +2,6 @@
 .dockerignore
 .editorconfig
 .gitignore
-.env.*
 Dockerfile
 Makefile
 LICENSE

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Run `mizu <command> ...`
+2. Click on '...'
+3. Scroll down to '...'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Logs**
+Upload logs:
+1. Run the mizu command with `--set dump-logs=true` (e.g `mizu tap --set dump-logs=true`)
+2. Try to reproduce the issue
+3. <kbd>CTRL</kbd>+<kbd>C</kbd> on terminal tab which runs `mizu`
+4. Upload the logs zip file from `~/.mizu/mizu_logs_**.zip`
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. macOS]
+ - Web Browser: [e.g. Google Chrome]
+
+**Additional context**
+Add any other context about the problem here.

.github/workflows/acceptance_tests.yml

@@ -0,0 +1,66 @@
+name: acceptance tests
+
+on:
+  pull_request:
+    branches:
+      - 'main'
+  push:
+    branches:
+      - 'develop'
+
+env:
+  MIZU_CI_IMAGE: mizu/ci:0.0
+
+jobs:
+  run-acceptance-tests:
+    name: Run acceptance tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          load: true
+          tags: ${{ env.MIZU_CI_IMAGE }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Setup acceptance test
+        run: ./acceptanceTests/setup.sh
+
+      - name: Create k8s users and change context
+        env:
+          USERNAME_UNRESTRICTED: user-with-clusterwide-access
+          USERNAME_RESTRICTED: user-with-restricted-access
+        run: |
+          ./acceptanceTests/create_user.sh "${USERNAME_UNRESTRICTED}"
+          ./acceptanceTests/create_user.sh "${USERNAME_RESTRICTED}"
+          kubectl apply -f cli/cmd/permissionFiles/permissions-all-namespaces-tap.yaml
+          kubectl config use-context ${USERNAME_UNRESTRICTED}
+
+      - name: Test
+        run: make acceptance-test
+
+      - name: Slack notification on failure
+        uses: ravsamhq/notify-slack-action@v1
+        if: always()
+        with:
+          status: ${{ job.status }}
+          notification_title: 'Mizu {workflow} has {status_message}'
+          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit <{commit_url}|{commit_sha}> by ${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}> ```${{ github.event.head_commit.message }}```'
+          footer: 'Linked Repo <{repo_url}|{repo}>'
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/acceptance_tests_on_pr.yml

@@ -0,0 +1,54 @@
+name: Acceptance tests on PR
+
+on: push
+
+env:
+  MIZU_CI_IMAGE: mizu/ci:0.0
+
+concurrency:
+  group: acceptance-tests-on-pr-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-tests:
+    name: Run tests
+    runs-on: ubuntu-latest
+    if: ${{ contains(github.event.head_commit.message, '#run_acceptance_tests') }}
+
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          load: true
+          tags: ${{ env.MIZU_CI_IMAGE }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Setup acceptance test
+        run: ./acceptanceTests/setup.sh
+
+      - name: Create k8s users and change context
+        env:
+          USERNAME_UNRESTRICTED: user-with-clusterwide-access
+          USERNAME_RESTRICTED: user-with-restricted-access
+        run: |
+          ./acceptanceTests/create_user.sh "${USERNAME_UNRESTRICTED}"
+          ./acceptanceTests/create_user.sh "${USERNAME_RESTRICTED}"
+          kubectl apply -f cli/cmd/permissionFiles/permissions-all-namespaces-tap.yaml
+          kubectl config use-context ${USERNAME_UNRESTRICTED}
+
+      - name: Test
+        run: make acceptance-test

.github/workflows/build-custom-branch.yml

@@ -0,0 +1,44 @@
+name: Build Custom Branch
+
+on: push
+
+concurrency:
+  group: custom-branch-build-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Push custom branch image to GCR
+    runs-on: ubuntu-latest
+    if: ${{ contains(github.event.head_commit.message, '#build_and_publish_custom_image') }}
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
+        with:
+          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v0'
+
+      - name: Get base image name
+        shell: bash
+        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
+        id: base_image_step
+
+      - name: Login to GCR
+        uses: docker/login-action@v1
+        with:
+          registry: gcr.io
+          username: _json_key
+          password: ${{ secrets.GCR_JSON_KEY }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.base_image_step.outputs.image }}:latest

.github/workflows/build.yml

@@ -0,0 +1,62 @@
+name: Build
+
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+
+concurrency:
+  group: mizu-pr-validation-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-cli:
+    name: CLI executable build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      
+      - name: Check modified files
+        id: modified_files
+        run: devops/check_modified_files.sh cli/
+
+      - name: Set up Go 1.17
+        if: steps.modified_files.outputs.matched == 'true'
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - name: Build CLI
+        if: steps.modified_files.outputs.matched == 'true'
+        run: make cli
+
+  build-agent:
+    name: Agent docker image build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      
+      - name: Check modified files
+        id: modified_files
+        run: devops/check_modified_files.sh agent/ shared/ tap/ ui/ ui-common/ Dockerfile
+
+      - name: Set up Docker Buildx
+        if: steps.modified_files.outputs.matched == 'true'
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        if: steps.modified_files.outputs.matched == 'true'
+        with:
+          context: .
+          push: false
+          tags: up9inc/mizu:devlatest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/inactive-issues-close.yaml

@@ -0,0 +1,23 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v3
+        with:
+          days-before-issue-stale: 30
+          days-before-issue-close: -1
+          exempt-issue-labels: "enhancement"
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
+          close-issue-message: ""
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/publish-cli.yml

@@ -1,47 +0,0 @@
-name: public-cli
-on:
-  push:
-    branches:
-      - develop
-      - main
-      - my-temp-release-check
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@master
-        with:
-          service_account_key: ${{ secrets.GCR_JSON_KEY }}
-          export_default_credentials: true
-      - uses: haya14busa/action-cond@v1
-        id: condval
-        with:
-          cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "minor"
-          if_false: "patch"
-      - name: Auto Increment Semver Action
-        uses: MCKanpolat/auto-semver-action@1.0.5
-        id: versioning
-        with:
-          releaseType: ${{ steps.condval.outputs.value }}
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Get base image name
-        shell: bash
-        run: |
-          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
-          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: version_parameters
-      - name: Build and Push CLI
-        run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
-      - name: publish
-        uses: ncipollo/release-action@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          artifacts: "cli/bin/*"
-          commit: ${{ steps.version_parameters.outputs.branch }}
-          tag: ${{ steps.versioning.outputs.version }}
-          prerelease: ${{ github.ref != 'refs/heads/main' }}
-          bodyFile: 'cli/bin/README.md'

.github/workflows/publish-docker.yml

@@ -1,39 +0,0 @@
-name: publish-docker
-on:
-  push:
-    branches:
-      - 'develop'
-      - 'main'
-jobs:
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get base image name
-        shell: bash
-        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
-        id: base_image_step
-      - name: Docker meta
-        id: meta
-        uses: crazy-max/ghaction-docker-meta@v2
-        with:
-          images: ${{ steps.base_image_step.outputs.image }}
-          tags: |
-            type=sha
-            type=raw,${{ github.sha }}
-            type=raw,latest
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          registry: gcr.io
-          username: _json_key
-          password: ${{ secrets.GCR_JSON_KEY }}
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-

.github/workflows/release.yml

@@ -0,0 +1,304 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - 'develop'
+      - 'main'
+
+concurrency:
+  group: mizu-publish-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  docker-registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+
+    strategy:
+      max-parallel: 2
+      fail-fast: false
+      matrix:
+        target:
+          - amd64
+          - arm64v8
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Determine versioning strategy
+        uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "stable"
+          if_false: "dev"
+
+      - name: Auto Increment Ver Action
+        uses: docker://igorgov/auto-inc-ver:v2.0.0
+        id: versioning
+        with:
+          mode: ${{ steps.condval.outputs.value }}
+          suffix: 'dev'
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            up9inc/mizu
+          tags: |
+            type=raw,${{ steps.versioning.outputs.version }}
+            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
+            type=raw,value=dev-latest,enable=${{ steps.condval.outputs.value == 'dev' }}
+          flavor: |
+            latest=auto
+            prefix=
+            suffix=-${{ matrix.target }},onlatest=true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            TARGETARCH=${{ matrix.target }}
+            VER=${{ steps.versioning.outputs.version }}
+            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
+            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
+            COMMIT_HASH=${{ github.sha }}
+
+  gcp-registry:
+    name: Push Docker image to GCR
+    runs-on: ubuntu-latest
+
+    strategy:
+      max-parallel: 2
+      fail-fast: false
+      matrix:
+        target:
+          - amd64
+          - arm64v8
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
+        with:
+          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v0'
+
+      - name: Determine versioning strategy
+        uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "stable"
+          if_false: "dev"
+
+      - name: Auto Increment Ver Action
+        uses: docker://igorgov/auto-inc-ver:v2.0.0
+        id: versioning
+        with:
+          mode: ${{ steps.condval.outputs.value }}
+          suffix: 'dev'
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+
+      - name: Get base image name
+        shell: bash
+        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
+        id: base_image_step
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ${{ steps.base_image_step.outputs.image }}
+          tags: |
+            type=raw,${{ steps.versioning.outputs.version }}
+            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
+            type=raw,value=dev-latest,enable=${{ steps.condval.outputs.value == 'dev' }}
+          flavor: |
+            latest=auto
+            prefix=
+            suffix=-${{ matrix.target }},onlatest=true
+
+      - name: Login to GCR
+        uses: docker/login-action@v1
+        with:
+          registry: gcr.io
+          username: _json_key
+          password: ${{ secrets.GCR_JSON_KEY }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            TARGETARCH=${{ matrix.target }}
+            VER=${{ steps.versioning.outputs.version }}
+            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
+            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
+            COMMIT_HASH=${{ github.sha }}
+
+  docker-manifest:
+    name: Create and Push a Docker Manifest
+    runs-on: ubuntu-latest
+    needs: [docker-registry]
+    steps:
+      - name: Determine versioning strategy
+        uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "stable"
+          if_false: "dev"
+
+      - name: Auto Increment Ver Action
+        uses: docker://igorgov/auto-inc-ver:v2.0.0
+        id: versioning
+        with:
+          mode: ${{ steps.condval.outputs.value }}
+          suffix: 'dev'
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            up9inc/mizu
+          tags: |
+            type=raw,${{ steps.versioning.outputs.version }}
+            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
+            type=raw,value=dev-latest,enable=${{ steps.condval.outputs.value == 'dev' }}
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+
+      - name: Create manifest
+        run: |
+          while IFS= read -r line; do
+            docker manifest create $line --amend $line-amd64 --amend $line-arm64v8
+          done <<< "${{ steps.meta.outputs.tags }}"
+
+      - name: Push manifest
+        run: |
+          while IFS= read -r line; do
+            docker manifest push $line
+          done <<< "${{ steps.meta.outputs.tags }}"
+
+  cli:
+    name: Build the CLI and publish
+    runs-on: ubuntu-latest
+    needs: [docker-manifest, gcp-registry]
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
+
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
+        with:
+          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v0'
+
+      - name: Determine versioning strategy
+        uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "stable"
+          if_false: "dev"
+
+      - name: Auto Increment Ver Action
+        uses: docker://igorgov/auto-inc-ver:v2.0.0
+        id: versioning
+        with:
+          mode: ${{ steps.condval.outputs.value }}
+          suffix: 'dev'
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+
+      - name: Build and Push CLI
+        run: make push-cli VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
+
+      - name: Log the version into a .txt file
+        shell: bash
+        run: |
+          echo '${{ steps.versioning.outputs.version }}' >> cli/bin/version.txt
+
+      - name: Release
+        uses: ncipollo/release-action@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          artifacts: "cli/bin/*"
+          commit: ${{ steps.version_parameters.outputs.branch }}
+          tag: ${{ steps.versioning.outputs.version }}
+          prerelease: ${{ github.ref != 'refs/heads/main' }}
+          bodyFile: 'cli/bin/README.md'
+      
+      - name: Slack notification on failure
+        uses: ravsamhq/notify-slack-action@v1
+        if: always()
+        with:
+          status: ${{ job.status }}
+          notification_title: 'Mizu enterprise {workflow} has {status_message}'
+          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit <{commit_url}|{commit_sha}> by ${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}> ```${{ github.event.head_commit.message }}```'
+          footer: 'Linked Repo <{repo_url}|{repo}>'
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/static_code_analysis.yml

@@ -0,0 +1,206 @@
+name: Static code analysis
+
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+
+permissions:
+  contents: read
+
+jobs:
+  go-lint:
+    name: Go lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - uses: actions/setup-go@v2
+        with:
+            go-version: '^1.17'
+
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y libpcap-dev
+          ./devops/install-capstone.sh
+
+      - name: Check Agent modified files
+        id: agent_modified_files
+        run: devops/check_modified_files.sh agent/
+
+      - name: Generate eBPF object files and go bindings
+        id: generate_ebpf
+        run: make bpf
+
+      - name: Go lint - agent
+        uses: golangci/golangci-lint-action@v2
+        if: steps.agent_modified_files.outputs.matched == 'true'
+        with:
+          version: latest
+          working-directory: agent
+          args: --timeout=10m
+
+      - name: Check shared modified files
+        id: shared_modified_files
+        run: devops/check_modified_files.sh shared/
+
+      - name: Go lint - shared
+        uses: golangci/golangci-lint-action@v2
+        if: steps.shared_modified_files.outputs.matched == 'true'
+        with:
+          version: latest
+          working-directory: shared
+          args: --timeout=10m
+
+      - name: Check tap modified files
+        id: tap_modified_files
+        run: devops/check_modified_files.sh tap/
+
+      - name: Go lint - tap
+        uses: golangci/golangci-lint-action@v2
+        if: steps.tap_modified_files.outputs.matched == 'true'
+        with:
+          version: latest
+          working-directory: tap
+          args: --timeout=10m
+
+      - name: Check cli modified files
+        id: cli_modified_files
+        run: devops/check_modified_files.sh cli/
+
+      - name: Go lint - CLI
+        uses: golangci/golangci-lint-action@v2
+        if: steps.cli_modified_files.outputs.matched == 'true'
+        with:
+          version: latest
+          working-directory: cli
+          args: --timeout=10m
+
+      - name: Check acceptanceTests modified files
+        id: acceptanceTests_modified_files
+        run: devops/check_modified_files.sh acceptanceTests/
+
+      - name: Go lint - acceptanceTests
+        uses: golangci/golangci-lint-action@v2
+        if: steps.acceptanceTests_modified_files.outputs.matched == 'true'
+        with:
+          version: latest
+          working-directory: acceptanceTests
+          args: --timeout=10m
+
+      - name: Check tap/api modified files
+        id: tap_api_modified_files
+        run: devops/check_modified_files.sh tap/api/
+
+      - name: Go lint - tap/api
+        uses: golangci/golangci-lint-action@v2
+        if: steps.tap_api_modified_files.outputs.matched == 'true'
+        with:
+          version: latest
+          working-directory: tap/api
+
+      - name: Check tap/extensions/amqp modified files
+        id: tap_amqp_modified_files
+        run: devops/check_modified_files.sh tap/extensions/amqp/
+
+      - name: Go lint - tap/extensions/amqp
+        uses: golangci/golangci-lint-action@v2
+        if: steps.tap_amqp_modified_files.outputs.matched == 'true'
+        with:
+          version: latest
+          working-directory: tap/extensions/amqp
+      
+      - name: Check tap/extensions/http modified files
+        id: tap_http_modified_files
+        run: devops/check_modified_files.sh tap/extensions/http/
+
+      - name: Go lint - tap/extensions/http
+        uses: golangci/golangci-lint-action@v2
+        if: steps.tap_http_modified_files.outputs.matched == 'true'
+        with:
+          version: latest
+          working-directory: tap/extensions/http
+
+      - name: Check tap/extensions/kafka modified files
+        id: tap_kafka_modified_files
+        run: devops/check_modified_files.sh tap/extensions/kafka/
+
+      - name: Go lint - tap/extensions/kafka
+        uses: golangci/golangci-lint-action@v2
+        if: steps.tap_kafka_modified_files.outputs.matched == 'true'
+        with:
+          version: latest
+          working-directory: tap/extensions/kafka
+
+      - name: Check tap/extensions/redis modified files
+        id: tap_redis_modified_files
+        run: devops/check_modified_files.sh tap/extensions/redis/
+
+      - name: Go lint - tap/extensions/redis
+        uses: golangci/golangci-lint-action@v2
+        if: steps.tap_redis_modified_files.outputs.matched == 'true'
+        with:
+          version: latest
+          working-directory: tap/extensions/redis
+
+      - name: Check logger modified files
+        id: logger_modified_files
+        run: devops/check_modified_files.sh logger/
+
+      - name: Go lint - logger
+        uses: golangci/golangci-lint-action@v2
+        if: steps.logger_modified_files.outputs.matched == 'true'
+        with:
+          version: latest
+          working-directory: logger
+
+  es-lint:
+    name: ES lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16
+        
+      - name: Check modified UI files
+        id: ui_modified_files
+        run: devops/check_modified_files.sh ui/
+      
+      - name: ESLint prerequisites ui
+        if: steps.ui_modified_files.outputs.matched == 'true'
+        run: |
+          sudo npm install -g eslint
+          cd ui
+          npm run prestart
+          npm i
+
+      - name: ESLint ui
+        if: steps.ui_modified_files.outputs.matched == 'true'
+        run: |
+          cd ui
+          npm run eslint
+      
+      - name: Check modified ui-common files
+        id: ui_common_modified_files
+        run: devops/check_modified_files.sh ui-common/
+      
+      - name: ESLint prerequisites ui-common
+        if: steps.ui_common_modified_files.outputs.matched == 'true'
+        run: |
+          sudo npm install -g eslint
+          cd ui-common
+          npm i
+
+      - name: ESLint ui-common
+        if: steps.ui_common_modified_files.outputs.matched == 'true'
+        run: |
+          cd ui-common
+          npm run eslint

.github/workflows/test.yml

@@ -0,0 +1,75 @@
+name: Test
+
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+  push: # needed to upload test coverage report to codecov
+    branches:
+      - 'develop'
+      - 'main'
+
+concurrency:
+  group: mizu-tests-validation-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-unit-tests:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17'
+
+      - name: Install libpcap
+        shell: bash
+        run: |
+          sudo apt-get install libpcap-dev
+
+      - name: Install Capstone
+        shell: bash
+        run: |
+          ./devops/install-capstone.sh
+
+      - name: Generate eBPF object files and go bindings
+        id: generate_ebpf
+        run: make bpf
+
+      - name: Check CLI modified files
+        id: cli_modified_files
+        run: devops/check_modified_files.sh cli/
+
+      - name: CLI Test
+        if: github.event_name == 'push' || steps.cli_modified_files.outputs.matched == 'true'
+        run: make test-cli
+
+      - name: Check Agent modified files
+        id: agent_modified_files
+        run: devops/check_modified_files.sh agent/
+
+      - name: Agent Test
+        if: github.event_name == 'push' || steps.agent_modified_files.outputs.matched == 'true'
+        run: make test-agent
+
+      - name: Shared Test
+        run: make test-shared
+      
+      - name: Check extensions modified files
+        id: ext_modified_files
+        run: devops/check_modified_files.sh tap/extensions/ tap/api/
+
+      - name: Extensions Test
+        if: github.event_name == 'push' || steps.ext_modified_files.outputs.matched == 'true'
+        run: make test-extensions
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2

.gitignore

@@ -15,7 +15,47 @@
 # vendor/
 .idea/
 build
-*.db
 
 # Mac OS
 .DS_Store
+.vscode/
+
+# Ignore the scripts that are created for development
+*dev.*
+
+# Environment variables
+.env
+
+# pprof
+pprof/*
+
+# Database Files
+*.db
+*.gob
+
+# Nohup Files - https://man7.org/linux/man-pages/man1/nohup.1p.html
+nohup.*
+
+# Cypress tests
+cypress.env.json
+*/cypress/downloads
+*/cypress/fixtures
+*/cypress/plugins
+*/cypress/screenshots
+*/cypress/videos
+*/cypress/support
+
+# Ignore test data in extensions
+tap/extensions/*/bin
+tap/extensions/*/expect
+
+# UI folders to ignore
+**/node_modules/**
+**/dist/**
+*.editorconfig
+
+# Ignore *.log files
+*.log
+
+# Object files
+*.o

Dockerfile

@@ -1,43 +1,142 @@
-FROM node:14-slim AS site-build
+ARG BUILDARCH=amd64
+ARG TARGETARCH=amd64
+
+### Front-end common
+FROM node:16 AS front-end-common
+
+WORKDIR /app/ui-build
+COPY ui-common/package.json .
+COPY ui-common/package-lock.json .
+RUN npm i
+COPY ui-common .
+RUN npm pack
+
+### Front-end
+FROM node:16 AS front-end
 
 WORKDIR /app/ui-build
 
-COPY ui .
+COPY ui/package.json ui/package-lock.json ./
+COPY --from=front-end-common ["/app/ui-build/up9-mizu-common-0.0.0.tgz", "."]
 RUN npm i
+COPY ui .
 RUN npm run build
 
+### Base builder image for native builds architecture
+FROM golang:1.17-alpine AS builder-native-base
+ENV CGO_ENABLED=1 GOOS=linux
+RUN apk add --no-cache \
+    libpcap-dev \
+    g++ \
+    perl-utils \
+    curl \
+    build-base \
+    binutils-gold \
+    bash \
+    clang \
+    llvm \
+    libbpf-dev \
+    linux-headers
+COPY devops/install-capstone.sh .
+RUN ./install-capstone.sh
 
-FROM golang:1.16-alpine AS builder
-# Set necessary environment variables needed for our image.
-ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 
-RUN apk add libpcap-dev gcc g++ make
+### Intermediate builder image for x86-64 to x86-64 native builds
+FROM builder-native-base AS builder-from-amd64-to-amd64
+ENV GOARCH=amd64
+ENV BPF_TARGET=amd64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86"
 
-# Move to api working directory (/api-build).
-WORKDIR /app/api-build
 
-COPY api/go.mod api/go.sum ./
+### Intermediate builder image for AArch64 to AArch64 native builds
+FROM builder-native-base AS builder-from-arm64v8-to-arm64v8
+ENV GOARCH=arm64
+ENV BPF_TARGET=arm64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64"
+
+
+### Builder image for x86-64 to AArch64 cross-compilation
+FROM up9inc/linux-arm64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 AS builder-from-amd64-to-arm64v8
+ENV CGO_ENABLED=1 GOOS=linux
+ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap -I/work/capstone/include"
+ENV BPF_TARGET=arm64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64 -I/usr/xcc/aarch64-linux-musl-cross/aarch64-linux-musl/include/"
+
+
+### Builder image for AArch64 to x86-64 cross-compilation
+FROM up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 AS builder-from-arm64v8-to-amd64
+ENV CGO_ENABLED=1 GOOS=linux
+ENV GOARCH=amd64 CGO_CFLAGS="-I/libpcap -I/capstone/include"
+ENV BPF_TARGET=amd64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86  -I/usr/local/musl/x86_64-unknown-linux-musl/include/"
+
+
+### Final builder image where the build happens
+# Possible build strategies:
+# BUILDARCH=amd64 TARGETARCH=amd64
+# BUILDARCH=arm64v8 TARGETARCH=arm64v8
+# BUILDARCH=amd64 TARGETARCH=arm64v8
+# BUILDARCH=arm64v8 TARGETARCH=amd64
+ARG BUILDARCH=amd64
+ARG TARGETARCH=amd64
+FROM builder-from-${BUILDARCH}-to-${TARGETARCH} AS builder
+
+# Move to agent working directory (/agent-build)
+WORKDIR /app/agent-build
+
+COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
+COPY logger/go.mod logger/go.mod ../logger/
+COPY tap/go.mod tap/go.mod ../tap/
+COPY tap/api/go.mod ../tap/api/
+COPY tap/dbgctl/go.mod ../tap/dbgctl/
+COPY tap/extensions/amqp/go.mod ../tap/extensions/amqp/
+COPY tap/extensions/http/go.mod ../tap/extensions/http/
+COPY tap/extensions/kafka/go.mod ../tap/extensions/kafka/
+COPY tap/extensions/redis/go.mod ../tap/extensions/redis/
 RUN go mod download
-# cheap trick to make the build faster (As long as go.mod wasn't changes)
-RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
 
-# Copy and build api code
+# Copy and build agent code
 COPY shared ../shared
-COPY api .
-RUN go build -ldflags="-s -w" -o mizuagent .
+COPY logger ../logger
+COPY tap ../tap
+COPY agent .
 
+ARG COMMIT_HASH
+ARG GIT_BRANCH
+ARG BUILD_TIMESTAMP
+ARG VER=0.0
 
-FROM alpine:3.13.5
+WORKDIR /app/tap/tlstapper
 
-RUN apk add bash libpcap-dev tcpdump
+RUN rm tlstapper_bpf*
+RUN GOARCH=${BUILDARCH} go generate tls_tapper.go
+
+WORKDIR /app/agent-build
+
+RUN go build -ldflags="-extldflags=-static -s -w \
+    -X 'github.com/up9inc/mizu/agent/pkg/version.GitCommitHash=${COMMIT_HASH}' \
+    -X 'github.com/up9inc/mizu/agent/pkg/version.Branch=${GIT_BRANCH}' \
+    -X 'github.com/up9inc/mizu/agent/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
+    -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
+
+# Download Basenine executable, verify the sha1sum
+ADD https://github.com/up9inc/basenine/releases/download/v0.8.3/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
+ADD https://github.com/up9inc/basenine/releases/download/v0.8.3/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
+
+RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
+    chmod +x ./basenine_linux_"${GOARCH}" && \
+    mv ./basenine_linux_"${GOARCH}" ./basenine
+
+### The shipped image
+ARG TARGETARCH=amd64
+FROM ${TARGETARCH}/busybox:latest
+# gin-gonic runs in debug mode without this
+ENV GIN_MODE=release
+
+WORKDIR /app/data/
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
-COPY --from=builder ["/app/api-build/mizuagent", "."]
-COPY --from=site-build ["/app/ui-build/build", "site"]
-
-COPY api/start.sh .
+COPY --from=builder ["/app/agent-build/mizuagent", "."]
+COPY --from=builder ["/app/agent-build/basenine", "/usr/local/bin/basenine"]
+COPY --from=front-end ["/app/ui-build/build", "site"]
 
 # this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
-ENTRYPOINT "/app/mizuagent"
+ENTRYPOINT ["/app/mizuagent"]

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui api cli tap docker
+.PHONY: help ui agent agent-debug cli tap docker bpf clean-bpf
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -19,34 +19,55 @@ help: ## This help.
 TS_SUFFIX="$(shell date '+%s')"
 GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
 BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
+export VER?=0.0
+ARCH=$(shell uname -m)
+ifeq ($(ARCH),$(filter $(ARCH),aarch64 arm64))
+	BPF_O_ARCH_LABEL=arm64
+else
+	BPF_O_ARCH_LABEL=x86
+endif
+BPF_O_FILES = tap/tlstapper/tlstapper46_bpfel_$(BPF_O_ARCH_LABEL).o tap/tlstapper/tlstapper_bpfel_$(BPF_O_ARCH_LABEL).o
 
-ui: ## build UI
+ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
-	@ls -l ui/build  
+	@ls -l ui/build
 
-cli: # build CLI
+cli: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build
 
-api: ## build API server
-	@(echo "building API server .." )
-	@(cd api; go build -o build/apiserver main.go)
-	@ls -l api/build
+cli-debug: ## Build CLI.
+	@echo "building cli"; cd cli && $(MAKE) build-debug
 
-#tap: ## build tap binary
-#	@(cd tap; go build -o build/tap ./src)
-#	@ls -l tap/build
+agent: bpf ## Build agent.
+	@(echo "building mizu agent .." )
+	@(cd agent; go build -o build/mizuagent main.go)
+	@ls -l agent/build
 
-docker: ## build Docker image 
-	@(echo "building docker image" )
-	./build-push-featurebranch.sh
+bpf: $(BPF_O_FILES)
 
-push: push-docker push-cli ## build and publish Mizu docker image & CLI
+$(BPF_O_FILES): $(wildcard tap/tlstapper/bpf/**/*.[ch])
+	@(echo "building tlstapper bpf")
+	@(./tap/tlstapper/bpf-builder/build.sh)
 
-push-docker: 
+agent-debug: ## Build agent for debug.
+	@(echo "building mizu agent for debug.." )
+	@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
+	@ls -l agent/build
+
+docker: ## Build and publish agent docker image.
+	$(MAKE) push-docker
+
+agent-docker: ## Build agent docker image.
+	@echo "Building agent docker image"
+	@docker build -t up9inc/mizu:devlatest .
+
+push: push-docker push-cli ## Build and publish agent docker image & CLI.
+
+push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
-	./build-push-featurebranch.sh
+	devops/build-push-featurebranch.sh
 
-push-cli:
+push-cli: ## Build and publish CLI.
 	@echo "publishing CLI .. "
 	@cd cli; $(MAKE) build-all
 	@echo "publishing file ${OUTPUT_FILE} .."
@@ -54,18 +75,47 @@ push-cli:
 	gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/
 	gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
 
+clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
 
-clean: clean-ui clean-api clean-cli clean-docker ## Clean all build artifacts
-
-clean-ui: 
+clean-ui: ## Clean UI.
 	@(rm -rf ui/build ; echo "UI cleanup done" )
 
-clean-api: 
-	@(rm -rf api/build ; echo "api cleanup done" )
+clean-agent: ## Clean agent.
+	@(rm -rf agent/build ; echo "agent cleanup done" )
 
-clean-cli: 
+clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
-clean-docker: 
+clean-docker:  ## Run clean docker
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
+clean-bpf:
+	@(rm $(BPF_O_FILES) ; echo "bpf cleanup done" )
+
+test-lint:  ## Run lint on all modules
+	cd agent && golangci-lint run
+	cd shared && golangci-lint run
+	cd tap && golangci-lint run
+	cd cli && golangci-lint run
+	cd acceptanceTests && golangci-lint run
+	cd tap/api && golangci-lint run
+	cd tap/dbgctl && golangci-lint run
+	cd tap/extensions/ && for D in */; do cd $$D && golangci-lint run && cd ..; done
+
+test-cli:  ## Run cli tests
+	@echo "running cli tests"; cd cli && $(MAKE) test
+
+test-agent:  ## Run agent tests
+	@echo "running agent tests"; cd agent && $(MAKE) test
+
+test-shared:  ## Run shared tests
+	@echo "running shared tests"; cd shared && $(MAKE) test
+
+test-extensions:  ## Run extensions tests
+	@echo "running http tests"; cd tap/extensions/http && $(MAKE) test
+	@echo "running redis tests"; cd tap/extensions/redis && $(MAKE) test
+	@echo "running kafka tests"; cd tap/extensions/kafka && $(MAKE) test
+	@echo "running amqp tests"; cd tap/extensions/amqp && $(MAKE) test
+
+acceptance-test:  ## Run acceptance tests
+	@echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test

README.md

@@ -1,24 +1,42 @@
-# 水 mizu
-standalone web app traffic viewer for Kubernetes
+![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
 
-## Download
+<p align="center">
+    <a href="https://github.com/up9inc/mizu/blob/main/LICENSE">
+        <img alt="GitHub License" src="https://img.shields.io/github/license/up9inc/mizu?logo=GitHub&style=flat-square">
+    </a>
+    <a href="https://github.com/up9inc/mizu/releases/latest">
+        <img alt="GitHub Latest Release" src="https://img.shields.io/github/v/release/up9inc/mizu?logo=GitHub&style=flat-square">
+    </a>
+    <a href="https://hub.docker.com/r/up9inc/mizu">
+      <img alt="Docker pulls" src="https://img.shields.io/docker/pulls/up9inc/mizu?color=%23099cec&logo=Docker&style=flat-square">
+    </a>
+    <a href="https://hub.docker.com/r/up9inc/mizu">
+      <img alt="Image size" src="https://img.shields.io/docker/image-size/up9inc/mizu/latest?logo=Docker&style=flat-square">
+    </a>
+    <a href="https://join.slack.com/t/up9/shared_invite/zt-tfjnduli-QzlR8VV4Z1w3YnPIAJfhlQ">
+      <img alt="Slack" src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social">
+    </a>
+</p>
 
-Download `mizu` for your platform and operating system
+# The API Traffic Viewer for Kubernetes
 
-### Latest stable release
+A simple-yet-powerful API traffic viewer for Kubernetes enabling you to view all API communication between microservices to help your debug and troubleshoot regressions.
 
-* for MacOS - `curl -o mizu https://github.com/up9inc/mizu/releases/download/latest/mizu_darwin_amd64 && chmod 755 mizu`
-* for Linux - `curl -o mizu https://github.com/up9inc/mizu/releases/download/latest/mizu_linux_amd64 && chmod 755 mizu`
+Think TCPDump and Wireshark re-invented for Kubernetes.
 
-### Development (unstable) build
-Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page.
+![Simple UI](assets/mizu-ui.png)
 
-## How to run
+## Quickstart and documentation
 
-1. Find pod you'd like to tap to in your Kubernetes cluster
-2. Run `mizu PODNAME` or `mizu REGEX` 
-3. Open browser on `http://localhost:8899` as instructed .. 
-4. Watch the WebAPI traffic flowing ..
+You can run Mizu on any Kubernetes cluster (version of 1.16.0 or higher) in a matter of seconds. See the [Mizu Getting Started Guide](https://getmizu.io/docs/) for how.
 
-## Examples
-TBD
+For more comprehensive documentation, start with the [docs](https://getmizu.io/docs/mizu/mizu-cli).
+
+## Working in this repo
+
+We ❤️ pull requests! See [CONTRIBUTING.md](docs/CONTRIBUTING.md) for info on contributing changes. <br />
+In the wiki you can find an intorduction to [mizu components](https://github.com/up9inc/mizu/wiki/Introduction-to-Mizu), and [development workflows](https://github.com/up9inc/mizu/wiki/Development-Workflows).
+
+## Code of Conduct
+
+This project is for everyone. We ask that our users and contributors take a few minutes to review our [Code of Conduct](docs/CODE_OF_CONDUCT.md).

acceptanceTests/.snyk

@@ -0,0 +1,6 @@
+# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
+version: v1.14.0
+ignore:
+  SNYK-GOLANG-GITHUBCOMPKGSFTP-569475:
+    - '*':
+      reason: None Given

acceptanceTests/Makefile

@@ -0,0 +1,3 @@
+test: ## Run acceptance tests.
+	@npm install cypress@10.0.1 -y
+	@go test ./... -timeout 1h -v

acceptanceTests/config_test.go

@@ -0,0 +1,284 @@
+package acceptanceTests
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+type tapConfig struct {
+	GuiPort uint16 `yaml:"gui-port"`
+}
+
+type configStruct struct {
+	Tap tapConfig `yaml:"tap"`
+}
+
+func TestConfigRegenerate(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	configPath, configPathErr := GetConfigPath()
+	if configPathErr != nil {
+		t.Errorf("failed to get config path, err: %v", cliPathErr)
+		return
+	}
+
+	configCmdArgs := GetDefaultConfigCommandArgs()
+
+	configCmdArgs = append(configCmdArgs, "-r")
+
+	configCmd := exec.Command(cliPath, configCmdArgs...)
+	t.Logf("running command: %v", configCmd.String())
+
+	t.Cleanup(func() {
+		if err := os.Remove(configPath); err != nil {
+			t.Logf("failed to delete config file, err: %v", err)
+		}
+	})
+
+	if err := configCmd.Start(); err != nil {
+		t.Errorf("failed to start config command, err: %v", err)
+		return
+	}
+
+	if err := configCmd.Wait(); err != nil {
+		t.Errorf("failed to wait config command, err: %v", err)
+		return
+	}
+
+	_, readFileErr := ioutil.ReadFile(configPath)
+	if readFileErr != nil {
+		t.Errorf("failed to read config file, err: %v", readFileErr)
+		return
+	}
+}
+
+func TestConfigGuiPort(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []uint16{8898}
+
+	for _, guiPort := range tests {
+		t.Run(fmt.Sprintf("%d", guiPort), func(t *testing.T) {
+			cliPath, cliPathErr := GetCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			configPath, configPathErr := GetConfigPath()
+			if configPathErr != nil {
+				t.Errorf("failed to get config path, err: %v", cliPathErr)
+				return
+			}
+
+			config := configStruct{}
+			config.Tap.GuiPort = guiPort
+
+			configBytes, marshalErr := yaml.Marshal(config)
+			if marshalErr != nil {
+				t.Errorf("failed to marshal config, err: %v", marshalErr)
+				return
+			}
+
+			if writeErr := ioutil.WriteFile(configPath, configBytes, 0644); writeErr != nil {
+				t.Errorf("failed to write config to file, err: %v", writeErr)
+				return
+			}
+
+			tapCmdArgs := GetDefaultTapCommandArgs()
+
+			tapNamespace := GetDefaultTapNamespace()
+			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+			tapCmd := exec.Command(cliPath, tapCmdArgs...)
+			t.Logf("running command: %v", tapCmd.String())
+
+			t.Cleanup(func() {
+				if err := CleanupCommand(tapCmd); err != nil {
+					t.Logf("failed to cleanup tap command, err: %v", err)
+				}
+
+				if err := os.Remove(configPath); err != nil {
+					t.Logf("failed to delete config file, err: %v", err)
+				}
+			})
+
+			if err := tapCmd.Start(); err != nil {
+				t.Errorf("failed to start tap command, err: %v", err)
+				return
+			}
+
+			apiServerUrl := GetApiServerUrl(guiPort)
+
+			if err := WaitTapPodsReady(apiServerUrl); err != nil {
+				t.Errorf("failed to start tap pods on time, err: %v", err)
+				return
+			}
+		})
+	}
+}
+
+func TestConfigSetGuiPort(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []struct {
+		ConfigFileGuiPort uint16
+		SetGuiPort        uint16
+	}{
+		{ConfigFileGuiPort: 8898, SetGuiPort: 8897},
+	}
+
+	for _, guiPortStruct := range tests {
+		t.Run(fmt.Sprintf("%d", guiPortStruct.SetGuiPort), func(t *testing.T) {
+			cliPath, cliPathErr := GetCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			configPath, configPathErr := GetConfigPath()
+			if configPathErr != nil {
+				t.Errorf("failed to get config path, err: %v", cliPathErr)
+				return
+			}
+
+			config := configStruct{}
+			config.Tap.GuiPort = guiPortStruct.ConfigFileGuiPort
+
+			configBytes, marshalErr := yaml.Marshal(config)
+			if marshalErr != nil {
+				t.Errorf("failed to marshal config, err: %v", marshalErr)
+				return
+			}
+
+			if writeErr := ioutil.WriteFile(configPath, configBytes, 0644); writeErr != nil {
+				t.Errorf("failed to write config to file, err: %v", writeErr)
+				return
+			}
+
+			tapCmdArgs := GetDefaultTapCommandArgs()
+
+			tapNamespace := GetDefaultTapNamespace()
+			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+			tapCmdArgs = append(tapCmdArgs, "--set", fmt.Sprintf("tap.gui-port=%v", guiPortStruct.SetGuiPort))
+
+			tapCmd := exec.Command(cliPath, tapCmdArgs...)
+			t.Logf("running command: %v", tapCmd.String())
+
+			t.Cleanup(func() {
+				if err := CleanupCommand(tapCmd); err != nil {
+					t.Logf("failed to cleanup tap command, err: %v", err)
+				}
+
+				if err := os.Remove(configPath); err != nil {
+					t.Logf("failed to delete config file, err: %v", err)
+				}
+			})
+
+			if err := tapCmd.Start(); err != nil {
+				t.Errorf("failed to start tap command, err: %v", err)
+				return
+			}
+
+			apiServerUrl := GetApiServerUrl(guiPortStruct.SetGuiPort)
+
+			if err := WaitTapPodsReady(apiServerUrl); err != nil {
+				t.Errorf("failed to start tap pods on time, err: %v", err)
+				return
+			}
+		})
+	}
+}
+
+func TestConfigFlagGuiPort(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []struct {
+		ConfigFileGuiPort uint16
+		FlagGuiPort       uint16
+	}{
+		{ConfigFileGuiPort: 8898, FlagGuiPort: 8896},
+	}
+
+	for _, guiPortStruct := range tests {
+		t.Run(fmt.Sprintf("%d", guiPortStruct.FlagGuiPort), func(t *testing.T) {
+			cliPath, cliPathErr := GetCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			configPath, configPathErr := GetConfigPath()
+			if configPathErr != nil {
+				t.Errorf("failed to get config path, err: %v", cliPathErr)
+				return
+			}
+
+			config := configStruct{}
+			config.Tap.GuiPort = guiPortStruct.ConfigFileGuiPort
+
+			configBytes, marshalErr := yaml.Marshal(config)
+			if marshalErr != nil {
+				t.Errorf("failed to marshal config, err: %v", marshalErr)
+				return
+			}
+
+			if writeErr := ioutil.WriteFile(configPath, configBytes, 0644); writeErr != nil {
+				t.Errorf("failed to write config to file, err: %v", writeErr)
+				return
+			}
+
+			tapCmdArgs := GetDefaultTapCommandArgs()
+
+			tapNamespace := GetDefaultTapNamespace()
+			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+			tapCmdArgs = append(tapCmdArgs, "-p", fmt.Sprintf("%v", guiPortStruct.FlagGuiPort))
+
+			tapCmd := exec.Command(cliPath, tapCmdArgs...)
+			t.Logf("running command: %v", tapCmd.String())
+
+			t.Cleanup(func() {
+				if err := CleanupCommand(tapCmd); err != nil {
+					t.Logf("failed to cleanup tap command, err: %v", err)
+				}
+
+				if err := os.Remove(configPath); err != nil {
+					t.Logf("failed to delete config file, err: %v", err)
+				}
+			})
+
+			if err := tapCmd.Start(); err != nil {
+				t.Errorf("failed to start tap command, err: %v", err)
+				return
+			}
+
+			apiServerUrl := GetApiServerUrl(guiPortStruct.FlagGuiPort)
+
+			if err := WaitTapPodsReady(apiServerUrl); err != nil {
+				t.Errorf("failed to start tap pods on time, err: %v", err)
+				return
+			}
+		})
+	}
+}

acceptanceTests/create_user.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Create a user in Minikube cluster "minikube"
+# Create context for user
+# Usage:
+#  ./create_user.sh <username>
+
+set -e
+
+NEW_USERNAME=$1
+CERT_DIR="${HOME}/certs"
+KEY_FILE="${CERT_DIR}/${NEW_USERNAME}.key"
+CRT_FILE="${CERT_DIR}/${NEW_USERNAME}.crt"
+MINIKUBE_KEY_FILE="${HOME}/.minikube/ca.key"
+MINIKUBE_CRT_FILE="${HOME}/.minikube/ca.crt"
+DAYS=1
+
+echo "Creating user and context for username \"${NEW_USERNAME}\" in Minikube cluster"
+
+if ! command -v openssl &> /dev/null
+then
+	 echo "Installing openssl"
+	 sudo apt-get update
+	 sudo apt-get install openssl
+fi
+
+echo "Creating certificate for user \"${NEW_USERNAME}\""
+mkdir -p ${CERT_DIR}
+echo "Generating key \"${KEY_FILE}\""
+openssl genrsa -out "${KEY_FILE}" 2048
+echo "Generating crt \"${CRT_FILE}\""
+openssl req -new -key "${KEY_FILE}" -out "${CRT_FILE}" -subj "/CN=${NEW_USERNAME}/O=group1"
+openssl x509 -req -in "${CRT_FILE}" -CA "${MINIKUBE_CRT_FILE}" -CAkey "${MINIKUBE_KEY_FILE}" -CAcreateserial -out "${CRT_FILE}" -days $DAYS
+
+echo "Creating context for user \"${NEW_USERNAME}\""
+kubectl config set-credentials "${NEW_USERNAME}" --client-certificate="${CRT_FILE}" --client-key="${KEY_FILE}"
+kubectl config set-context "${NEW_USERNAME}" --cluster=minikube --user="${NEW_USERNAME}"

acceptanceTests/cypress.config.js

@@ -0,0 +1,31 @@
+const { defineConfig } = require('cypress')
+
+module.exports = defineConfig({
+  watchForFileChanges: false,
+  viewportWidth: 1920,
+  viewportHeight: 1080,
+  video: false,
+  screenshotOnRunFailure: false,
+  defaultCommandTimeout: 6000,
+  env: {
+    testUrl: 'http://localhost:8899/',
+    redactHeaderContent: 'User-Header[REDACTED]',
+    redactBodyContent: '{ "User": "[REDACTED]" }',
+    regexMaskingBodyContent: '[REDACTED]',
+    greenFilterColor: 'rgb(210, 250, 210)',
+    redFilterColor: 'rgb(250, 214, 220)',
+    bodyJsonClass: '.hljs',
+    mizuWidth: 1920,
+    normalMizuHeight: 1080,
+    hugeMizuHeight: 3500,
+  },
+  e2e: {
+    // We've imported your old cypress plugins here.
+    // You may want to clean this up later by importing these.
+    // setupNodeEvents(on, config) {
+    //   return require('./cypress/plugins/index.js')(on, config)
+    // },
+    specPattern: 'cypress/e2e/tests/*.js',
+    supportFile: false
+  },
+})

acceptanceTests/cypress/e2e/testHelpers/StatusBarHelper.js

@@ -0,0 +1,45 @@
+const columns = {podName : 1, namespace : 2, tapping : 3};
+
+function getDomPathInStatusBar(line, column) {
+    return `[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) > :nth-child(${line}) > :nth-child(${column})`;
+}
+
+export function checkLine(line, expectedValues) {
+    cy.get(getDomPathInStatusBar(line, columns.podName)).invoke('text').then(podValue => {
+        const podName = getOnlyPodName(podValue);
+        expect(podName).to.equal(expectedValues.podName);
+
+        cy.get(getDomPathInStatusBar(line, columns.namespace)).invoke('text').then(namespaceValue => {
+            expect(namespaceValue).to.equal(expectedValues.namespace);
+            cy.get(getDomPathInStatusBar(line, columns.tapping)).children().should('have.attr', 'src').and("match", /success.*\.svg/);
+        });
+    });
+}
+
+export function findLineAndCheck(expectedValues) {
+    cy.get('[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) > > :nth-child(1)').then(pods => {
+        cy.get('[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) > > :nth-child(2)').then(namespaces => {
+            // organizing namespaces array
+            const podObjectsArray = Object.values(pods ?? {});
+            const namespacesObjectsArray = Object.values(namespaces ?? {});
+            let lineNumber = -1;
+            namespacesObjectsArray.forEach((namespaceObj, index) => {
+                const currentLine = index + 1;
+                lineNumber = (namespaceObj.getAttribute && namespaceObj.innerHTML === expectedValues.namespace && (getOnlyPodName(podObjectsArray[index].innerHTML)) === expectedValues.podName) ? currentLine : lineNumber;
+            });
+            lineNumber === -1 ? throwError(expectedValues) : checkLine(lineNumber, expectedValues);
+        });
+    });
+}
+
+function throwError(expectedValues) {
+    throw new Error(`The pod named ${expectedValues.podName} doesn't match any namespace named ${expectedValues.namespace}`);
+}
+
+export function getExpectedDetailsDict(podName, namespace) {
+    return {podName : podName, namespace : namespace};
+}
+
+function getOnlyPodName(podElementFullStr) {
+    return podElementFullStr.substring(0, podElementFullStr.indexOf('-'));
+}

acceptanceTests/cypress/e2e/testHelpers/TrafficHelper.js

@@ -0,0 +1,181 @@
+export const valueTabs = {
+    response: 'RESPONSE',
+    request: 'REQUEST',
+    none: null
+}
+
+export function isValueExistsInElement(shouldInclude, content, domPathToContainer){
+    it(`should ${shouldInclude ? '' : 'not'} include '${content}'`, function () {
+        cy.get(domPathToContainer).then(htmlText => {
+            const allTextString = htmlText.text();
+            if (allTextString.includes(content) !== shouldInclude)
+                throw new Error(`One of the containers part contains ${content}`)
+        });
+    });
+}
+
+export function resizeToHugeMizu() {
+    cy.viewport(Cypress.env('mizuWidth'), Cypress.env('hugeMizuHeight'));
+}
+
+export function resizeToNormalMizu() {
+    cy.viewport(Cypress.env('mizuWidth'), Cypress.env('normalMizuHeight'));
+}
+
+export function verifyMinimumEntries() {
+    const entriesSent = Cypress.env('entriesCount');
+    const minimumEntries = Math.round((0.75 * entriesSent));
+
+    it(`Making sure that mizu shows at least ${minimumEntries} entries`, function () {
+        cy.get('#total-entries').then(number => {
+            const getNum = () => {
+                return parseInt(number.text());
+            };
+
+            cy.wrap({num: getNum}).invoke('num').should('be.gt', minimumEntries);
+        });
+    });
+}
+
+export function leftTextCheck(entryId, path, expectedText) {
+    cy.get(`#list #entry-${entryId} ${path}`).invoke('text').should('eq', expectedText);
+}
+
+export function leftOnHoverCheck(entryId, path, filterName) {
+    cy.get(`#list #entry-${entryId} ${path}`).trigger('mouseover');
+    cy.get(`#list #entry-${entryId} [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(filterName));
+}
+
+export function rightTextCheck(path, expectedText) {
+    cy.get(`#rightSideContainer ${path}`).should('have.text', expectedText);
+}
+
+export function rightOnHoverCheck(path, expectedText) {
+    cy.get(`#rightSideContainer ${path}`).trigger('mouseover');
+    cy.get(`#rightSideContainer [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(expectedText));
+}
+
+export function checkFilterByMethod(funcDict) {
+    const {protocol, method, methodQuery, summary, summaryQuery, numberOfRecords} = funcDict;
+    const summaryDict = getSummaryDict(summary, summaryQuery);
+    const methodDict = getMethodDict(method, methodQuery);
+    const protocolDict = getProtocolDict(protocol.name, protocol.text);
+
+    it(`Testing the method: ${method}`, function () {
+        // applying filter
+        cy.get('.w-tc-editor-text').clear().type(methodQuery);
+        cy.get('[type="submit"]').click();
+        cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
+
+        waitForFetch(numberOfRecords);
+        pauseStream();
+
+        cy.get(`#list [id^=entry]`).then(elements => {
+            const listElmWithIdAttr = Object.values(elements);
+            let doneCheckOnFirst = false;
+
+            cy.get('#entries-length').invoke('text').then(len => {
+                listElmWithIdAttr.forEach(entry => {
+                    if (entry?.id && entry.id.match(RegExp(/entry-(\d{24})$/gm))) {
+                        const entryId = getEntryId(entry.id);
+
+                        leftTextCheck(entryId, methodDict.pathLeft, methodDict.expectedText);
+                        leftTextCheck(entryId, protocolDict.pathLeft, protocolDict.expectedTextLeft);
+                        if (summaryDict)
+                            leftTextCheck(entryId, summaryDict.pathLeft, summaryDict.expectedText);
+
+                        if (!doneCheckOnFirst) {
+                            deepCheck(funcDict, protocolDict, methodDict, entry);
+                            doneCheckOnFirst = true;
+                        }
+                    }
+                });
+            });
+        });
+    });
+}
+
+export const refreshWaitTimeout = 10000;
+
+export function waitForFetch(gt) {
+    cy.get('#entries-length', {timeout: refreshWaitTimeout}).should((el) => {
+        expect(parseInt(el.text().trim(), 10)).to.be.greaterThan(gt);
+    });
+}
+
+export function pauseStream() {
+    cy.get('#pause-icon').click();
+    cy.get('#pause-icon').should('not.be.visible');
+}
+
+
+export function getEntryId(id) {
+    // take the second part from the string (entry-<ID>)
+    return id.split('-')[1];
+}
+
+function deepCheck(generalDict, protocolDict, methodDict, entry) {
+    const entryId = getEntryId(entry.id);
+    const {summary, value} = generalDict;
+    const summaryDict = getSummaryDict(summary);
+
+    leftOnHoverCheck(entryId, methodDict.pathLeft, methodDict.expectedOnHover);
+    leftOnHoverCheck(entryId, protocolDict.pathLeft, protocolDict.expectedOnHover);
+    if (summaryDict)
+        leftOnHoverCheck(entryId, summaryDict.pathLeft, summaryDict.expectedOnHover);
+
+    cy.get(`#${entry.id}`).click();
+
+    rightTextCheck(methodDict.pathRight, methodDict.expectedText);
+    rightTextCheck(protocolDict.pathRight, protocolDict.expectedTextRight);
+    if (summaryDict)
+        rightTextCheck(summaryDict.pathRight, summaryDict.expectedText);
+
+    rightOnHoverCheck(methodDict.pathRight, methodDict.expectedOnHover);
+    rightOnHoverCheck(protocolDict.pathRight, protocolDict.expectedOnHover);
+    if (summaryDict)
+        rightOnHoverCheck(summaryDict.pathRight, summaryDict.expectedOnHover);
+
+    if (value) {
+        if (value.tab === valueTabs.response)
+            // temporary fix, change to some "data-cy" attribute,
+            // this will fix the issue that happen because we have "response:" in the header of the right side
+            cy.get('#rightSideContainer > :nth-child(3)').contains('Response').click();
+        cy.get(Cypress.env('bodyJsonClass')).then(text => {
+            expect(text.text()).to.match(value.regex)
+        });
+    }
+}
+
+function getSummaryDict(value, query) {
+    if (value) {
+        return {
+            pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+            pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+            expectedText: value,
+            expectedOnHover: query
+        };
+    }
+    else {
+        return null;
+    }
+}
+
+function getMethodDict(value, query) {
+    return {
+        pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+        pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+        expectedText: value,
+        expectedOnHover: query
+    };
+}
+
+function getProtocolDict(protocol, protocolText) {
+    return {
+        pathLeft: '> :nth-child(1) > :nth-child(1)',
+        pathRight: '> :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(1)',
+        expectedTextLeft: protocol.toUpperCase(),
+        expectedTextRight: protocolText,
+        expectedOnHover: protocol.toLowerCase()
+    };
+}

acceptanceTests/cypress/e2e/tests/GuiPort.js

@@ -0,0 +1,13 @@
+import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
+
+it('check', function () {
+    const podName = Cypress.env('name'), namespace = Cypress.env('namespace');
+    const port = Cypress.env('port');
+    cy.intercept('GET', `http://localhost:${port}/status/tap`).as('statusTap');
+
+    cy.visit(`http://localhost:${port}`);
+    cy.wait('@statusTap').its('response.statusCode').should('match', /^2\d{2}/);
+
+    cy.get(`[data-cy="expandedStatusBar"]`).trigger('mouseover',{force: true});
+    findLineAndCheck(getExpectedDetailsDict(podName, namespace));
+});

acceptanceTests/cypress/e2e/tests/IgnoredUserAgents.js

@@ -0,0 +1,25 @@
+import {
+    isValueExistsInElement,
+    resizeToHugeMizu,
+} from "../testHelpers/TrafficHelper";
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+checkEntries();
+
+function checkEntries() {
+    it('checking all entries', function () {
+        cy.get('#entries-length').should('not.have.text', '0').then(() => {
+            resizeToHugeMizu();
+
+            cy.get('#list [id^=entry]').each(entryElement => {
+                entryElement.click();
+                cy.get('#tbody-Headers').should('be.visible');
+                isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
+            });
+        });
+    });
+}
+

acceptanceTests/cypress/e2e/tests/MultipleNamespaces.js

@@ -0,0 +1,17 @@
+import {findLineAndCheck, getExpectedDetailsDict} from '../testHelpers/StatusBarHelper';
+
+it('opening', function () {
+    cy.visit(Cypress.env('testUrl'));
+    cy.get(`[data-cy="podsCountText"]`).trigger('mouseover');
+});
+
+[1, 2, 3].map(doItFunc);
+
+function doItFunc(number) {
+    const podName = Cypress.env(`name${number}`);
+    const namespace = Cypress.env(`namespace${number}`);
+
+    it(`verifying the pod (${podName}, ${namespace})`, function () {
+        findLineAndCheck(getExpectedDetailsDict(podName, namespace));
+    });
+}

acceptanceTests/cypress/e2e/tests/NoRedact.js

@@ -0,0 +1,8 @@
+import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+isValueExistsInElement(false, Cypress.env('redactHeaderContent'), '#tbody-Headers');
+isValueExistsInElement(false, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/e2e/tests/Rabbit.js

@@ -0,0 +1,68 @@
+import {checkFilterByMethod, valueTabs,} from "../testHelpers/TrafficHelper";
+
+it('opening mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+const rabbitProtocolDetails = {name: 'AMQP', text: 'Advanced Message Queuing Protocol 0-9-1'};
+const numberOfRecords = 5;
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method: 'exchange declare',
+    methodQuery: 'request.method == "exchange declare"',
+    summary: 'exchange',
+    summaryQuery: 'request.exchange == "exchange"',
+    numberOfRecords: numberOfRecords,
+    value: null
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method: 'queue declare',
+    methodQuery: 'request.method == "queue declare"',
+    summary: 'queue',
+    summaryQuery: 'request.queue == "queue"',
+    numberOfRecords: numberOfRecords,
+    value: null
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method:  'queue bind',
+    methodQuery: 'request.method == "queue bind"',
+    summary: 'queue',
+    summaryQuery: 'request.queue == "queue"',
+    numberOfRecords: numberOfRecords,
+    value: null
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method: 'basic publish',
+    methodQuery: 'request.method == "basic publish"',
+    summary: 'exchange',
+    summaryQuery: 'request.exchange == "exchange"',
+    numberOfRecords: numberOfRecords,
+    value: {tab: valueTabs.request, regex: /^message$/mg}
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method:  'basic consume',
+    methodQuery: 'request.method == "basic consume"',
+    summary: 'queue',
+    summaryQuery: 'request.queue == "queue"',
+    numberOfRecords: numberOfRecords,
+    value: null
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method:  'basic deliver',
+    methodQuery: 'request.method == "basic deliver"',
+    summary: 'exchange',
+    summaryQuery: 'request.queue == "exchange"',
+    numberOfRecords: numberOfRecords,
+    value: {tab: valueTabs.request, regex: /^message$/mg}
+});

acceptanceTests/cypress/e2e/tests/Redact.js

@@ -0,0 +1,8 @@
+import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+isValueExistsInElement(true, Cypress.env('redactHeaderContent'), '#tbody-Headers');
+isValueExistsInElement(true, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/e2e/tests/Redis.js

@@ -0,0 +1,58 @@
+import {checkFilterByMethod, valueTabs,} from "../testHelpers/TrafficHelper";
+
+it('opening mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+const redisProtocolDetails = {name: 'redis', text: 'Redis Serialization Protocol'};
+const numberOfRecords = 5;
+
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
+    method: 'PING',
+    methodQuery: 'request.command == "PING"',
+    summary: null,
+    summaryQuery: '',
+    numberOfRecords: numberOfRecords,
+    value: null
+})
+
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
+    method: 'SET',
+    methodQuery: 'request.command == "SET"',
+    summary: 'key',
+    summaryQuery: 'request.key == "key"',
+    numberOfRecords: numberOfRecords,
+    value: {tab: valueTabs.request, regex: /^\[value, keepttl]$/mg}
+})
+
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
+    method: 'EXISTS',
+    methodQuery: 'request.command == "EXISTS"',
+    summary: 'key',
+    summaryQuery: 'request.key == "key"',
+    numberOfRecords: numberOfRecords,
+    value: {tab: valueTabs.response, regex: /^1$/mg}
+})
+
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
+    method: 'GET',
+    methodQuery: 'request.command == "GET"',
+    summary: 'key',
+    summaryQuery: 'request.key == "key"',
+    numberOfRecords: numberOfRecords,
+    value: {tab: valueTabs.response, regex: /^value$/mg}
+})
+
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
+    method: 'DEL',
+    methodQuery: 'request.command == "DEL"',
+    summary: 'key',
+    summaryQuery: 'request.key == "key"',
+    numberOfRecords: numberOfRecords,
+    value: {tab: valueTabs.response, regex: /^1$|^0$/mg}
+})

acceptanceTests/cypress/e2e/tests/Regex.js

@@ -0,0 +1,11 @@
+import {getExpectedDetailsDict, checkLine} from '../testHelpers/StatusBarHelper';
+
+
+it('opening', function () {
+    cy.visit(Cypress.env('testUrl'));
+    cy.get(`[data-cy="podsCountText"]`).trigger('mouseover');
+
+    cy.get('[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) >').should('have.length', 1); // one line
+
+    checkLine(1, getExpectedDetailsDict(Cypress.env('name'), Cypress.env('namespace')));
+});

acceptanceTests/cypress/e2e/tests/RegexMasking.js

@@ -0,0 +1,7 @@
+import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/e2e/tests/UiTest.js

@@ -0,0 +1,351 @@
+import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
+import {
+    getEntryId,
+    leftOnHoverCheck,
+    leftTextCheck,
+    resizeToHugeMizu,
+    resizeToNormalMizu,
+    rightOnHoverCheck,
+    rightTextCheck,
+    verifyMinimumEntries,
+    refreshWaitTimeout,
+    waitForFetch,
+    pauseStream
+} from "../testHelpers/TrafficHelper";
+
+const fullParam = Cypress.env('arrayDict'); // "Name:fooNamespace:barName:foo1Namespace:bar1"
+const podsArray = fullParam.split('Name:').slice(1); // ["fooNamespace:bar", "foo1Namespace:bar1"]
+podsArray.forEach((podStr, index) => {
+    const podAndNamespaceArr = podStr.split('Namespace:'); // [foo, bar] / [foo1, bar1]
+    podsArray[index] = getExpectedDetailsDict(podAndNamespaceArr[0], podAndNamespaceArr[1]);
+});
+
+it('opening mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+verifyMinimumEntries();
+
+it('top bar check', function () {
+    cy.get(`[data-cy="podsCountText"]`).trigger('mouseover');
+    podsArray.map(findLineAndCheck);
+    cy.reload();
+});
+
+it('filtering guide check', function () {
+    cy.reload();
+    cy.get('[title="Open Filtering Guide (Cheatsheet)"]').click();
+    cy.get('#modal-modal-title').should('be.visible');
+    cy.get('[lang="en"]').click(0, 0);
+    cy.get('#modal-modal-title').should('not.exist');
+});
+
+it('right side sanity test', function () {
+    cy.get('#entryDetailedTitleElapsedTime').then(timeInMs => {
+        const time = timeInMs.text();
+        if (time < '0ms') {
+            throw new Error(`The time in the top line cannot be negative ${time}`);
+        }
+    });
+
+    // temporary fix, change to some "data-cy" attribute,
+    // this will fix the issue that happen because we have "response:" in the header of the right side
+    cy.get('#rightSideContainer > :nth-child(3)').contains('Response').click();
+
+    cy.get('#rightSideContainer [title="Status Code"]').then(status => {
+        const statusCode = status.text();
+        cy.contains('Status').parent().next().then(statusInDetails => {
+            const statusCodeInDetails = statusInDetails.text();
+
+            expect(statusCode).to.equal(statusCodeInDetails, 'The status code in the top line should match the status code in details');
+        });
+    });
+});
+
+checkIllegalFilter('invalid filter');
+
+checkFilter({
+    filter: 'http',
+    leftSidePath: '> :nth-child(1) > :nth-child(1)',
+    leftSideExpectedText: 'HTTP',
+    rightSidePath: '[title=HTTP]',
+    rightSideExpectedText: 'Hypertext Transfer Protocol -- HTTP/1.1',
+    applyByCtrlEnter: true,
+    numberOfRecords: 20,
+});
+
+checkFilter({
+    filter: 'response.status == 200',
+    leftSidePath: '[title="Status Code"]',
+    leftSideExpectedText: '200',
+    rightSidePath: '> :nth-child(2) [title="Status Code"]',
+    rightSideExpectedText: '200',
+    applyByCtrlEnter: false,
+    numberOfRecords: 20
+});
+
+if (Cypress.env('shouldCheckSrcAndDest')) {
+    serviceMapCheck();
+
+    checkFilter({
+        filter: 'src.name == ""',
+        leftSidePath: '[title="Source Name"]',
+        leftSideExpectedText: '[Unresolved]',
+        rightSidePath: '> :nth-child(2) [title="Source Name"]',
+        rightSideExpectedText: '[Unresolved]',
+        applyByCtrlEnter: false,
+        numberOfRecords: 20
+    });
+
+    checkFilter({
+        filter: `dst.name == "httpbin.mizu-tests"`,
+        leftSidePath: '> :nth-child(3) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
+        leftSideExpectedText: 'httpbin.mizu-tests',
+        rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
+        rightSideExpectedText: 'httpbin.mizu-tests',
+        applyByCtrlEnter: false,
+        numberOfRecords: 20
+    });
+}
+
+checkFilter({
+    filter: 'request.method == "GET"',
+    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+    leftSideExpectedText: 'GET',
+    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+    rightSideExpectedText: 'GET',
+    applyByCtrlEnter: true,
+    numberOfRecords: 20
+});
+
+checkFilter({
+    filter: 'request.path == "/get"',
+    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+    leftSideExpectedText: '/get',
+    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+    rightSideExpectedText: '/get',
+    applyByCtrlEnter: false,
+    numberOfRecords: 20
+});
+
+checkFilter({
+    filter: 'src.ip == "127.0.0.1"',
+    leftSidePath: '[title="Source IP"]',
+    leftSideExpectedText: '127.0.0.1',
+    rightSidePath: '> :nth-child(2) [title="Source IP"]',
+    rightSideExpectedText: '127.0.0.1',
+    applyByCtrlEnter: false,
+    numberOfRecords: 20
+});
+
+checkFilterNoResults('request.method == "POST"');
+
+function checkFilterNoResults(filterName) {
+    it(`checking the filter: ${filterName}. Expecting no results`, function () {
+        cy.get('#total-entries').then(number => {
+            const totalEntries = number.text();
+
+            // applying the filter
+            cy.get('.w-tc-editor-text').type(filterName);
+            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
+            cy.get('[type="submit"]').click();
+
+            // waiting for the entries number to load
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+
+            // the DOM should show 0 entries
+            cy.get('#entries-length').should('have.text', '0');
+
+            cy.get('[title="Fetch old records"]').click();
+            cy.get('#noMoreDataTop', {timeout: refreshWaitTimeout}).should('be.visible');
+            cy.get('#entries-length').should('have.text', '0'); // after loading all entries there should still be 0 entries
+
+            // reloading then waiting for the entries number to load
+            cy.reload();
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+        });
+    });
+}
+
+function checkIllegalFilter(illegalFilterName) {
+    it(`should show red search bar with the input: ${illegalFilterName}`, function () {
+        cy.reload();
+        cy.get('#total-entries').then(number => {
+            const totalEntries = number.text();
+
+            cy.get('.w-tc-editor-text').type(illegalFilterName);
+            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('redFilterColor'));
+            cy.get('[type="submit"]').click();
+
+            cy.get('[role="alert"]').should('be.visible');
+            cy.get('.w-tc-editor-text').clear();
+
+            // reloading then waiting for the entries number to load
+            cy.reload();
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+        });
+    });
+}
+
+function checkFilter(filterDetails) {
+    const {
+        filter,
+        leftSidePath,
+        rightSidePath,
+        rightSideExpectedText,
+        leftSideExpectedText,
+        applyByCtrlEnter,
+        numberOfRecords
+    } = filterDetails;
+
+    const entriesForDeeperCheck = 5;
+
+    it(`checking the filter: ${filter}`, function () {
+        cy.get('.w-tc-editor-text').clear();
+        // applying the filter with alt+enter or with the button
+        cy.get('.w-tc-editor-text').type(`${filter}${applyByCtrlEnter ? '{ctrl+enter}' : ''}`);
+        cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
+        if (!applyByCtrlEnter)
+            cy.get('[type="submit"]').click();
+
+        waitForFetch(numberOfRecords);
+        pauseStream();
+
+        cy.get(`#list [id^=entry]`).last().then(elem => {
+            const element = elem[0];
+            const entryId = getEntryId(element.id);
+
+            // only one entry in DOM after filtering, checking all checks on it
+            leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
+            leftOnHoverCheck(entryId, leftSidePath, filter);
+
+            rightTextCheck(rightSidePath, rightSideExpectedText);
+            rightOnHoverCheck(rightSidePath, filter);
+            checkRightSideResponseBody();
+        });
+
+        resizeToHugeMizu();
+
+        // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
+        cy.get(`#list [id^=entry]`).each(elem => {
+            const element = elem[0];
+            let entryId = getEntryId(element.id);
+            leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
+        });
+
+        // making the other 3 checks on the first X entries (longer time for each check)
+        deeperCheck(leftSidePath, rightSidePath, filter, rightSideExpectedText, entriesForDeeperCheck);
+
+        // reloading then waiting for the entries number to load
+        resizeToNormalMizu();
+        cy.reload();
+        waitForFetch(numberOfRecords);
+        pauseStream();
+    });
+}
+
+function deeperCheck(leftSidePath, rightSidePath, filterName, rightSideExpectedText, entriesNumToCheck) {
+    cy.get(`#list [id^=entry]`).each((element, index) => {
+        if (index < entriesNumToCheck) {
+            const entryId = getEntryId(element[0].id);
+            leftOnHoverCheck(entryId, leftSidePath, filterName);
+
+            cy.get(`#list #entry-${entryId}`).click();
+            rightTextCheck(rightSidePath, rightSideExpectedText);
+            rightOnHoverCheck(rightSidePath, filterName);
+        }
+    });
+}
+
+function checkRightSideResponseBody() {
+    // temporary fix, change to some "data-cy" attribute,
+    // this will fix the issue that happen because we have "response:" in the header of the right side
+    cy.get('#rightSideContainer > :nth-child(3)').contains('Response').click();
+    clickCheckbox('Decode Base64');
+
+    cy.get(`${Cypress.env('bodyJsonClass')}`).then(value => {
+        const encodedBody = value.text();
+        const decodedBody = atob(encodedBody);
+        const responseBody = JSON.parse(decodedBody);
+
+
+        const expectedJsonBody = {
+            args: RegExp({}),
+            url: RegExp('http://.*/get'),
+            headers: {
+                "User-Agent": RegExp('client'),
+                "Accept-Encoding": RegExp('gzip'),
+                "X-Forwarded-Uri": RegExp('/api/v1/namespaces/.*/services/.*/proxy/get')
+            }
+        };
+
+        const expectedStringInJsonBody = RegExp('/api/v1/namespaces/.*/services/.*/proxy/get');
+        
+
+        expect(responseBody.args).to.match(expectedJsonBody.args);
+        expect(responseBody.url).to.match(expectedJsonBody.url);
+        expect(responseBody.headers['User-Agent']).to.match(expectedJsonBody.headers['User-Agent']);
+        expect(responseBody.headers['Accept-Encoding']).to.match(expectedJsonBody.headers['Accept-Encoding']);
+        expect(responseBody.headers['X-Forwarded-Uri']).to.match(expectedJsonBody.headers['X-Forwarded-Uri']);
+
+        cy.get(`${Cypress.env('bodyJsonClass')}`).should('have.text', encodedBody);
+        cy.get(`[data-cy="lineNumbersCheckBoxInput"]`).should('be.disabled');
+
+        clickCheckbox('Decode Base64');
+        cy.get(`[data-cy="lineNumbersCheckBoxInput"]`).should('not.be.disabled');
+
+        cy.get(`${Cypress.env('bodyJsonClass')} > `).its('length').should('be.gt', 1).then(linesNum => {
+            cy.get(`${Cypress.env('bodyJsonClass')} > >`).its('length').should('be.gt', linesNum).then(jsonItemsNum => {
+                checkOnlyLineNumberes(jsonItemsNum, expectedStringInJsonBody);
+            });
+        });
+    });
+}
+
+function clickCheckbox(type) {
+    cy.contains(`${type}`).prev().children().click();
+}
+
+function checkOnlyLineNumberes(jsonItems, decodedText) {
+    cy.get(`${Cypress.env('bodyJsonClass')} > >`).should('have.length', jsonItems);
+    cy.get(`${Cypress.env('bodyJsonClass')} >`).contains(decodedText);
+}
+
+function serviceMapCheck() {
+    it('service map test', function () {
+        cy.intercept(`${Cypress.env('testUrl')}/servicemap/get`).as('serviceMapRequest');
+        cy.get('#total-entries').should('not.have.text', '0').then(() => {
+            cy.get('#total-entries').invoke('text').then(entriesNum => {
+                cy.get('[alt="service-map"]').click();
+                cy.wait('@serviceMapRequest').then(({response}) => {
+                    const body = response.body;
+                    const nodeParams = {
+                        destination: 'httpbin.mizu-tests',
+                        source: '127.0.0.1'
+                    };
+                    serviceMapAPICheck(body, parseInt(entriesNum), nodeParams);
+                    cy.reload();
+                });
+            });
+        });
+    });
+}
+
+function serviceMapAPICheck(body, entriesNum, nodeParams) {
+    const {nodes, edges} = body;
+
+    expect(nodes.length).to.equal(Object.keys(nodeParams).length, `Expected nodes count`);
+
+    expect(edges.some(edge => edge.source.name === nodeParams.source)).to.be.true;
+    expect(edges.some(edge => edge.destination.name === nodeParams.destination)).to.be.true;
+
+    let count = 0;
+    edges.forEach(edge => {
+        count += edge.count;
+        if (edge.destination.name === nodeParams.destination) {
+            expect(edge.source.name).to.equal(nodeParams.source);
+        }
+    });
+
+    expect(count).to.equal(entriesNum);
+}

acceptanceTests/extensions_test.go

@@ -0,0 +1,243 @@
+package acceptanceTests
+
+import (
+	"context"
+	"fmt"
+	"github.com/go-redis/redis/v8"
+	amqp "github.com/rabbitmq/amqp091-go"
+	"os/exec"
+	"testing"
+	"time"
+)
+
+func TestRedis(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	ctx := context.Background()
+
+	kubernetesProvider, err := NewKubernetesProvider()
+	if err != nil {
+		t.Errorf("failed to create k8s provider, err %v", err)
+		return
+	}
+
+	redisExternalIp, err := kubernetesProvider.GetServiceExternalIp(ctx, DefaultNamespaceName, "redis")
+	if err != nil {
+		t.Errorf("failed to get redis external ip, err: %v", err)
+		return
+	}
+
+	rdb := redis.NewClient(&redis.Options{
+		Addr: fmt.Sprintf("%v:6379", redisExternalIp),
+	})
+
+	for i := 0; i < DefaultEntriesCount/5; i++ {
+		requestErr := rdb.Ping(ctx).Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < DefaultEntriesCount/5; i++ {
+		requestErr := rdb.Set(ctx, "key", "value", -1).Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < DefaultEntriesCount/5; i++ {
+		requestErr := rdb.Exists(ctx, "key").Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < DefaultEntriesCount/5; i++ {
+		requestErr := rdb.Get(ctx, "key").Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < DefaultEntriesCount/5; i++ {
+		requestErr := rdb.Del(ctx, "key").Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/Redis.js\"")
+}
+
+func TestAmqp(t *testing.T) {
+
+	t.Skip("ignoredd for now because those tests are not stable")
+
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	ctx := context.Background()
+
+	kubernetesProvider, err := NewKubernetesProvider()
+	if err != nil {
+		t.Errorf("failed to create k8s provider, err %v", err)
+		return
+	}
+
+	rabbitmqExternalIp, err := kubernetesProvider.GetServiceExternalIp(ctx, DefaultNamespaceName, "rabbitmq")
+	if err != nil {
+		t.Errorf("failed to get RabbitMQ external ip, err: %v", err)
+		return
+	}
+
+	conn, err := amqp.Dial(fmt.Sprintf("amqp://guest:guest@%v:5672/", rabbitmqExternalIp))
+	if err != nil {
+		t.Errorf("failed to connect to RabbitMQ, err: %v", err)
+		return
+	}
+	defer conn.Close()
+
+	// Temporary fix for missing amqp entries
+	time.Sleep(10 * time.Second)
+
+	for i := 0; i < DefaultEntriesCount/5; i++ {
+		ch, err := conn.Channel()
+		if err != nil {
+			t.Errorf("failed to open a channel, err: %v", err)
+			return
+		}
+
+		exchangeName := "exchange"
+		err = ch.ExchangeDeclare(exchangeName, "direct", true, false, false, false, nil)
+		if err != nil {
+			t.Errorf("failed to declare an exchange, err: %v", err)
+			return
+		}
+
+		q, err := ch.QueueDeclare("queue", true, false, false, false, nil)
+		if err != nil {
+			t.Errorf("failed to declare a queue, err: %v", err)
+			return
+		}
+
+		routingKey := "routing_key"
+		err = ch.QueueBind(q.Name, routingKey, exchangeName, false, nil)
+		if err != nil {
+			t.Errorf("failed to bind the queue, err: %v", err)
+			return
+		}
+
+		err = ch.Publish(exchangeName, routingKey, false, false,
+			amqp.Publishing{
+				DeliveryMode: amqp.Persistent,
+				ContentType:  "text/plain",
+				Body:         []byte("message"),
+			})
+		if err != nil {
+			t.Errorf("failed to publish a message, err: %v", err)
+			return
+		}
+
+		msgChan, err := ch.Consume(q.Name, "Consumer", true, false, false, false, nil)
+		if err != nil {
+			t.Errorf("failed to create a consumer, err: %v", err)
+			return
+		}
+
+		select {
+		case <-msgChan:
+			break
+		case <-time.After(3 * time.Second):
+			t.Errorf("failed to consume a message on time")
+			return
+		}
+
+		err = ch.ExchangeDelete(exchangeName, false, false)
+		if err != nil {
+			t.Errorf("failed to delete the exchange, err: %v", err)
+			return
+		}
+
+		_, err = ch.QueueDelete(q.Name, false, false, false)
+		if err != nil {
+			t.Errorf("failed to delete the queue, err: %v", err)
+			return
+		}
+
+		ch.Close()
+	}
+
+	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/Rabbit.js\"")
+}

acceptanceTests/go.mod

@@ -0,0 +1,58 @@
+module github.com/up9inc/mizu/acceptanceTests
+
+go 1.17
+
+require (
+	github.com/go-redis/redis/v8 v8.11.4
+	github.com/rabbitmq/amqp091-go v1.3.0
+	github.com/up9inc/mizu/shared v0.0.0
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	k8s.io/apimachinery v0.23.3
+	k8s.io/client-go v0.23.3
+)
+
+require (
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-logr/logr v1.2.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.7 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/up9inc/mizu/logger v0.0.0 // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/api v0.23.3 // indirect
+	k8s.io/klog/v2 v2.40.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf // indirect
+	k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
+	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
+replace github.com/up9inc/mizu/logger v0.0.0 => ../logger
+
+replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
+
+replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
+
+replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../tap/dbgctl

acceptanceTests/logs_test.go

@@ -0,0 +1,206 @@
+package acceptanceTests
+
+import (
+	"archive/zip"
+	"os/exec"
+	"testing"
+)
+
+func TestLogs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	logsCmdArgs := GetDefaultLogsCommandArgs()
+
+	logsCmd := exec.Command(cliPath, logsCmdArgs...)
+	t.Logf("running command: %v", logsCmd.String())
+
+	if err := logsCmd.Start(); err != nil {
+		t.Errorf("failed to start logs command, err: %v", err)
+		return
+	}
+
+	if err := logsCmd.Wait(); err != nil {
+		t.Errorf("failed to wait logs command, err: %v", err)
+		return
+	}
+
+	logsPath, logsPathErr := GetLogsPath()
+	if logsPathErr != nil {
+		t.Errorf("failed to get logs path, err: %v", logsPathErr)
+		return
+	}
+
+	zipReader, zipError := zip.OpenReader(logsPath)
+	if zipError != nil {
+		t.Errorf("failed to get zip reader, err: %v", zipError)
+		return
+	}
+
+	t.Cleanup(func() {
+		if err := zipReader.Close(); err != nil {
+			t.Logf("failed to close zip reader, err: %v", err)
+		}
+	})
+
+	var logsFileNames []string
+	for _, file := range zipReader.File {
+		logsFileNames = append(logsFileNames, file.Name)
+	}
+
+	if !Contains(logsFileNames, "mizu.mizu-api-server.mizu-api-server.log") {
+		t.Errorf("api server logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu.mizu-api-server.basenine.log") {
+		t.Errorf("basenine logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_cli.log") {
+		t.Errorf("cli logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_events.log") {
+		t.Errorf("events logs not found")
+		return
+	}
+
+	if !ContainsPartOfValue(logsFileNames, "mizu.mizu-tapper-daemon-set") {
+		t.Errorf("tapper logs not found")
+		return
+	}
+}
+
+func TestLogsPath(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	logsCmdArgs := GetDefaultLogsCommandArgs()
+
+	logsPath := "../logs.zip"
+	logsCmdArgs = append(logsCmdArgs, "-f", logsPath)
+
+	logsCmd := exec.Command(cliPath, logsCmdArgs...)
+	t.Logf("running command: %v", logsCmd.String())
+
+	if err := logsCmd.Start(); err != nil {
+		t.Errorf("failed to start logs command, err: %v", err)
+		return
+	}
+
+	if err := logsCmd.Wait(); err != nil {
+		t.Errorf("failed to wait logs command, err: %v", err)
+		return
+	}
+
+	zipReader, zipError := zip.OpenReader(logsPath)
+	if zipError != nil {
+		t.Errorf("failed to get zip reader, err: %v", zipError)
+		return
+	}
+
+	t.Cleanup(func() {
+		if err := zipReader.Close(); err != nil {
+			t.Logf("failed to close zip reader, err: %v", err)
+		}
+	})
+
+	var logsFileNames []string
+	for _, file := range zipReader.File {
+		logsFileNames = append(logsFileNames, file.Name)
+	}
+
+	if !Contains(logsFileNames, "mizu.mizu-api-server.mizu-api-server.log") {
+		t.Errorf("api server logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu.mizu-api-server.basenine.log") {
+		t.Errorf("basenine logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_cli.log") {
+		t.Errorf("cli logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_events.log") {
+		t.Errorf("events logs not found")
+		return
+	}
+
+	if !ContainsPartOfValue(logsFileNames, "mizu.mizu-tapper-daemon-set") {
+		t.Errorf("tapper logs not found")
+		return
+	}
+}

acceptanceTests/setup.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+set -e
+
+PREFIX=$HOME/local/bin
+VERSION=v1.22.0
+TUNNEL_LOG="tunnel.log"
+PROXY_LOG="proxy.log"
+
+echo "Attempting to install minikube and assorted tools to $PREFIX"
+
+if ! [ -x "$(command -v kubectl)" ]; then
+  echo "Installing kubectl version $VERSION"
+  curl -LO "https://storage.googleapis.com/kubernetes-release/release/$VERSION/bin/linux/amd64/kubectl"
+  chmod +x kubectl
+  mv kubectl "$PREFIX"
+else
+  echo "kubectl is already installed"
+fi
+
+if ! [ -x "$(command -v minikube)" ]; then
+  echo "Installing minikube version $VERSION"
+  curl -Lo minikube https://storage.googleapis.com/minikube/releases/$VERSION/minikube-linux-amd64
+  chmod +x minikube
+  mv minikube "$PREFIX"
+else
+  echo "minikube is already installed"
+fi
+
+echo "Starting minikube..."
+minikube start --cpus 2 --memory 6946
+
+echo "Creating mizu tests namespaces"
+kubectl create namespace mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl create namespace mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
+
+echo "Creating httpbin deployments"
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
+
+echo "Creating redis deployment"
+kubectl create deployment redis --image=redis -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+
+echo "Creating rabbitmq deployment"
+kubectl create deployment rabbitmq --image=rabbitmq -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+
+echo "Creating httpbin services"
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
+
+echo "Creating redis service"
+kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+
+echo "Creating rabbitmq service"
+kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+
+echo "Starting proxy"
+rm -f ${PROXY_LOG}
+kubectl proxy --port=8080 > ${PROXY_LOG} &
+PID1=$!
+echo "kubectl proxy process id is ${PID1} and log of proxy in ${PROXY_LOG}"
+
+if [[ -z "${CI}" ]]; then
+  echo "Setting env var of mizu ci image"
+  export MIZU_CI_IMAGE="mizu/ci:0.0"
+  echo "Build agent image"
+  docker build -t "${MIZU_CI_IMAGE}" .
+else
+  echo "not building docker image in CI because it is created as separate step"
+fi
+
+minikube image load "${MIZU_CI_IMAGE}"
+
+echo "Build cli"
+cd cli && make build GIT_BRANCH=ci SUFFIX=ci
+
+echo "Starting tunnel"
+rm -f ${TUNNEL_LOG}
+minikube tunnel > ${TUNNEL_LOG} &
+PID2=$!
+echo "Minikube tunnel process id is ${PID2} and log of tunnel in ${TUNNEL_LOG}"

acceptanceTests/tap_test.go

@@ -0,0 +1,693 @@
+package acceptanceTests
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os/exec"
+	"path"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestTap(t *testing.T) {
+	basicTapTest(t, false)
+}
+
+func basicTapTest(t *testing.T, shouldCheckSrcAndDest bool, extraArgs... string) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []int{50}
+
+	for _, entriesCount := range tests {
+		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
+			cliPath, cliPathErr := GetCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			tapCmdArgs := GetDefaultTapCommandArgs()
+
+			tapNamespace := GetDefaultTapNamespace()
+			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+			tapCmdArgs = append(tapCmdArgs, extraArgs...)
+
+			tapCmd := exec.Command(cliPath, tapCmdArgs...)
+			t.Logf("running command: %v", tapCmd.String())
+
+			t.Cleanup(func() {
+				if err := CleanupCommand(tapCmd); err != nil {
+					t.Logf("failed to cleanup tap command, err: %v", err)
+				}
+			})
+
+			if err := tapCmd.Start(); err != nil {
+				t.Errorf("failed to start tap command, err: %v", err)
+				return
+			}
+
+			apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+			if err := WaitTapPodsReady(apiServerUrl); err != nil {
+				t.Errorf("failed to start tap pods on time, err: %v", err)
+				return
+			}
+
+			proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
+			for i := 0; i < entriesCount; i++ {
+				if _, requestErr := ExecuteHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+					t.Errorf("failed to send proxy request, err: %v", requestErr)
+					return
+				}
+			}
+
+			expectedPods := []PodDescriptor{
+				{Name: "httpbin", Namespace: "mizu-tests"},
+				{Name: "httpbin2", Namespace: "mizu-tests"},
+			}
+
+			var expectedPodsStr string
+			for i := 0; i < len(expectedPods); i++ {
+				expectedPodsStr += fmt.Sprintf("Name:%vNamespace:%v", expectedPods[i].Name, expectedPods[i].Namespace)
+			}
+
+			RunCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/e2e/tests/UiTest.js\" --env entriesCount=%d,arrayDict=%v,shouldCheckSrcAndDest=%v",
+				entriesCount, expectedPodsStr, shouldCheckSrcAndDest))
+		})
+	}
+}
+
+func TestTapGuiPort(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	tests := []uint16{8898}
+
+	for _, guiPort := range tests {
+		t.Run(fmt.Sprintf("%d", guiPort), func(t *testing.T) {
+			cliPath, cliPathErr := GetCliPath()
+			if cliPathErr != nil {
+				t.Errorf("failed to get cli path, err: %v", cliPathErr)
+				return
+			}
+
+			tapCmdArgs := GetDefaultTapCommandArgs()
+
+			tapNamespace := GetDefaultTapNamespace()
+			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+			tapCmdArgs = append(tapCmdArgs, "-p", fmt.Sprintf("%d", guiPort))
+
+			tapCmd := exec.Command(cliPath, tapCmdArgs...)
+			t.Logf("running command: %v", tapCmd.String())
+
+			t.Cleanup(func() {
+				if err := CleanupCommand(tapCmd); err != nil {
+					t.Logf("failed to cleanup tap command, err: %v", err)
+				}
+			})
+
+			if err := tapCmd.Start(); err != nil {
+				t.Errorf("failed to start tap command, err: %v", err)
+				return
+			}
+
+			apiServerUrl := GetApiServerUrl(guiPort)
+
+			if err := WaitTapPodsReady(apiServerUrl); err != nil {
+				t.Errorf("failed to start tap pods on time, err: %v", err)
+				return
+			}
+
+			proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
+			for i := 0; i < DefaultEntriesCount; i++ {
+				if _, requestErr := ExecuteHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+					t.Errorf("failed to send proxy request, err: %v", requestErr)
+					return
+				}
+			}
+
+			RunCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/e2e/tests/GuiPort.js\" --env name=%v,namespace=%v,port=%d",
+				"httpbin", "mizu-tests", guiPort))
+		})
+	}
+}
+
+func TestTapAllNamespaces(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	expectedPods := []PodDescriptor{
+		{Name: "httpbin", Namespace: "mizu-tests"},
+		{Name: "httpbin2", Namespace: "mizu-tests"},
+		{Name: "httpbin", Namespace: "mizu-tests2"},
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+	tapCmdArgs = append(tapCmdArgs, "-A")
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	RunCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/e2e/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
+		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
+}
+
+func TestTapMultipleNamespaces(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	expectedPods := []PodDescriptor{
+		{Name: "httpbin", Namespace: "mizu-tests"},
+		{Name: "httpbin2", Namespace: "mizu-tests"},
+		{Name: "httpbin", Namespace: "mizu-tests2"},
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+	var namespacesCmd []string
+	for _, expectedPod := range expectedPods {
+		namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace)
+	}
+	tapCmdArgs = append(tapCmdArgs, namespacesCmd...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	RunCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/e2e/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
+		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
+}
+
+func TestTapRegex(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	regexPodName := "httpbin2"
+	expectedPods := []PodDescriptor{
+		{Name: regexPodName, Namespace: "mizu-tests"},
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgsWithRegex(regexPodName)
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	RunCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/e2e/tests/Regex.js\" --env name=%v,namespace=%v",
+		expectedPods[0].Name, expectedPods[0].Namespace))
+}
+
+func TestTapDryRun(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmdArgs = append(tapCmdArgs, "--dry-run")
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	resultChannel := make(chan string, 1)
+
+	go func() {
+		if err := tapCmd.Wait(); err != nil {
+			resultChannel <- "fail"
+			return
+		}
+		resultChannel <- "success"
+	}()
+
+	go func() {
+		time.Sleep(ShortRetriesCount * time.Second)
+		resultChannel <- "fail"
+	}()
+
+	testResult := <-resultChannel
+	if testResult != "success" {
+		t.Errorf("unexpected result - dry run cmd not done")
+	}
+}
+
+func TestTapRedact(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+	tapCmdArgs = append(tapCmdArgs, "--redact")
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
+	requestHeaders := map[string]string{"User-Header": "Mizu"}
+	requestBody := map[string]string{"User": "Mizu"}
+	for i := 0; i < DefaultEntriesCount; i++ {
+		if _, requestErr := ExecuteHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
+			t.Errorf("failed to send proxy request, err: %v", requestErr)
+			return
+		}
+	}
+
+	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/Redact.js\"")
+}
+
+func TestTapNoRedact(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
+	requestHeaders := map[string]string{"User-Header": "Mizu"}
+	requestBody := map[string]string{"User": "Mizu"}
+	for i := 0; i < DefaultEntriesCount; i++ {
+		if _, requestErr := ExecuteHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
+			t.Errorf("failed to send proxy request, err: %v", requestErr)
+			return
+		}
+	}
+
+	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/NoRedact.js\"")
+}
+
+func TestTapRegexMasking(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmdArgs = append(tapCmdArgs, "--redact")
+
+	tapCmdArgs = append(tapCmdArgs, "-r", "Mizu")
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
+	for i := 0; i < DefaultEntriesCount; i++ {
+		response, requestErr := http.Post(fmt.Sprintf("%v/post", proxyUrl), "text/plain", bytes.NewBufferString("Mizu"))
+		if _, requestErr = ExecuteHttpRequest(response, requestErr); requestErr != nil {
+			t.Errorf("failed to send proxy request, err: %v", requestErr)
+			return
+		}
+	}
+
+	RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/RegexMasking.js\"")
+
+}
+
+func TestTapIgnoredUserAgents(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	ignoredUserAgentValue := "ignore"
+	tapCmdArgs = append(tapCmdArgs, "--set", fmt.Sprintf("tap.ignored-user-agents=%v", ignoredUserAgentValue))
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
+
+	ignoredUserAgentCustomHeader := "Ignored-User-Agent"
+	headers := map[string]string{"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
+	for i := 0; i < DefaultEntriesCount; i++ {
+		if _, requestErr := ExecuteHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil {
+			t.Errorf("failed to send proxy request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < DefaultEntriesCount; i++ {
+		if _, requestErr := ExecuteHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+			t.Errorf("failed to send proxy request, err: %v", requestErr)
+			return
+		}
+	}
+
+	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/IgnoredUserAgents.js\"")
+}
+
+func TestTapDumpLogs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmdArgs = append(tapCmdArgs, "--set", "dump-logs=true")
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	if err := CleanupCommand(tapCmd); err != nil {
+		t.Errorf("failed to cleanup tap command, err: %v", err)
+		return
+	}
+
+	mizuFolderPath, mizuPathErr := GetMizuFolderPath()
+	if mizuPathErr != nil {
+		t.Errorf("failed to get mizu folder path, err: %v", mizuPathErr)
+		return
+	}
+
+	files, readErr := ioutil.ReadDir(mizuFolderPath)
+	if readErr != nil {
+		t.Errorf("failed to read mizu folder files, err: %v", readErr)
+		return
+	}
+
+	var dumpLogsPath string
+	for _, file := range files {
+		fileName := file.Name()
+		if strings.Contains(fileName, "mizu_logs") {
+			dumpLogsPath = path.Join(mizuFolderPath, fileName)
+			break
+		}
+	}
+
+	if dumpLogsPath == "" {
+		t.Errorf("dump logs file not found")
+		return
+	}
+
+	zipReader, zipError := zip.OpenReader(dumpLogsPath)
+	if zipError != nil {
+		t.Errorf("failed to get zip reader, err: %v", zipError)
+		return
+	}
+
+	t.Cleanup(func() {
+		if err := zipReader.Close(); err != nil {
+			t.Logf("failed to close zip reader, err: %v", err)
+		}
+	})
+
+	var logsFileNames []string
+	for _, file := range zipReader.File {
+		logsFileNames = append(logsFileNames, file.Name)
+	}
+
+	if !Contains(logsFileNames, "mizu.mizu-api-server.mizu-api-server.log") {
+		t.Errorf("api server logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu.mizu-api-server.basenine.log") {
+		t.Errorf("basenine logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_cli.log") {
+		t.Errorf("cli logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_events.log") {
+		t.Errorf("events logs not found")
+		return
+	}
+
+	if !ContainsPartOfValue(logsFileNames, "mizu.mizu-tapper-daemon-set") {
+		t.Errorf("tapper logs not found")
+		return
+	}
+}
+
+func TestIpResolving(t *testing.T) {
+	namespace := AllNamespaces
+
+	t.Log("add permissions for ip-resolution for current user")
+	if err := ApplyKubeFilesForTest(
+		t,
+		"minikube",
+		namespace,
+		"../cli/cmd/permissionFiles/permissions-all-namespaces-ip-resolution-optional.yaml",
+	); err != nil {
+		t.Errorf("failed to create k8s permissions, %v", err)
+		return
+	}
+
+	basicTapTest(t, true)
+}
+
+func TestRestrictedMode(t *testing.T) {
+	namespace := "mizu-tests"
+
+	t.Log("creating permissions for restricted user")
+	if err := ApplyKubeFilesForTest(
+		t,
+		"minikube",
+		namespace,
+		"../cli/cmd/permissionFiles/permissions-ns-tap.yaml",
+	); err != nil {
+		t.Errorf("failed to create k8s permissions, %v", err)
+		return
+	}
+
+	t.Log("switching k8s context to user")
+	if err := SwitchKubeContextForTest(t, "user-with-restricted-access"); err != nil {
+		t.Errorf("failed to switch k8s context, %v", err)
+		return
+	}
+
+	extraArgs := []string{"--set", fmt.Sprintf("mizu-resources-namespace=%s", namespace)}
+	t.Run("basic tap", func (testingT *testing.T) {basicTapTest(testingT, false, extraArgs...)})
+}

acceptanceTests/testsUtils.go

@@ -0,0 +1,423 @@
+package acceptanceTests
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/homedir"
+
+	"github.com/up9inc/mizu/shared"
+)
+
+const (
+	LongRetriesCount      = 100
+	ShortRetriesCount     = 10
+	DefaultApiServerPort  = shared.DefaultApiServerPort
+	DefaultNamespaceName  = "mizu-tests"
+	DefaultServiceName    = "httpbin"
+	DefaultEntriesCount   = 50
+	WaitAfterTapPodsReady = 3 * time.Second
+	AllNamespaces         = ""
+)
+
+type PodDescriptor struct {
+	Name      string
+	Namespace string
+}
+
+func GetCliPath() (string, error) {
+	dir, filePathErr := os.Getwd()
+	if filePathErr != nil {
+		return "", filePathErr
+	}
+
+	cliPath := path.Join(dir, "../cli/bin/mizu_ci")
+	return cliPath, nil
+}
+
+func GetMizuFolderPath() (string, error) {
+	home, homeDirErr := os.UserHomeDir()
+	if homeDirErr != nil {
+		return "", homeDirErr
+	}
+
+	return path.Join(home, ".mizu"), nil
+}
+
+func GetConfigPath() (string, error) {
+	mizuFolderPath, mizuPathError := GetMizuFolderPath()
+	if mizuPathError != nil {
+		return "", mizuPathError
+	}
+
+	return path.Join(mizuFolderPath, "config.yaml"), nil
+}
+
+func GetProxyUrl(namespace string, service string) string {
+	return fmt.Sprintf("http://localhost:8080/api/v1/namespaces/%v/services/%v/proxy", namespace, service)
+}
+
+func GetApiServerUrl(port uint16) string {
+	return fmt.Sprintf("http://localhost:%v", port)
+}
+
+func NewKubernetesProvider() (*KubernetesProvider, error) {
+	home := homedir.HomeDir()
+	configLoadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(home, ".kube", "config")}
+	clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		configLoadingRules,
+		&clientcmd.ConfigOverrides{
+			CurrentContext: "",
+		},
+	)
+
+	restClientConfig, err := clientConfig.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	clientSet, err := kubernetes.NewForConfig(restClientConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return &KubernetesProvider{clientSet}, nil
+}
+
+type KubernetesProvider struct {
+	clientSet *kubernetes.Clientset
+}
+
+func (kp *KubernetesProvider) GetServiceExternalIp(ctx context.Context, namespace string, service string) (string, error) {
+	serviceObj, err := kp.clientSet.CoreV1().Services(namespace).Get(ctx, service, metav1.GetOptions{})
+	if err != nil {
+		return "", err
+	}
+
+	externalIp := serviceObj.Status.LoadBalancer.Ingress[0].IP
+	return externalIp, nil
+}
+
+func SwitchKubeContextForTest(t *testing.T, newContextName string) error {
+	prevKubeContextName, err := GetKubeCurrentContextName()
+	if err != nil {
+		return err
+	}
+
+	if err := SetKubeCurrentContext(newContextName); err != nil {
+		return err
+	}
+
+	t.Cleanup(func() {
+		if err := SetKubeCurrentContext(prevKubeContextName); err != nil {
+			t.Errorf("failed to set Kubernetes context to %s, err: %v", prevKubeContextName, err)
+			t.Errorf("cleanup failed, subsequent tests may be affected")
+		}
+	})
+
+	return nil
+}
+
+func GetKubeCurrentContextName() (string, error) {
+	cmd := exec.Command("kubectl", "config", "current-context")
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("%v, %s", err, string(output))
+	}
+
+	return string(bytes.TrimSpace(output)), nil
+}
+
+func SetKubeCurrentContext(contextName string) error {
+	cmd := exec.Command("kubectl", "config", "use-context", contextName)
+
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("%v, %s", err, string(output))
+	}
+
+	return nil
+}
+
+func ApplyKubeFilesForTest(t *testing.T, kubeContext string, namespace string, filename ...string) error {
+	for i := range filename {
+		fname := filename[i]
+		if err := ApplyKubeFile(kubeContext, namespace, fname); err != nil {
+			return err
+		}
+
+		t.Cleanup(func() {
+			if err := DeleteKubeFile(kubeContext, namespace, fname); err != nil {
+				t.Errorf(
+					"failed to delete Kubernetes resources in namespace %s from filename %s, err: %v",
+					namespace,
+					fname,
+					err,
+				)
+			}
+		})
+	}
+
+	return nil
+}
+
+func ApplyKubeFile(kubeContext string, namespace string, filename string) error {
+	cmdArgs := []string{
+		"apply",
+		"--context", kubeContext,
+		"-f", filename,
+	}
+	if namespace != AllNamespaces {
+		cmdArgs = append(cmdArgs, "-n", namespace)
+	}
+	cmd := exec.Command("kubectl", cmdArgs...)
+
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("%v, %s", err, string(output))
+	}
+
+	return nil
+}
+
+func DeleteKubeFile(kubeContext string, namespace string, filename string) error {
+	cmdArgs := []string{
+		"delete",
+		"--context", kubeContext,
+		"-f", filename,
+	}
+	if namespace != AllNamespaces {
+		cmdArgs = append(cmdArgs, "-n", namespace)
+	}
+	cmd := exec.Command("kubectl", cmdArgs...)
+
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("%v, %s", err, string(output))
+	}
+
+	return nil
+}
+
+func getDefaultCommandArgs() []string {
+	agentImageValue := os.Getenv("MIZU_CI_IMAGE")
+	setFlag := "--set"
+	telemetry := "telemetry=false"
+	agentImage := fmt.Sprintf("agent-image=%s", agentImageValue)
+	imagePullPolicy := "image-pull-policy=IfNotPresent"
+	headless := "headless=true"
+
+	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
+}
+
+func GetDefaultTapCommandArgs() []string {
+	tapCommand := "tap"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{tapCommand}, defaultCmdArgs...)
+}
+
+func GetDefaultTapCommandArgsWithRegex(regex string) []string {
+	tapCommand := "tap"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{tapCommand, regex}, defaultCmdArgs...)
+}
+
+func GetDefaultLogsCommandArgs() []string {
+	logsCommand := "logs"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{logsCommand}, defaultCmdArgs...)
+}
+
+func GetDefaultTapNamespace() []string {
+	return []string{"-n", "mizu-tests"}
+}
+
+func GetDefaultConfigCommandArgs() []string {
+	configCommand := "config"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{configCommand}, defaultCmdArgs...)
+}
+
+func RunCypressTests(t *testing.T, cypressRunCmd string) {
+	cypressCmd := exec.Command("bash", "-c", cypressRunCmd)
+	t.Logf("running command: %v", cypressCmd.String())
+	out, err := cypressCmd.CombinedOutput()
+	if err != nil {
+		t.Errorf("error running cypress, error: %v, output: %v", err, string(out))
+		return
+	}
+
+	t.Logf("%s", out)
+}
+
+func RetriesExecute(retriesCount int, executeFunc func() error) error {
+	var lastError interface{}
+
+	for i := 0; i < retriesCount; i++ {
+		if err := TryExecuteFunc(executeFunc); err != nil {
+			lastError = err
+
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		return nil
+	}
+
+	return fmt.Errorf("reached max retries count, retries count: %v, last err: %v", retriesCount, lastError)
+}
+
+func TryExecuteFunc(executeFunc func() error) (err interface{}) {
+	defer func() {
+		if panicErr := recover(); panicErr != nil {
+			err = panicErr
+		}
+	}()
+
+	return executeFunc()
+}
+
+func WaitTapPodsReady(apiServerUrl string) error {
+	resolvingUrl := fmt.Sprintf("%v/status/connectedTappersCount", apiServerUrl)
+	tapPodsReadyFunc := func() error {
+		requestResult, requestErr := ExecuteHttpGetRequest(resolvingUrl)
+		if requestErr != nil {
+			return requestErr
+		}
+
+		connectedTappersCount := requestResult.(float64)
+		if connectedTappersCount == 0 {
+			return fmt.Errorf("no connected tappers running")
+		}
+		time.Sleep(WaitAfterTapPodsReady)
+		return nil
+	}
+
+	return RetriesExecute(LongRetriesCount, tapPodsReadyFunc)
+}
+
+func JsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
+	var result interface{}
+	if parseErr := json.Unmarshal(jsonBytes, &result); parseErr != nil {
+		return nil, parseErr
+	}
+
+	return result, nil
+}
+
+func ExecuteHttpRequest(response *http.Response, requestErr error) (interface{}, error) {
+	if requestErr != nil {
+		return nil, requestErr
+	} else if response.StatusCode != 200 {
+		return nil, fmt.Errorf("invalid status code %v", response.StatusCode)
+	}
+
+	defer func() { response.Body.Close() }()
+
+	data, readErr := ioutil.ReadAll(response.Body)
+	if readErr != nil {
+		return nil, readErr
+	}
+
+	return JsonBytesToInterface(data)
+}
+
+func ExecuteHttpGetRequestWithHeaders(url string, headers map[string]string) (interface{}, error) {
+	request, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	for headerKey, headerValue := range headers {
+		request.Header.Add(headerKey, headerValue)
+	}
+
+	client := &http.Client{}
+	response, requestErr := client.Do(request)
+	return ExecuteHttpRequest(response, requestErr)
+}
+
+func ExecuteHttpGetRequest(url string) (interface{}, error) {
+	response, requestErr := http.Get(url)
+	return ExecuteHttpRequest(response, requestErr)
+}
+
+func ExecuteHttpPostRequestWithHeaders(url string, headers map[string]string, body interface{}) (interface{}, error) {
+	requestBody, jsonErr := json.Marshal(body)
+	if jsonErr != nil {
+		return nil, jsonErr
+	}
+
+	request, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(requestBody))
+	if err != nil {
+		return nil, err
+	}
+
+	request.Header.Add("Content-Type", "application/json")
+	for headerKey, headerValue := range headers {
+		request.Header.Add(headerKey, headerValue)
+	}
+
+	client := &http.Client{}
+	response, requestErr := client.Do(request)
+	return ExecuteHttpRequest(response, requestErr)
+}
+
+func CleanupCommand(cmd *exec.Cmd) error {
+	if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
+		return err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func GetLogsPath() (string, error) {
+	dir, filePathErr := os.Getwd()
+	if filePathErr != nil {
+		return "", filePathErr
+	}
+
+	logsPath := path.Join(dir, "mizu_logs.zip")
+	return logsPath, nil
+}
+
+func Contains(slice []string, containsValue string) bool {
+	for _, sliceValue := range slice {
+		if sliceValue == containsValue {
+			return true
+		}
+	}
+
+	return false
+}
+
+func ContainsPartOfValue(slice []string, containsValue string) bool {
+	for _, sliceValue := range slice {
+		if strings.Contains(sliceValue, containsValue) {
+			return true
+		}
+	}
+
+	return false
+}

agent/.snyk

@@ -0,0 +1,6 @@
+# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
+version: v1.14.0
+ignore:
+  SNYK-GOLANG-GITHUBCOMGINGONICGIN-1041736:
+    - '*':
+      reason: None Given

agent/Makefile

@@ -0,0 +1,2 @@
+test: ## Run agent tests.
+	@go test ./... -coverpkg=./... -race -coverprofile=coverage.out -covermode=atomic

agent/go.mod

@@ -0,0 +1,164 @@
+module github.com/up9inc/mizu/agent
+
+go 1.17
+
+require (
+	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
+	github.com/chanced/openapi v0.0.8
+	github.com/djherbis/atime v1.1.0
+	github.com/gin-contrib/pprof v1.3.0
+	github.com/gin-contrib/static v0.0.1
+	github.com/gin-gonic/gin v1.7.7
+	github.com/go-playground/locales v0.14.0
+	github.com/go-playground/universal-translator v0.18.0
+	github.com/go-playground/validator/v10 v10.10.0
+	github.com/google/uuid v1.3.0
+	github.com/gorilla/websocket v1.4.2
+	github.com/jinzhu/copier v0.3.5
+	github.com/nav-inc/datetime v0.1.3
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
+	github.com/orcaman/concurrent-map v1.0.0
+	github.com/stretchr/testify v1.7.0
+	github.com/up9inc/basenine/client/go v0.0.0-20220612112747-3b28eeac9c51
+	github.com/up9inc/mizu/logger v0.0.0
+	github.com/up9inc/mizu/shared v0.0.0
+	github.com/up9inc/mizu/tap v0.0.0
+	github.com/up9inc/mizu/tap/api v0.0.0
+	github.com/up9inc/mizu/tap/dbgctl v0.0.0
+	github.com/up9inc/mizu/tap/extensions/amqp v0.0.0
+	github.com/up9inc/mizu/tap/extensions/http v0.0.0
+	github.com/up9inc/mizu/tap/extensions/kafka v0.0.0
+	github.com/up9inc/mizu/tap/extensions/redis v0.0.0
+	github.com/wI2L/jsondiff v0.1.1
+	k8s.io/api v0.23.3
+	k8s.io/apimachinery v0.23.3
+	k8s.io/client-go v0.23.3
+)
+
+require (
+	cloud.google.com/go/compute v1.2.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.24 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.1 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/MakeNowJust/heredoc v1.0.0 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/beevik/etree v1.1.0 // indirect
+	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
+	github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect
+	github.com/cilium/ebpf v0.9.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
+	github.com/fatih/camelcase v1.0.0 // indirect
+	github.com/fvbommel/sortorder v1.0.2 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-logr/logr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.6 // indirect
+	github.com/go-openapi/swag v0.21.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/btree v1.0.1 // indirect
+	github.com/google/go-cmp v0.5.7 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/martian v2.1.0+incompatible // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.14.2 // indirect
+	github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9 // indirect
+	github.com/leodido/go-urn v1.2.1 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mertyildiran/gqlparser/v2 v2.4.6 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/moby/moby v20.10.17+incompatible // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/ohler55/ojg v1.12.12 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/russross/blackfriday v1.6.0 // indirect
+	github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
+	github.com/segmentio/kafka-go v0.4.27 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/spf13/cobra v1.3.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/struCoder/pidusage v0.2.1 // indirect
+	github.com/tidwall/gjson v1.14.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/sjson v1.2.4 // indirect
+	github.com/tklauser/go-sysconf v0.3.10 // indirect
+	github.com/tklauser/numcpus v0.4.0 // indirect
+	github.com/ugorji/go/codec v1.2.6 // indirect
+	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
+	github.com/xlab/treeprint v1.1.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	go.starlark.net v0.0.0-20220203230714-bb14e151c28f // indirect
+	golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	k8s.io/cli-runtime v0.23.3 // indirect
+	k8s.io/component-base v0.23.3 // indirect
+	k8s.io/klog/v2 v2.40.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf // indirect
+	k8s.io/kubectl v0.23.3 // indirect
+	k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
+	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
+	sigs.k8s.io/kustomize/api v0.11.1 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.13.3 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
+replace github.com/up9inc/mizu/logger v0.0.0 => ../logger
+
+replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
+
+replace github.com/up9inc/mizu/tap v0.0.0 => ../tap
+
+replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
+
+replace github.com/up9inc/mizu/tap/extensions/amqp v0.0.0 => ../tap/extensions/amqp
+
+replace github.com/up9inc/mizu/tap/extensions/http v0.0.0 => ../tap/extensions/http
+
+replace github.com/up9inc/mizu/tap/extensions/kafka v0.0.0 => ../tap/extensions/kafka
+
+replace github.com/up9inc/mizu/tap/extensions/redis v0.0.0 => ../tap/extensions/redis
+
+replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../tap/dbgctl

agent/main.go

@@ -0,0 +1,372 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/gin-contrib/pprof"
+	"github.com/gin-contrib/static"
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/entries"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/oas"
+	"github.com/up9inc/mizu/agent/pkg/routes"
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+
+	"github.com/up9inc/mizu/agent/pkg/api"
+	"github.com/up9inc/mizu/agent/pkg/app"
+	"github.com/up9inc/mizu/agent/pkg/config"
+
+	"github.com/gorilla/websocket"
+	"github.com/op/go-logging"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap"
+	tapApi "github.com/up9inc/mizu/tap/api"
+	"github.com/up9inc/mizu/tap/dbgctl"
+)
+
+var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API")
+var apiServerMode = flag.Bool("api-server", false, "Run in API server mode with API")
+var standaloneMode = flag.Bool("standalone", false, "Run in standalone tapper and API mode")
+var apiServerAddress = flag.String("api-server-address", "", "Address of mizu API server")
+var namespace = flag.String("namespace", "", "Resolve IPs if they belong to resources in this namespace (default is all)")
+var harsReaderMode = flag.Bool("hars-read", false, "Run in hars-read mode")
+var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
+var profiler = flag.Bool("profiler", false, "Run pprof server")
+
+const (
+	socketConnectionRetries    = 30
+	socketConnectionRetryDelay = time.Second * 2
+	socketHandshakeTimeout     = time.Second * 2
+)
+
+func main() {
+	initializeDependencies()
+	logLevel := determineLogLevel()
+	logger.InitLoggerStd(logLevel)
+	flag.Parse()
+
+	app.LoadExtensions()
+
+	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode {
+		panic("One of the flags --tap, --api-server, --standalone or --hars-read must be provided")
+	}
+
+	if *standaloneMode {
+		runInStandaloneMode()
+	} else if *tapperMode {
+		runInTapperMode()
+	} else if *apiServerMode {
+		ginApp := runInApiServerMode(*namespace)
+
+		if *profiler {
+			pprof.Register(ginApp)
+		}
+
+		utils.StartServer(ginApp)
+
+	} else if *harsReaderMode {
+		runInHarReaderMode()
+	}
+
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, os.Interrupt)
+	<-signalChan
+
+	logger.Log.Info("Exiting")
+}
+
+func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engine {
+	ginApp := gin.Default()
+
+	ginApp.GET("/echo", func(c *gin.Context) {
+		c.JSON(http.StatusOK, "Here is Mizu agent")
+	})
+
+	eventHandlers := api.RoutesEventHandlers{
+		SocketOutChannel: socketHarOutputChannel,
+	}
+
+	ginApp.Use(disableRootStaticCache())
+
+	staticFolder := "./site"
+	indexStaticFile := staticFolder + "/index.html"
+	if err := setUIFlags(indexStaticFile); err != nil {
+		logger.Log.Errorf("Error setting ui flags, err: %v", err)
+	}
+
+	ginApp.Use(static.ServeRoot("/", staticFolder))
+	ginApp.NoRoute(func(c *gin.Context) {
+		c.File(indexStaticFile)
+	})
+
+	ginApp.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if it's called before
+
+	api.WebSocketRoutes(ginApp, &eventHandlers)
+
+	if config.Config.OAS.Enable {
+		routes.OASRoutes(ginApp)
+	}
+
+	if config.Config.ServiceMap {
+		routes.ServiceMapRoutes(ginApp)
+	}
+
+	routes.QueryRoutes(ginApp)
+	routes.EntriesRoutes(ginApp)
+	routes.MetadataRoutes(ginApp)
+	routes.StatusRoutes(ginApp)
+	routes.DbRoutes(ginApp)
+	routes.ReplayRoutes(ginApp)
+
+	return ginApp
+}
+
+func runInApiServerMode(namespace string) *gin.Engine {
+	if err := config.LoadConfig(); err != nil {
+		logger.Log.Fatalf("Error loading config file %v", err)
+	}
+	app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel, config.Config.InsertionFilter)
+	api.StartResolving(namespace)
+
+	enableExpFeatureIfNeeded()
+
+	return hostApi(app.GetEntryInputChannel())
+}
+
+func runInTapperMode() {
+	logger.Log.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
+	if *apiServerAddress == "" {
+		panic("API server address must be provided with --api-server-address when using --tap")
+	}
+
+	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+	tapOpts := &tap.TapOpts{
+		HostMode: hostMode,
+	}
+
+	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+
+	filteringOptions := getTrafficFilteringOptions()
+	tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, app.Extensions, filteringOptions)
+	socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
+	if err != nil {
+		panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
+	}
+	logger.Log.Infof("Connected successfully to websocket %s", *apiServerAddress)
+
+	go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
+}
+
+func runInStandaloneMode() {
+	api.StartResolving(*namespace)
+
+	outputItemsChannel := make(chan *tapApi.OutputChannelItem)
+	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+
+	filteringOptions := getTrafficFilteringOptions()
+	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+	tapOpts := &tap.TapOpts{HostMode: hostMode}
+	tap.StartPassiveTapper(tapOpts, outputItemsChannel, app.Extensions, filteringOptions)
+
+	go app.FilterItems(outputItemsChannel, filteredOutputItemsChannel)
+	go api.StartReadingEntries(filteredOutputItemsChannel, nil, app.ExtensionsMap)
+
+	ginApp := hostApi(nil)
+	utils.StartServer(ginApp)
+}
+
+func runInHarReaderMode() {
+	outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
+	filteredHarChannel := make(chan *tapApi.OutputChannelItem)
+
+	go app.FilterItems(outputItemsChannel, filteredHarChannel)
+	go api.StartReadingEntries(filteredHarChannel, harsDir, app.ExtensionsMap)
+	ginApp := hostApi(nil)
+	utils.StartServer(ginApp)
+}
+
+func enableExpFeatureIfNeeded() {
+	if config.Config.OAS.Enable {
+		oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
+		oasGenerator.Start()
+	}
+	if config.Config.ServiceMap {
+		serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap)
+		serviceMapGenerator.Enable()
+	}
+}
+
+func disableRootStaticCache() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		if c.Request.RequestURI == "/" {
+			// Disable cache only for the main static route
+			c.Writer.Header().Set("Cache-Control", "no-store")
+		}
+
+		c.Next()
+	}
+}
+
+func setUIFlags(uiIndexPath string) error {
+	read, err := ioutil.ReadFile(uiIndexPath)
+	if err != nil {
+		return err
+	}
+
+	replacedContent := strings.Replace(string(read), "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS.Enable), 1)
+	replacedContent = strings.Replace(replacedContent, "__IS_SERVICE_MAP_ENABLED__", strconv.FormatBool(config.Config.ServiceMap), 1)
+
+	err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
+	filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar)
+	if filteringOptionsJson == "" {
+		return &tapApi.TrafficFilteringOptions{
+			IgnoredUserAgents: []string{},
+		}
+	}
+	var filteringOptions tapApi.TrafficFilteringOptions
+	err := json.Unmarshal([]byte(filteringOptionsJson), &filteringOptions)
+	if err != nil {
+		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the api.TrafficFilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err))
+	}
+
+	return &filteringOptions
+}
+
+func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tapApi.OutputChannelItem) {
+	if connection == nil {
+		panic("Websocket connection is nil")
+	}
+
+	if messageDataChannel == nil {
+		panic("Channel of captured messages is nil")
+	}
+
+	for messageData := range messageDataChannel {
+		marshaledData, err := models.CreateWebsocketTappedEntryMessage(messageData)
+		if err != nil {
+			logger.Log.Errorf("error converting message to json %v, err: %s, (%v,%+v)", messageData, err, err, err)
+			continue
+		}
+
+		if dbgctl.MizuTapperDisableSending {
+			continue
+		}
+
+		// NOTE: This is where the `*tapApi.OutputChannelItem` leaves the code
+		// and goes into the intermediate WebSocket.
+		err = connection.WriteMessage(websocket.TextMessage, marshaledData)
+		if err != nil {
+			logger.Log.Errorf("error sending message through socket server %v, err: %s, (%v,%+v)", messageData, err, err, err)
+			if errors.Is(err, syscall.EPIPE) {
+				logger.Log.Warning("detected socket disconnection, reestablishing socket connection")
+				connection, err = dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
+				if err != nil {
+					logger.Log.Fatalf("error reestablishing socket connection: %v", err)
+				} else {
+					logger.Log.Info("recovered connection successfully")
+				}
+			}
+			continue
+		}
+	}
+}
+
+func determineLogLevel() (logLevel logging.Level) {
+	logLevel, err := logging.LogLevel(os.Getenv(shared.LogLevelEnvVar))
+	if err != nil {
+		logLevel = logging.INFO
+	}
+
+	return
+}
+
+func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time.Duration) (*websocket.Conn, error) {
+	var lastErr error
+	dialer := &websocket.Dialer{ // we use our own dialer instead of the default due to the default's 45 sec handshake timeout, we occasionally encounter hanging socket handshakes when tapper tries to connect to api too soon
+		Proxy:            http.ProxyFromEnvironment,
+		HandshakeTimeout: socketHandshakeTimeout,
+	}
+	for i := 1; i < retryAmount; i++ {
+		socketConnection, _, err := dialer.Dial(socketAddress, nil)
+		if err != nil {
+			lastErr = err
+			if i < retryAmount {
+				logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay/time.Second)
+				time.Sleep(retryDelay)
+			}
+		} else {
+			go handleIncomingMessageAsTapper(socketConnection)
+			return socketConnection, nil
+		}
+	}
+	return nil, lastErr
+}
+
+func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
+	for {
+		if _, message, err := socketConnection.ReadMessage(); err != nil {
+			logger.Log.Errorf("error reading message from socket connection, err: %s, (%v,%+v)", err, err, err)
+			if errors.Is(err, syscall.EPIPE) {
+				// socket has disconnected, we can safely stop this goroutine
+				return
+			}
+		} else {
+			var socketMessageBase shared.WebSocketMessageMetadata
+			if err := json.Unmarshal(message, &socketMessageBase); err != nil {
+				logger.Log.Errorf("Could not unmarshal websocket message %v", err)
+			} else {
+				switch socketMessageBase.MessageType {
+				case shared.WebSocketMessageTypeTapConfig:
+					var tapConfigMessage *shared.WebSocketTapConfigMessage
+					if err := json.Unmarshal(message, &tapConfigMessage); err != nil {
+						logger.Log.Errorf("received unknown message from socket connection: %s, err: %s, (%v,%+v)", string(message), err, err, err)
+					} else {
+						tap.UpdateTapTargets(tapConfigMessage.TapTargets)
+					}
+				case shared.WebSocketMessageTypeUpdateTappedPods:
+					var tappedPodsMessage shared.WebSocketTappedPodsMessage
+					if err := json.Unmarshal(message, &tappedPodsMessage); err != nil {
+						logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
+						return
+					}
+					nodeName := os.Getenv(shared.NodeNameEnvVar)
+					tap.UpdateTapTargets(tappedPodsMessage.NodeToTappedPodMap[nodeName])
+				default:
+					logger.Log.Warningf("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
+				}
+			}
+		}
+	}
+}
+
+func initializeDependencies() {
+	dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
+	dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} { return oas.GetDefaultOasGeneratorInstance(config.Config.OAS.MaxExampleLen) })
+	dependency.RegisterGenerator(dependency.EntriesInserter, func() interface{} { return api.GetBasenineEntryInserterInstance() })
+	dependency.RegisterGenerator(dependency.EntriesProvider, func() interface{} { return &entries.BasenineEntriesProvider{} })
+	dependency.RegisterGenerator(dependency.EntriesSocketStreamer, func() interface{} { return &api.BasenineEntryStreamer{} })
+	dependency.RegisterGenerator(dependency.EntryStreamerSocketConnector, func() interface{} { return &api.DefaultEntryStreamerSocketConnector{} })
+}

agent/pkg/api/entry_streamer_socket_connector.go

@@ -0,0 +1,71 @@
+package api
+
+import (
+	"fmt"
+
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+type EntryStreamerSocketConnector interface {
+	SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error
+	SendMetadata(socketId int, metadata *basenine.Metadata) error
+	SendToastError(socketId int, err error) error
+	CleanupSocket(socketId int)
+}
+
+type DefaultEntryStreamerSocketConnector struct{}
+
+func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error {
+	var message []byte
+	if params.EnableFullEntries {
+		message, _ = models.CreateFullEntryWebSocketMessage(entry)
+	} else {
+		protocol, ok := protocolsMap[entry.Protocol.ToString()]
+		if !ok {
+			return fmt.Errorf("protocol not found, protocol: %v", protocol)
+		}
+
+		extension, ok := extensionsMap[protocol.Name]
+		if !ok {
+			return fmt.Errorf("extension not found, extension: %v", protocol.Name)
+		}
+
+		base := extension.Dissector.Summarize(entry)
+		message, _ = models.CreateBaseEntryWebSocketMessage(base)
+	}
+
+	if err := SendToSocket(socketId, message); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (e *DefaultEntryStreamerSocketConnector) SendMetadata(socketId int, metadata *basenine.Metadata) error {
+	metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
+	if err := SendToSocket(socketId, metadataBytes); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (e *DefaultEntryStreamerSocketConnector) SendToastError(socketId int, err error) error {
+	toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
+		Type:      "error",
+		AutoClose: 5000,
+		Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
+	})
+	if err := SendToSocket(socketId, toastBytes); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (e *DefaultEntryStreamerSocketConnector) CleanupSocket(socketId int) {
+	socketObj := connectedWebsockets[socketId]
+	socketCleanup(socketId, socketObj)
+}

agent/pkg/api/main.go

@@ -0,0 +1,171 @@
+package api
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/oas"
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/agent/pkg/holder"
+	"github.com/up9inc/mizu/agent/pkg/providers"
+
+	"github.com/up9inc/mizu/agent/pkg/resolver"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+
+	"github.com/up9inc/mizu/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+var k8sResolver *resolver.Resolver
+
+func StartResolving(namespace string) {
+	errOut := make(chan error, 100)
+	res, err := resolver.NewFromInCluster(errOut, namespace)
+	if err != nil {
+		logger.Log.Infof("error creating k8s resolver %s", err)
+		return
+	}
+	ctx := context.Background()
+	res.Start(ctx)
+	go func() {
+		for {
+			err := <-errOut
+			logger.Log.Infof("name resolving error %s", err)
+		}
+	}()
+
+	k8sResolver = res
+	holder.SetResolver(res)
+}
+
+func StartReadingEntries(harChannel <-chan *tapApi.OutputChannelItem, workingDir *string, extensionsMap map[string]*tapApi.Extension) {
+	if workingDir != nil && *workingDir != "" {
+		startReadingFiles(*workingDir)
+	} else {
+		startReadingChannel(harChannel, extensionsMap)
+	}
+}
+
+func startReadingFiles(workingDir string) {
+	if err := os.MkdirAll(workingDir, os.ModePerm); err != nil {
+		logger.Log.Errorf("Failed to make dir: %s, err: %v", workingDir, err)
+		return
+	}
+
+	for {
+		dir, _ := os.Open(workingDir)
+		dirFiles, _ := dir.Readdir(-1)
+
+		var harFiles []os.FileInfo
+		for _, fileInfo := range dirFiles {
+			if strings.HasSuffix(fileInfo.Name(), ".har") {
+				harFiles = append(harFiles, fileInfo)
+			}
+		}
+		sort.Sort(utils.ByModTime(harFiles))
+
+		if len(harFiles) == 0 {
+			logger.Log.Infof("Waiting for new files")
+			time.Sleep(3 * time.Second)
+			continue
+		}
+		fileInfo := harFiles[0]
+		inputFilePath := path.Join(workingDir, fileInfo.Name())
+		file, err := os.Open(inputFilePath)
+		utils.CheckErr(err)
+
+		var inputHar har.HAR
+		decErr := json.NewDecoder(bufio.NewReader(file)).Decode(&inputHar)
+		utils.CheckErr(decErr)
+
+		rmErr := os.Remove(inputFilePath)
+		utils.CheckErr(rmErr)
+	}
+}
+
+func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extensionsMap map[string]*tapApi.Extension) {
+	if outputItems == nil {
+		panic("Channel of captured messages is nil")
+	}
+
+	for item := range outputItems {
+		extension := extensionsMap[item.Protocol.Name]
+		resolvedSource, resolvedDestination, namespace := resolveIP(item.ConnectionInfo)
+
+		if namespace == "" && item.Namespace != tapApi.UnknownNamespace {
+			namespace = item.Namespace
+		}
+
+		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestination, namespace)
+
+		data, err := json.Marshal(mizuEntry)
+		if err != nil {
+			logger.Log.Errorf("Error while marshaling entry: %v", err)
+			continue
+		}
+
+		entryInserter := dependency.GetInstance(dependency.EntriesInserter).(EntryInserter)
+		if err := entryInserter.Insert(mizuEntry); err != nil {
+			logger.Log.Errorf("Error inserting entry, err: %v", err)
+		}
+
+		summary := extension.Dissector.Summarize(mizuEntry)
+		providers.EntryAdded(len(data), summary)
+
+		serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMapSink)
+		serviceMapGenerator.NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
+
+		oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGeneratorSink)
+		oasGenerator.HandleEntry(mizuEntry)
+	}
+}
+
+func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, resolvedDestination string, namespace string) {
+	if k8sResolver != nil {
+		unresolvedSource := connectionInfo.ClientIP
+		resolvedSourceObject := k8sResolver.Resolve(unresolvedSource)
+		if resolvedSourceObject == nil {
+			logger.Log.Debugf("Cannot find resolved name to source: %s", unresolvedSource)
+			if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
+				return
+			}
+		} else {
+			resolvedSource = resolvedSourceObject.FullAddress
+			namespace = resolvedSourceObject.Namespace
+		}
+
+		unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
+		resolvedDestinationObject := k8sResolver.Resolve(unresolvedDestination)
+		if resolvedDestinationObject == nil {
+			logger.Log.Debugf("Cannot find resolved name to dest: %s", unresolvedDestination)
+			if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
+				return
+			}
+		} else {
+			resolvedDestination = resolvedDestinationObject.FullAddress
+			// Overwrite namespace (if it was set according to the source)
+			// Only overwrite if non-empty
+			if resolvedDestinationObject.Namespace != "" {
+				namespace = resolvedDestinationObject.Namespace
+			}
+		}
+	}
+	return resolvedSource, resolvedDestination, namespace
+}
+
+func CheckIsServiceIP(address string) bool {
+	if k8sResolver == nil {
+		return false
+	}
+	return k8sResolver.CheckIsServiceIP(address)
+}

agent/pkg/api/socket_data_inserter.go

@@ -0,0 +1,71 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap/api"
+	"sync"
+	"time"
+)
+
+type EntryInserter interface {
+	Insert(entry *api.Entry) error
+}
+
+type BasenineEntryInserter struct {
+	connection *basenine.Connection
+}
+
+var instance *BasenineEntryInserter
+var once sync.Once
+
+func GetBasenineEntryInserterInstance() *BasenineEntryInserter {
+	once.Do(func() {
+		instance = &BasenineEntryInserter{}
+	})
+
+	return instance
+}
+
+func (e *BasenineEntryInserter) Insert(entry *api.Entry) error {
+	if e.connection == nil {
+		e.connection = initializeConnection()
+	}
+
+	data, err := json.Marshal(entry)
+	if err != nil {
+		return fmt.Errorf("error marshling entry, err: %v", err)
+	}
+
+	if err := e.connection.SendText(string(data)); err != nil {
+		e.connection.Close()
+		e.connection = nil
+
+		return fmt.Errorf("error sending text to database, err: %v", err)
+	}
+
+	return nil
+}
+
+func initializeConnection() *basenine.Connection{
+	for {
+		connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+		if err != nil {
+			logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
+			time.Sleep(shared.BasenineReconnectInterval * time.Second)
+			continue
+		}
+
+		if err = connection.InsertMode(); err != nil {
+			logger.Log.Errorf("Insert mode call failed: %v", err)
+			connection.Close()
+			time.Sleep(shared.BasenineReconnectInterval * time.Second)
+			continue
+		}
+
+		return connection
+	}
+}

agent/pkg/api/socket_data_streamer.go

@@ -0,0 +1,171 @@
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+type EntryStreamer interface {
+	Get(ctx context.Context, socketId int, params *WebSocketParams) error
+}
+
+type BasenineEntryStreamer struct{}
+
+func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *WebSocketParams) error {
+	var connection *basenine.Connection
+
+	entryStreamerSocketConnector := dependency.GetInstance(dependency.EntryStreamerSocketConnector).(EntryStreamerSocketConnector)
+
+	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+	if err != nil {
+		logger.Log.Errorf("Failed to establish a connection to Basenine: %v", err)
+		entryStreamerSocketConnector.CleanupSocket(socketId)
+		return err
+	}
+
+	data := make(chan []byte)
+	meta := make(chan []byte)
+
+	query := params.Query
+	if err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query); err != nil {
+		if err := entryStreamerSocketConnector.SendToastError(socketId, err); err != nil {
+			return err
+		}
+
+		entryStreamerSocketConnector.CleanupSocket(socketId)
+		return err
+	}
+
+	leftOff, err := e.fetch(socketId, params, entryStreamerSocketConnector)
+	if err != nil {
+		logger.Log.Errorf("Fetch error: %v", err)
+	}
+
+	handleDataChannel := func(c *basenine.Connection, data chan []byte) {
+		for {
+			bytes := <-data
+
+			if string(bytes) == basenine.CloseChannel {
+				return
+			}
+
+			var entry *tapApi.Entry
+			if err = json.Unmarshal(bytes, &entry); err != nil {
+				logger.Log.Debugf("Error unmarshalling entry: %v", err)
+				continue
+			}
+
+			if err := entryStreamerSocketConnector.SendEntry(socketId, entry, params); err != nil {
+				logger.Log.Errorf("Error sending entry to socket, err: %v", err)
+				return
+			}
+		}
+	}
+
+	handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
+		for {
+			bytes := <-meta
+
+			if string(bytes) == basenine.CloseChannel {
+				return
+			}
+
+			var metadata *basenine.Metadata
+			if err = json.Unmarshal(bytes, &metadata); err != nil {
+				logger.Log.Debugf("Error unmarshalling metadata: %v", err)
+				continue
+			}
+
+			if err := entryStreamerSocketConnector.SendMetadata(socketId, metadata); err != nil {
+				logger.Log.Errorf("Error sending metadata to socket, err: %v", err)
+				return
+			}
+		}
+	}
+
+	go handleDataChannel(connection, data)
+	go handleMetaChannel(connection, meta)
+
+	if err = connection.Query(leftOff, query, data, meta); err != nil {
+		logger.Log.Errorf("Query mode call failed: %v", err)
+		entryStreamerSocketConnector.CleanupSocket(socketId)
+		return err
+	}
+
+	go func() {
+		<-ctx.Done()
+		data <- []byte(basenine.CloseChannel)
+		meta <- []byte(basenine.CloseChannel)
+		connection.Close()
+	}()
+
+	return nil
+}
+
+// Reverses a []byte slice.
+func (e *BasenineEntryStreamer) fetch(socketId int, params *WebSocketParams, connector EntryStreamerSocketConnector) (leftOff string, err error) {
+	if params.Fetch <= 0 {
+		leftOff = params.LeftOff
+		return
+	}
+
+	var data [][]byte
+	var firstMeta []byte
+	var lastMeta []byte
+	data, firstMeta, lastMeta, err = basenine.Fetch(
+		shared.BasenineHost,
+		shared.BaseninePort,
+		params.LeftOff,
+		-1,
+		params.Query,
+		params.Fetch,
+		time.Duration(params.TimeoutMs)*time.Millisecond,
+	)
+	if err != nil {
+		return
+	}
+
+	var firstMetadata *basenine.Metadata
+	if err = json.Unmarshal(firstMeta, &firstMetadata); err != nil {
+		return
+	}
+
+	leftOff = firstMetadata.LeftOff
+
+	var lastMetadata *basenine.Metadata
+	if err = json.Unmarshal(lastMeta, &lastMetadata); err != nil {
+		return
+	}
+
+	if err = connector.SendMetadata(socketId, lastMetadata); err != nil {
+		return
+	}
+
+	data = e.reverseBytesSlice(data)
+	for _, row := range data {
+		var entry *tapApi.Entry
+		if err = json.Unmarshal(row, &entry); err != nil {
+			break
+		}
+
+		if err = connector.SendEntry(socketId, entry, params); err != nil {
+			return
+		}
+	}
+	return
+}
+
+// Reverses a []byte slice.
+func (e *BasenineEntryStreamer) reverseBytesSlice(arr [][]byte) (newArr [][]byte) {
+	for i := len(arr) - 1; i >= 0; i-- {
+		newArr = append(newArr, arr[i])
+	}
+	return newArr
+}

agent/pkg/api/socket_routes.go

@@ -0,0 +1,166 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/gorilla/websocket"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+	"github.com/up9inc/mizu/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+var (
+	extensionsMap map[string]*tapApi.Extension // global
+	protocolsMap  map[string]*tapApi.Protocol  //global
+)
+
+func InitMaps(extensions map[string]*tapApi.Extension, protocols map[string]*tapApi.Protocol) {
+	extensionsMap = extensions
+	protocolsMap = protocols
+}
+
+type EventHandlers interface {
+	WebSocketConnect(c *gin.Context, socketId int, isTapper bool)
+	WebSocketDisconnect(socketId int, isTapper bool)
+	WebSocketMessage(socketId int, isTapper bool, message []byte)
+}
+
+type SocketConnection struct {
+	connection    *websocket.Conn
+	lock          *sync.Mutex
+	eventHandlers EventHandlers
+	isTapper      bool
+}
+
+type WebSocketParams struct {
+	LeftOff           string `json:"leftOff"`
+	Query             string `json:"query"`
+	EnableFullEntries bool   `json:"enableFullEntries"`
+	Fetch             int    `json:"fetch"`
+	TimeoutMs         int    `json:"timeoutMs"`
+}
+
+var (
+	websocketUpgrader = websocket.Upgrader{
+		ReadBufferSize:  1024,
+		WriteBufferSize: 1024,
+	}
+
+	websocketIdsLock            = sync.Mutex{}
+	connectedWebsockets         map[int]*SocketConnection
+	connectedWebsocketIdCounter = 0
+	SocketGetBrowserHandler     gin.HandlerFunc
+	SocketGetTapperHandler      gin.HandlerFunc
+)
+
+func init() {
+	websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
+	connectedWebsockets = make(map[int]*SocketConnection)
+}
+
+func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
+	SocketGetBrowserHandler = func(c *gin.Context) {
+		websocketHandler(c, eventHandlers, false)
+	}
+
+	SocketGetTapperHandler = func(c *gin.Context) {
+		websocketHandler(c, eventHandlers, true)
+	}
+
+	app.GET("/ws", func(c *gin.Context) {
+		SocketGetBrowserHandler(c)
+	})
+
+	app.GET("/wsTapper", func(c *gin.Context) { // TODO: add m2m authentication to this route
+		SocketGetTapperHandler(c)
+	})
+}
+
+func websocketHandler(c *gin.Context, eventHandlers EventHandlers, isTapper bool) {
+	ws, err := websocketUpgrader.Upgrade(c.Writer, c.Request, nil)
+	if err != nil {
+		logger.Log.Errorf("failed to set websocket upgrade: %v", err)
+		return
+	}
+
+	websocketIdsLock.Lock()
+
+	connectedWebsocketIdCounter++
+	socketId := connectedWebsocketIdCounter
+	connectedWebsockets[socketId] = &SocketConnection{connection: ws, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
+
+	websocketIdsLock.Unlock()
+
+	defer func() {
+		if socketConnection := connectedWebsockets[socketId]; socketConnection != nil {
+			socketCleanup(socketId, socketConnection)
+		}
+	}()
+
+	eventHandlers.WebSocketConnect(c, socketId, isTapper)
+
+	startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(utils.StartTime)
+
+	if err = SendToSocket(socketId, startTimeBytes); err != nil {
+		logger.Log.Error(err)
+	}
+
+	for {
+		_, msg, err := ws.ReadMessage()
+		if err != nil {
+			if _, ok := err.(*websocket.CloseError); ok {
+				logger.Log.Debugf("received websocket close message, socket id: %d", socketId)
+			} else {
+				logger.Log.Errorf("error reading message, socket id: %d, error: %v", socketId, err)
+			}
+
+			break
+		}
+
+		eventHandlers.WebSocketMessage(socketId, isTapper, msg)
+	}
+}
+
+func SendToSocket(socketId int, message []byte) error {
+	socketObj := connectedWebsockets[socketId]
+	if socketObj == nil {
+		return fmt.Errorf("socket %v is disconnected", socketId)
+	}
+
+	socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
+	defer socketObj.lock.Unlock()
+
+	if connectedWebsockets[socketId] == nil {
+		return fmt.Errorf("socket %v is disconnected", socketId)
+	}
+
+	if err := socketObj.connection.SetWriteDeadline(time.Now().Add(time.Second * 10)); err != nil {
+		socketCleanup(socketId, socketObj)
+		return fmt.Errorf("error setting timeout to socket %v, err: %v", socketId, err)
+	}
+
+	if err := socketObj.connection.WriteMessage(websocket.TextMessage, message); err != nil {
+		socketCleanup(socketId, socketObj)
+		return fmt.Errorf("failed to write message to socket %v, err: %v", socketId, err)
+	}
+
+	return nil
+}
+
+func socketCleanup(socketId int, socketConnection *SocketConnection) {
+	err := socketConnection.connection.Close()
+	if err != nil {
+		logger.Log.Errorf("error closing socket connection for socket id %d: %v", socketId, err)
+	}
+
+	websocketIdsLock.Lock()
+	connectedWebsockets[socketId] = nil
+	websocketIdsLock.Unlock()
+
+	socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
+}

agent/pkg/api/socket_server_handlers.go

@@ -0,0 +1,158 @@
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
+
+	tapApi "github.com/up9inc/mizu/tap/api"
+
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+)
+
+type BrowserClient struct {
+	dataStreamCancelFunc context.CancelFunc
+}
+
+var browserClients = make(map[int]*BrowserClient, 0)
+var tapperClientSocketUUIDs = make([]int, 0)
+var socketListLock = sync.Mutex{}
+
+type RoutesEventHandlers struct {
+	EventHandlers
+	SocketOutChannel chan<- *tapApi.OutputChannelItem
+}
+
+func (h *RoutesEventHandlers) WebSocketConnect(_ *gin.Context, socketId int, isTapper bool) {
+	if isTapper {
+		logger.Log.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
+		tappers.Connected()
+
+		socketListLock.Lock()
+		tapperClientSocketUUIDs = append(tapperClientSocketUUIDs, socketId)
+		socketListLock.Unlock()
+
+		nodeToTappedPodMap := tappedPods.GetNodeToTappedPodMap()
+		SendTappedPods(socketId, nodeToTappedPodMap)
+	} else {
+		logger.Log.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
+
+		socketListLock.Lock()
+		browserClients[socketId] = &BrowserClient{}
+		socketListLock.Unlock()
+
+		BroadcastTappedPodsStatus()
+	}
+}
+
+func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
+	if isTapper {
+		logger.Log.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
+		tappers.Disconnected()
+
+		socketListLock.Lock()
+		removeSocketUUIDFromTapperSlice(socketId)
+		socketListLock.Unlock()
+	} else {
+		logger.Log.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
+		socketListLock.Lock()
+		if browserClients[socketId] != nil && browserClients[socketId].dataStreamCancelFunc != nil {
+			browserClients[socketId].dataStreamCancelFunc()
+		}
+		delete(browserClients, socketId)
+		socketListLock.Unlock()
+	}
+}
+
+func BroadcastToBrowserClients(message []byte) {
+	for socketId := range browserClients {
+		go func(socketId int) {
+			if err := SendToSocket(socketId, message); err != nil {
+				logger.Log.Error(err)
+			}
+		}(socketId)
+	}
+}
+
+func BroadcastToTapperClients(message []byte) {
+	for _, socketId := range tapperClientSocketUUIDs {
+		go func(socketId int) {
+			if err := SendToSocket(socketId, message); err != nil {
+				logger.Log.Error(err)
+			}
+		}(socketId)
+	}
+}
+
+func (h *RoutesEventHandlers) WebSocketMessage(socketId int, isTapper bool, message []byte) {
+	if isTapper {
+		HandleTapperIncomingMessage(message, h.SocketOutChannel, BroadcastToBrowserClients)
+	} else {
+		// we initiate the basenine stream after the first websocket message we receive (it contains the entry query), we then store a cancelfunc to later cancel this stream
+		if browserClients[socketId] != nil && browserClients[socketId].dataStreamCancelFunc == nil {
+			var params WebSocketParams
+			if err := json.Unmarshal(message, &params); err != nil {
+				logger.Log.Errorf("Error: %v", socketId, err)
+				return
+			}
+
+			entriesStreamer := dependency.GetInstance(dependency.EntriesSocketStreamer).(EntryStreamer)
+			ctx, cancelFunc := context.WithCancel(context.Background())
+			err := entriesStreamer.Get(ctx, socketId, &params)
+
+			if err != nil {
+				logger.Log.Errorf("error initializing basenine stream for browser socket %d %+v", socketId, err)
+				cancelFunc()
+			} else {
+				browserClients[socketId].dataStreamCancelFunc = cancelFunc
+			}
+		}
+	}
+}
+
+func HandleTapperIncomingMessage(message []byte, socketOutChannel chan<- *tapApi.OutputChannelItem, broadcastMessageFunc func([]byte)) {
+	var socketMessageBase shared.WebSocketMessageMetadata
+	err := json.Unmarshal(message, &socketMessageBase)
+	if err != nil {
+		logger.Log.Infof("Could not unmarshal websocket message %v", err)
+	} else {
+		switch socketMessageBase.MessageType {
+		case shared.WebSocketMessageTypeTappedEntry:
+			var tappedEntryMessage models.WebSocketTappedEntryMessage
+			err := json.Unmarshal(message, &tappedEntryMessage)
+			if err != nil {
+				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
+			} else {
+				// NOTE: This is where the message comes back from the intermediate WebSocket to code.
+				socketOutChannel <- tappedEntryMessage.Data
+			}
+		case shared.WebSocketMessageTypeUpdateStatus:
+			var statusMessage shared.WebSocketStatusMessage
+			err := json.Unmarshal(message, &statusMessage)
+			if err != nil {
+				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
+			} else {
+				broadcastMessageFunc(message)
+			}
+		default:
+			logger.Log.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
+		}
+	}
+}
+
+func removeSocketUUIDFromTapperSlice(uuidToRemove int) {
+	newUUIDSlice := make([]int, 0, len(tapperClientSocketUUIDs))
+	for _, uuid := range tapperClientSocketUUIDs {
+		if uuid != uuidToRemove {
+			newUUIDSlice = append(newUUIDSlice, uuid)
+		}
+	}
+	tapperClientSocketUUIDs = newUUIDSlice
+}

agent/pkg/api/utils.go

@@ -0,0 +1,40 @@
+package api
+
+import (
+	"encoding/json"
+
+	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+)
+
+func BroadcastTappedPodsStatus() {
+	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
+
+	message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
+	if jsonBytes, err := json.Marshal(message); err != nil {
+		logger.Log.Errorf("Could not Marshal message %v", err)
+	} else {
+		BroadcastToBrowserClients(jsonBytes)
+	}
+}
+
+func SendTappedPods(socketId int, nodeToTappedPodMap shared.NodeToPodsMap) {
+	message := shared.CreateWebSocketTappedPodsMessage(nodeToTappedPodMap)
+	if jsonBytes, err := json.Marshal(message); err != nil {
+		logger.Log.Errorf("Could not Marshal message %v", err)
+	} else {
+		if err := SendToSocket(socketId, jsonBytes); err != nil {
+			logger.Log.Error(err)
+		}
+	}
+}
+
+func BroadcastTappedPodsToTappers(nodeToTappedPodMap shared.NodeToPodsMap) {
+	message := shared.CreateWebSocketTappedPodsMessage(nodeToTappedPodMap)
+	if jsonBytes, err := json.Marshal(message); err != nil {
+		logger.Log.Errorf("Could not Marshal message %v", err)
+	} else {
+		BroadcastToTapperClients(jsonBytes)
+	}
+}

agent/pkg/app/main.go

@@ -0,0 +1,138 @@
+package app
+
+import (
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/antelman107/net-wait-go/wait"
+	"github.com/op/go-logging"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/agent/pkg/api"
+	"github.com/up9inc/mizu/agent/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+	"github.com/up9inc/mizu/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
+	"github.com/up9inc/mizu/tap/dbgctl"
+	amqpExt "github.com/up9inc/mizu/tap/extensions/amqp"
+	httpExt "github.com/up9inc/mizu/tap/extensions/http"
+	kafkaExt "github.com/up9inc/mizu/tap/extensions/kafka"
+	redisExt "github.com/up9inc/mizu/tap/extensions/redis"
+)
+
+var (
+	Extensions    []*tapApi.Extension          // global
+	ExtensionsMap map[string]*tapApi.Extension // global
+	ProtocolsMap  map[string]*tapApi.Protocol  //global
+)
+
+func LoadExtensions() {
+	Extensions = make([]*tapApi.Extension, 0)
+	ExtensionsMap = make(map[string]*tapApi.Extension)
+	ProtocolsMap = make(map[string]*tapApi.Protocol)
+
+	extensionHttp := &tapApi.Extension{}
+	dissectorHttp := httpExt.NewDissector()
+	dissectorHttp.Register(extensionHttp)
+	extensionHttp.Dissector = dissectorHttp
+	Extensions = append(Extensions, extensionHttp)
+	ExtensionsMap[extensionHttp.Protocol.Name] = extensionHttp
+	protocolsHttp := dissectorHttp.GetProtocols()
+	for k, v := range protocolsHttp {
+		ProtocolsMap[k] = v
+	}
+
+	if !dbgctl.MizuTapperDisableNonHttpExtensions {
+		extensionAmqp := &tapApi.Extension{}
+		dissectorAmqp := amqpExt.NewDissector()
+		dissectorAmqp.Register(extensionAmqp)
+		extensionAmqp.Dissector = dissectorAmqp
+		Extensions = append(Extensions, extensionAmqp)
+		ExtensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
+		protocolsAmqp := dissectorAmqp.GetProtocols()
+		for k, v := range protocolsAmqp {
+			ProtocolsMap[k] = v
+		}
+
+		extensionKafka := &tapApi.Extension{}
+		dissectorKafka := kafkaExt.NewDissector()
+		dissectorKafka.Register(extensionKafka)
+		extensionKafka.Dissector = dissectorKafka
+		Extensions = append(Extensions, extensionKafka)
+		ExtensionsMap[extensionKafka.Protocol.Name] = extensionKafka
+		protocolsKafka := dissectorKafka.GetProtocols()
+		for k, v := range protocolsKafka {
+			ProtocolsMap[k] = v
+		}
+
+		extensionRedis := &tapApi.Extension{}
+		dissectorRedis := redisExt.NewDissector()
+		dissectorRedis.Register(extensionRedis)
+		extensionRedis.Dissector = dissectorRedis
+		Extensions = append(Extensions, extensionRedis)
+		ExtensionsMap[extensionRedis.Protocol.Name] = extensionRedis
+		protocolsRedis := dissectorRedis.GetProtocols()
+		for k, v := range protocolsRedis {
+			ProtocolsMap[k] = v
+		}
+	}
+
+	sort.Slice(Extensions, func(i, j int) bool {
+		return Extensions[i].Protocol.Priority < Extensions[j].Protocol.Priority
+	})
+
+	api.InitMaps(ExtensionsMap, ProtocolsMap)
+	providers.InitProtocolToColor(ProtocolsMap)
+}
+
+func ConfigureBasenineServer(host string, port string, dbSize int64, logLevel logging.Level, insertionFilter string) {
+	if !wait.New(
+		wait.WithProto("tcp"),
+		wait.WithWait(200*time.Millisecond),
+		wait.WithBreak(50*time.Millisecond),
+		wait.WithDeadline(20*time.Second),
+		wait.WithDebug(logLevel == logging.DEBUG),
+	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
+		logger.Log.Panicf("Basenine is not available!")
+	}
+
+	if err := basenine.Limit(host, port, dbSize); err != nil {
+		logger.Log.Panicf("Error while limiting database size: %v", err)
+	}
+
+	// Define the macros
+	for _, extension := range Extensions {
+		macros := extension.Dissector.Macros()
+		for macro, expanded := range macros {
+			if err := basenine.Macro(host, port, macro, expanded); err != nil {
+				logger.Log.Panicf("Error while adding a macro: %v", err)
+			}
+		}
+	}
+
+	// Set the insertion filter that comes from the config
+	if err := basenine.InsertionFilter(host, port, insertionFilter); err != nil {
+		logger.Log.Errorf("Error while setting the insertion filter: %v", err)
+	}
+
+	utils.StartTime = time.Now().UnixNano() / int64(time.Millisecond)
+}
+
+func GetEntryInputChannel() chan *tapApi.OutputChannelItem {
+	outputItemsChannel := make(chan *tapApi.OutputChannelItem)
+	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+	go FilterItems(outputItemsChannel, filteredOutputItemsChannel)
+	go api.StartReadingEntries(filteredOutputItemsChannel, nil, ExtensionsMap)
+
+	return outputItemsChannel
+}
+
+func FilterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
+	for message := range inChannel {
+		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
+			continue
+		}
+
+		outChannel <- message
+	}
+}

agent/pkg/config/config.go

@@ -0,0 +1,53 @@
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/up9inc/mizu/shared"
+	"io/ioutil"
+	"os"
+)
+
+// these values are used when the config.json file is not present
+const (
+	defaultMaxDatabaseSizeBytes int64  = 200 * 1000 * 1000
+	DefaultDatabasePath         string = "./entries"
+)
+
+var Config *shared.MizuAgentConfig
+
+func LoadConfig() error {
+	if Config != nil {
+		return nil
+	}
+	filePath := fmt.Sprintf("%s%s", shared.ConfigDirPath, shared.ConfigFileName)
+
+	content, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return applyDefaultConfig()
+		}
+		return err
+	}
+
+	if err = json.Unmarshal(content, &Config); err != nil {
+		return err
+	}
+	return nil
+}
+
+func applyDefaultConfig() error {
+	defaultConfig, err := getDefaultConfig()
+	if err != nil {
+		return err
+	}
+	Config = defaultConfig
+	return nil
+}
+
+func getDefaultConfig() (*shared.MizuAgentConfig, error) {
+	return &shared.MizuAgentConfig{
+		MaxDBSizeBytes:    defaultMaxDatabaseSizeBytes,
+		AgentDatabasePath: DefaultDatabasePath,
+	}, nil
+}

agent/pkg/controllers/db_controller.go

@@ -0,0 +1,28 @@
+package controllers
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/agent/pkg/app"
+	"github.com/up9inc/mizu/agent/pkg/config"
+	"github.com/up9inc/mizu/shared"
+)
+
+func Flush(c *gin.Context) {
+	if err := basenine.Flush(shared.BasenineHost, shared.BaseninePort); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	} else {
+		c.JSON(http.StatusOK, "Flushed.")
+	}
+}
+
+func Reset(c *gin.Context) {
+	if err := basenine.Reset(shared.BasenineHost, shared.BaseninePort); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	} else {
+		app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel, config.Config.InsertionFilter)
+		c.JSON(http.StatusOK, "Resetted.")
+	}
+}

agent/pkg/controllers/entries_controller.go

@@ -0,0 +1,79 @@
+package controllers
+
+import (
+	"net/http"
+
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/entries"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/validation"
+
+	"github.com/gin-gonic/gin"
+
+	"github.com/up9inc/mizu/logger"
+)
+
+func HandleEntriesError(c *gin.Context, err error) bool {
+	if err != nil {
+		logger.Log.Errorf("Error getting entry: %v", err)
+		_ = c.Error(err)
+		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       err.Error(),
+		})
+		return true // signal that there was an error and the caller should return
+	}
+	return false // no error, can continue
+}
+
+func GetEntries(c *gin.Context) {
+	entriesRequest := &models.EntriesRequest{}
+
+	if err := c.BindQuery(entriesRequest); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+	validationError := validation.Validate(entriesRequest)
+	if validationError != nil {
+		c.JSON(http.StatusBadRequest, validationError)
+	}
+
+	if entriesRequest.TimeoutMs == 0 {
+		entriesRequest.TimeoutMs = 3000
+	}
+
+	entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
+	entryWrappers, metadata, err := entriesProvider.GetEntries(entriesRequest)
+	if !HandleEntriesError(c, err) {
+		baseEntries := make([]interface{}, 0)
+		for _, entry := range entryWrappers {
+			baseEntries = append(baseEntries, entry.Base)
+		}
+		c.JSON(http.StatusOK, models.EntriesResponse{
+			Data: baseEntries,
+			Meta: metadata,
+		})
+	}
+}
+
+func GetEntry(c *gin.Context) {
+	singleEntryRequest := &models.SingleEntryRequest{}
+
+	if err := c.BindQuery(singleEntryRequest); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	}
+	validationError := validation.Validate(singleEntryRequest)
+	if validationError != nil {
+		c.JSON(http.StatusBadRequest, validationError)
+	}
+
+	id := c.Param("id")
+
+	entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
+	entry, err := entriesProvider.GetEntry(singleEntryRequest, id)
+
+	if !HandleEntriesError(c, err) {
+		c.JSON(http.StatusOK, entry)
+	}
+}

agent/pkg/controllers/metadata_controller.go

@@ -0,0 +1,14 @@
+package controllers
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/version"
+	"github.com/up9inc/mizu/shared"
+)
+
+func GetVersion(c *gin.Context) {
+	resp := shared.VersionResponse{Ver: version.Ver}
+	c.JSON(http.StatusOK, resp)
+}

agent/pkg/controllers/oas_controller.go

@@ -0,0 +1,68 @@
+package controllers
+
+import (
+	"net/http"
+
+	"github.com/chanced/openapi"
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/oas"
+	"github.com/up9inc/mizu/logger"
+)
+
+func GetOASServers(c *gin.Context) {
+	m := make([]string, 0)
+	oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
+	oasGenerator.GetServiceSpecs().Range(func(key, value interface{}) bool {
+		m = append(m, key.(string))
+		return true
+	})
+
+	c.JSON(http.StatusOK, m)
+}
+
+func GetOASSpec(c *gin.Context) {
+	oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
+	res, ok := oasGenerator.GetServiceSpecs().Load(c.Param("id"))
+	if !ok {
+		c.JSON(http.StatusNotFound, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       "Service not found among specs",
+		})
+		return // exit
+	}
+
+	gen := res.(*oas.SpecGen)
+	spec, err := gen.GetSpec()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{
+			"error":     true,
+			"type":      "error",
+			"autoClose": "5000",
+			"msg":       err,
+		})
+		return // exit
+	}
+
+	c.JSON(http.StatusOK, spec)
+}
+
+func GetOASAllSpecs(c *gin.Context) {
+	res := map[string]*openapi.OpenAPI{}
+
+	oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
+	oasGenerator.GetServiceSpecs().Range(func(key, value interface{}) bool {
+		svc := key.(string)
+		gen := value.(*oas.SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			logger.Log.Warningf("Failed to obtain spec for service %s: %s", svc, err)
+			return true
+		}
+		res[svc] = spec
+		return true
+	})
+	c.JSON(http.StatusOK, res)
+}

agent/pkg/controllers/oas_controller_test.go

@@ -0,0 +1,45 @@
+package controllers
+
+import (
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/oas"
+)
+
+func TestGetOASServers(t *testing.T) {
+	recorder, c := getRecorderAndContext()
+
+	GetOASServers(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+}
+
+func TestGetOASAllSpecs(t *testing.T) {
+	recorder, c := getRecorderAndContext()
+
+	GetOASAllSpecs(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+}
+
+func TestGetOASSpec(t *testing.T) {
+	recorder, c := getRecorderAndContext()
+
+	c.Params = []gin.Param{{Key: "id", Value: "some"}}
+
+	GetOASSpec(c)
+	t.Logf("Written body: %s", recorder.Body.String())
+}
+
+func getRecorderAndContext() (*httptest.ResponseRecorder, *gin.Context) {
+	dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} {
+		return oas.GetDefaultOasGeneratorInstance(-1)
+	})
+
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetDefaultOasGeneratorInstance(-1).Start()
+	oas.GetDefaultOasGeneratorInstance(-1).GetServiceSpecs().Store("some", oas.NewGen("some"))
+	return recorder, c
+}

agent/pkg/controllers/query_controller.go

@@ -0,0 +1,31 @@
+package controllers
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/shared"
+)
+
+type ValidateResponse struct {
+	Valid   bool   `json:"valid"`
+	Message string `json:"message"`
+}
+
+func PostValidate(c *gin.Context) {
+	query := c.PostForm("query")
+	valid := true
+	message := ""
+
+	err := basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
+	if err != nil {
+		valid = false
+		message = err.Error()
+	}
+
+	c.JSON(http.StatusOK, ValidateResponse{
+		Valid:   valid,
+		Message: message,
+	})
+}

agent/pkg/controllers/replay_controller.go

@@ -0,0 +1,34 @@
+package controllers
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/replay"
+	"github.com/up9inc/mizu/agent/pkg/validation"
+	"github.com/up9inc/mizu/logger"
+)
+
+const (
+	replayTimeout = 10 * time.Second
+)
+
+func ReplayRequest(c *gin.Context) {
+	logger.Log.Debug("Starting replay")
+	replayDetails := &replay.Details{}
+	if err := c.Bind(replayDetails); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+
+	logger.Log.Debugf("Validating replay, %v", replayDetails)
+	if err := validation.Validate(replayDetails); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+
+	logger.Log.Debug("Executing replay, %v", replayDetails)
+	result := replay.ExecuteRequest(replayDetails, replayTimeout)
+	c.JSON(http.StatusOK, result)
+}

agent/pkg/controllers/service_map_controller.go

@@ -0,0 +1,39 @@
+package controllers
+
+import (
+	"net/http"
+
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
+
+	"github.com/gin-gonic/gin"
+)
+
+type ServiceMapController struct {
+	service servicemap.ServiceMap
+}
+
+func NewServiceMapController() *ServiceMapController {
+	serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap)
+	return &ServiceMapController{
+		service: serviceMapGenerator,
+	}
+}
+
+func (s *ServiceMapController) Status(c *gin.Context) {
+	c.JSON(http.StatusOK, s.service.GetStatus())
+}
+
+func (s *ServiceMapController) Get(c *gin.Context) {
+	response := &servicemap.ServiceMapResponse{
+		Status: s.service.GetStatus(),
+		Nodes:  s.service.GetNodes(),
+		Edges:  s.service.GetEdges(),
+	}
+	c.JSON(http.StatusOK, response)
+}
+
+func (s *ServiceMapController) Reset(c *gin.Context) {
+	s.service.Reset()
+	s.Status(c)
+}

agent/pkg/controllers/service_map_controller_test.go

@@ -0,0 +1,151 @@
+package controllers
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
+
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/suite"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+const (
+	a    = "aService"
+	b    = "bService"
+	Ip   = "127.0.0.1"
+	Port = "80"
+)
+
+var (
+	TCPEntryA = &tapApi.TCP{
+		Name: a,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, a),
+	}
+	TCPEntryB = &tapApi.TCP{
+		Name: b,
+		Port: Port,
+		IP:   fmt.Sprintf("%s.%s", Ip, b),
+	}
+)
+
+var ProtocolHttp = &tapApi.Protocol{
+	ProtocolSummary: tapApi.ProtocolSummary{
+		Name:         "http",
+		Version:      "1.1",
+		Abbreviation: "HTTP",
+	},
+	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
+	Macro:           "http",
+	BackgroundColor: "#205cf5",
+	ForegroundColor: "#ffffff",
+	FontSize:        12,
+	ReferenceLink:   "https://datatracker.ietf.org/doc/html/rfc2616",
+	Ports:           []string{"80", "443", "8080"},
+	Priority:        0,
+}
+
+type ServiceMapControllerSuite struct {
+	suite.Suite
+
+	c *ServiceMapController
+	w *httptest.ResponseRecorder
+	g *gin.Context
+}
+
+func (s *ServiceMapControllerSuite) SetupTest() {
+	dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
+
+	s.c = NewServiceMapController()
+	s.c.service.Enable()
+	s.c.service.(servicemap.ServiceMapSink).NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
+
+	s.w = httptest.NewRecorder()
+	s.g, _ = gin.CreateTestContext(s.w)
+}
+
+func (s *ServiceMapControllerSuite) TestGetStatus() {
+	assert := s.Assert()
+
+	s.c.Status(s.g)
+	assert.Equal(http.StatusOK, s.w.Code)
+
+	var status servicemap.ServiceMapStatus
+	err := json.Unmarshal(s.w.Body.Bytes(), &status)
+	assert.NoError(err)
+	assert.Equal("enabled", status.Status)
+	assert.Equal(1, status.EntriesProcessedCount)
+	assert.Equal(2, status.NodeCount)
+	assert.Equal(1, status.EdgeCount)
+}
+
+func (s *ServiceMapControllerSuite) TestGet() {
+	assert := s.Assert()
+
+	s.c.Get(s.g)
+	assert.Equal(http.StatusOK, s.w.Code)
+
+	var response servicemap.ServiceMapResponse
+	err := json.Unmarshal(s.w.Body.Bytes(), &response)
+	assert.NoError(err)
+
+	// response status
+	assert.Equal("enabled", response.Status.Status)
+	assert.Equal(1, response.Status.EntriesProcessedCount)
+	assert.Equal(2, response.Status.NodeCount)
+	assert.Equal(1, response.Status.EdgeCount)
+
+	// response nodes
+	aNode := servicemap.ServiceMapNode{
+		Id:       1,
+		Name:     TCPEntryA.Name,
+		Entry:    TCPEntryA,
+		Resolved: true,
+		Count:    1,
+	}
+	bNode := servicemap.ServiceMapNode{
+		Id:       2,
+		Name:     TCPEntryB.Name,
+		Entry:    TCPEntryB,
+		Resolved: true,
+		Count:    1,
+	}
+	assert.Contains(response.Nodes, aNode)
+	assert.Contains(response.Nodes, bNode)
+	assert.Len(response.Nodes, 2)
+
+	// response edges
+	assert.Equal([]servicemap.ServiceMapEdge{
+		{
+			Source:      aNode,
+			Destination: bNode,
+			Protocol:    ProtocolHttp,
+			Count:       1,
+		},
+	}, response.Edges)
+}
+
+func (s *ServiceMapControllerSuite) TestGetReset() {
+	assert := s.Assert()
+
+	s.c.Reset(s.g)
+	assert.Equal(http.StatusOK, s.w.Code)
+
+	var status servicemap.ServiceMapStatus
+	err := json.Unmarshal(s.w.Body.Bytes(), &status)
+	assert.NoError(err)
+	assert.Equal("enabled", status.Status)
+	assert.Equal(0, status.EntriesProcessedCount)
+	assert.Equal(0, status.NodeCount)
+	assert.Equal(0, status.EdgeCount)
+}
+
+func TestServiceMapControllerSuite(t *testing.T) {
+	suite.Run(t, new(ServiceMapControllerSuite))
+}

agent/pkg/controllers/status_controller.go

@@ -0,0 +1,88 @@
+package controllers
+
+import (
+	"net/http"
+
+	core "k8s.io/api/core/v1"
+
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/api"
+	"github.com/up9inc/mizu/agent/pkg/holder"
+	"github.com/up9inc/mizu/agent/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
+	"github.com/up9inc/mizu/agent/pkg/validation"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/kubernetes"
+)
+
+func HealthCheck(c *gin.Context) {
+	tappersStatus := make([]*shared.TapperStatus, 0)
+	for _, value := range tappers.GetStatus() {
+		tappersStatus = append(tappersStatus, value)
+	}
+
+	response := shared.HealthResponse{
+		TappedPods:            tappedPods.Get(),
+		ConnectedTappersCount: tappers.GetConnectedCount(),
+		TappersStatus:         tappersStatus,
+	}
+	c.JSON(http.StatusOK, response)
+}
+
+func PostTappedPods(c *gin.Context) {
+	var requestTappedPods []core.Pod
+	if err := c.Bind(&requestTappedPods); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+
+	podInfos := kubernetes.GetPodInfosForPods(requestTappedPods)
+
+	logger.Log.Infof("[Status] POST request: %d tapped pods", len(requestTappedPods))
+	tappedPods.Set(podInfos)
+	api.BroadcastTappedPodsStatus()
+
+	nodeToTappedPodMap := kubernetes.GetNodeHostToTappedPodsMap(requestTappedPods)
+	tappedPods.SetNodeToTappedPodMap(nodeToTappedPodMap)
+	api.BroadcastTappedPodsToTappers(nodeToTappedPodMap)
+}
+
+func PostTapperStatus(c *gin.Context) {
+	tapperStatus := &shared.TapperStatus{}
+	if err := c.Bind(tapperStatus); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+
+	if err := validation.Validate(tapperStatus); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+
+	logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus)
+	tappers.SetStatus(tapperStatus)
+	api.BroadcastTappedPodsStatus()
+}
+
+func GetConnectedTappersCount(c *gin.Context) {
+	c.JSON(http.StatusOK, tappers.GetConnectedCount())
+}
+
+func GetTappingStatus(c *gin.Context) {
+	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
+	c.JSON(http.StatusOK, tappedPodsStatus)
+}
+
+func GetGeneralStats(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.GetGeneralStats())
+}
+
+func GetTrafficStats(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.GetTrafficStats())
+}
+
+func GetCurrentResolvingInformation(c *gin.Context) {
+	c.JSON(http.StatusOK, holder.GetResolver().GetMap())
+}

agent/pkg/dependency/container.go

@@ -0,0 +1,11 @@
+package dependency
+
+var typeInitializerMap = make(map[ContainerType]func() interface{}, 0)
+
+func RegisterGenerator(name ContainerType, fn func() interface{}) {
+	typeInitializerMap[name] = fn
+}
+
+func GetInstance(name ContainerType) interface{} {
+	return typeInitializerMap[name]()
+}

agent/pkg/dependency/type_names.go

@@ -0,0 +1,12 @@
+package dependency
+
+type ContainerType string
+
+const (
+	ServiceMapGeneratorDependency ContainerType = "ServiceMapGeneratorDependency"
+	OasGeneratorDependency        ContainerType = "OasGeneratorDependency"
+	EntriesInserter               ContainerType = "EntriesInserter"
+	EntriesProvider               ContainerType = "EntriesProvider"
+	EntriesSocketStreamer         ContainerType = "EntriesSocketStreamer"
+	EntryStreamerSocketConnector  ContainerType = "EntryStreamerSocketConnector"
+)

agent/pkg/entries/entries_provider.go

@@ -0,0 +1,103 @@
+package entries
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/agent/pkg/app"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+type EntriesProvider interface {
+	GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error)
+	GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error)
+}
+
+type BasenineEntriesProvider struct{}
+
+func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error) {
+	data, _, lastMeta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
+		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
+		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var dataSlice []*tapApi.EntryWrapper
+
+	for _, row := range data {
+		var entry *tapApi.Entry
+		err = json.Unmarshal(row, &entry)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		protocol, ok := app.ProtocolsMap[entry.Protocol.ToString()]
+		if !ok {
+			return nil, nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
+		}
+
+		extension, ok := app.ExtensionsMap[protocol.Name]
+		if !ok {
+			return nil, nil, fmt.Errorf("extension not found, extension: %v", protocol.Name)
+		}
+
+		base := extension.Dissector.Summarize(entry)
+
+		dataSlice = append(dataSlice, &tapApi.EntryWrapper{
+			Protocol: *protocol,
+			Data:     entry,
+			Base:     base,
+		})
+	}
+
+	var metadata *basenine.Metadata
+	err = json.Unmarshal(lastMeta, &metadata)
+	if err != nil {
+		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
+	}
+
+	return dataSlice, metadata, nil
+}
+
+func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error) {
+	var entry *tapApi.Entry
+	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, entryId, singleEntryRequest.Query)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(bytes, &entry)
+	if err != nil {
+		return nil, errors.New(string(bytes))
+	}
+
+	protocol, ok := app.ProtocolsMap[entry.Protocol.ToString()]
+	if !ok {
+		return nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
+	}
+
+	extension, ok := app.ExtensionsMap[protocol.Name]
+	if !ok {
+		return nil, fmt.Errorf("extension not found, extension: %v", protocol.Name)
+	}
+
+	base := extension.Dissector.Summarize(entry)
+	var representation []byte
+	representation, err = extension.Dissector.Represent(entry.Request, entry.Response)
+	if err != nil {
+		return nil, err
+	}
+
+	return &tapApi.EntryWrapper{
+		Protocol:       *protocol,
+		Representation: string(representation),
+		Data:           entry,
+		Base:           base,
+	}, nil
+}

agent/pkg/har/types.go

@@ -0,0 +1,376 @@
+package har
+
+import (
+	"encoding/base64"
+	"time"
+	"unicode/utf8"
+
+	"github.com/up9inc/mizu/logger"
+)
+
+/*
+HTTP Archive (HAR) format
+https://w3c.github.io/web-performance/specs/HAR/Overview.html
+*/
+
+// HAR is a container type for deserialization
+type HAR struct {
+	Log Log `json:"log"`
+}
+
+// Log represents the root of the exported data. This object MUST be present and its name MUST be "log".
+type Log struct {
+	// The object contains the following name/value pairs:
+
+	// Required. Version number of the format.
+	Version string `json:"version"`
+	// Required. An object of type creator that contains the name and version
+	// information of the log creator application.
+	Creator Creator `json:"creator"`
+	// Optional. An object of type browser that contains the name and version
+	// information of the user agent.
+	Browser Browser `json:"browser"`
+	// Optional. An array of objects of type page, each representing one exported
+	// (tracked) page. Leave out this field if the application does not support
+	// grouping by pages.
+	Pages []Page `json:"pages,omitempty"`
+	// Required. An array of objects of type entry, each representing one
+	// exported (tracked) HTTP request.
+	Entries []Entry `json:"entries"`
+	// Optional. A comment provided by the user or the application. Sorting
+	// entries by startedDateTime (starting from the oldest) is preferred way how
+	// to export data since it can make importing faster. However the reader
+	// application should always make sure the array is sorted (if required for
+	// the import).
+	Comment string `json:"comment"`
+}
+
+// Creator contains information about the log creator application
+type Creator struct {
+	// Required. The name of the application that created the log.
+	Name string `json:"name"`
+	// Required. The version number of the application that created the log.
+	Version string `json:"version"`
+	// Optional. A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Browser that created the log
+type Browser struct {
+	// Required. The name of the browser that created the log.
+	Name string `json:"name"`
+	// Required. The version number of the browser that created the log.
+	Version string `json:"version"`
+	// Optional. A comment provided by the user or the browser.
+	Comment string `json:"comment"`
+}
+
+// Page object for every exported web page and one <entry> object for every HTTP request.
+// In case when an HTTP trace tool isn't able to group requests by a page,
+// the <pages> object is empty and individual requests doesn't have a parent page.
+type Page struct {
+	/* There is one <page> object for every exported web page and one <entry>
+	object for every HTTP request. In case when an HTTP trace tool isn't able to
+	group requests by a page, the <pages> object is empty and individual
+	requests doesn't have a parent page.
+	*/
+
+	// Date and time stamp for the beginning of the page load
+	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00).
+	StartedDateTime string `json:"startedDateTime"`
+	// Unique identifier of a page within the . Entries use it to refer the parent page.
+	ID string `json:"id"`
+	// Page title.
+	Title string `json:"title"`
+	// Detailed timing info about page load.
+	PageTiming PageTiming `json:"pageTiming"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// PageTiming describes timings for various events (states) fired during the page load.
+// All times are specified in milliseconds. If a time info is not available appropriate field is set to -1.
+type PageTiming struct {
+	// Content of the page loaded. Number of milliseconds since page load started
+	// (page.startedDateTime). Use -1 if the timing does not apply to the current
+	// request.
+	// Depeding on the browser, onContentLoad property represents DOMContentLoad
+	// event or document.readyState == interactive.
+	OnContentLoad int `json:"onContentLoad"`
+	// Page is loaded (onLoad event fired). Number of milliseconds since page
+	// load started (page.startedDateTime). Use -1 if the timing does not apply
+	// to the current request.
+	OnLoad int `json:"onLoad"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment"`
+}
+
+// Entry is a unique, optional Reference to the parent page.
+// Leave out this field if the application does not support grouping by pages.
+type Entry struct {
+	Pageref string `json:"pageref,omitempty"`
+	// Date and time stamp of the request start
+	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD).
+	StartedDateTime string `json:"startedDateTime"`
+	// Total elapsed time of the request in milliseconds. This is the sum of all
+	// timings available in the timings object (i.e. not including -1 values) .
+	Time int `json:"time"`
+	// Detailed info about the request.
+	Request Request `json:"request"`
+	// Detailed info about the response.
+	Response Response `json:"response"`
+	// Info about cache usage.
+	Cache Cache `json:"cache"`
+	// Detailed timing info about request/response round trip.
+	PageTimings PageTimings `json:"pageTimings"`
+	// optional (new in 1.2) IP address of the server that was connected
+	// (result of DNS resolution).
+	ServerIPAddress string `json:"serverIPAddress,omitempty"`
+	// optional (new in 1.2) Unique ID of the parent TCP/IP connection, can be
+	// the client port number. Note that a port number doesn't have to be unique
+	// identifier in cases where the port is shared for more connections. If the
+	// port isn't available for the application, any other unique connection ID
+	// can be used instead (e.g. connection index). Leave out this field if the
+	// application doesn't support this info.
+	Connection string `json:"connection,omitempty"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Request contains detailed info about performed request.
+type Request struct {
+	// Request method (GET, POST, ...).
+	Method string `json:"method"`
+	// Absolute URL of the request (fragments are not included).
+	URL string `json:"url"`
+	// Request HTTP Version.
+	HTTPVersion string `json:"httpVersion"`
+	// List of cookie objects.
+	Cookies []Cookie `json:"cookies"`
+	// List of header objects.
+	Headers []NVP `json:"headers"`
+	// List of query parameter objects.
+	QueryString []NVP `json:"queryString"`
+	// Posted data.
+	PostData PostData `json:"postData"`
+	// Total number of bytes from the start of the HTTP request message until
+	// (and including) the double CRLF before the body. Set to -1 if the info
+	// is not available.
+	HeaderSize int `json:"headerSize"`
+	// Size of the request body (POST data payload) in bytes. Set to -1 if the
+	// info is not available.
+	BodySize int `json:"bodySize"`
+	// (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment"`
+}
+
+// Response contains detailed info about the response.
+type Response struct {
+	// Response status.
+	Status int `json:"status"`
+	// Response status description.
+	StatusText string `json:"statusText"`
+	// Response HTTP Version.
+	HTTPVersion string `json:"httpVersion"`
+	// List of cookie objects.
+	Cookies []Cookie `json:"cookies"`
+	// List of header objects.
+	Headers []NVP `json:"headers"`
+	// Details about the response body.
+	Content Content `json:"content"`
+	// Redirection target URL from the Location response header.
+	RedirectURL string `json:"redirectURL"`
+	// Total number of bytes from the start of the HTTP response message until
+	// (and including) the double CRLF before the body. Set to -1 if the info is
+	// not available.
+	// The size of received response-headers is computed only from headers that
+	// are really received from the server. Additional headers appended by the
+	// browser are not included in this number, but they appear in the list of
+	// header objects.
+	HeadersSize int `json:"headersSize"`
+	// Size of the received response body in bytes. Set to zero in case of
+	// responses coming from the cache (304). Set to -1 if the info is not
+	// available.
+	BodySize int `json:"bodySize"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Cookie contains list of all cookies (used in <request> and <response> objects).
+type Cookie struct {
+	// The name of the cookie.
+	Name string `json:"name"`
+	// The cookie value.
+	Value string `json:"value"`
+	// optional The path pertaining to the cookie.
+	Path string `json:"path,omitempty"`
+	// optional The host of the cookie.
+	Domain string `json:"domain,omitempty"`
+	// optional Cookie expiration time.
+	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.123+02:00).
+	Expires string `json:"expires,omitempty"`
+	// optional Set to true if the cookie is HTTP only, false otherwise.
+	HTTPOnly bool `json:"httpOnly,omitempty"`
+	// optional (new in 1.2) True if the cookie was transmitted over ssl, false
+	// otherwise.
+	Secure bool `json:"secure,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// NVP is simply a name/value pair with a comment
+type NVP struct {
+	Name    string `json:"name"`
+	Value   string `json:"value"`
+	Comment string `json:"comment,omitempty"`
+}
+
+// PostData describes posted data, if any (embedded in <request> object).
+type PostData struct {
+	//  Mime type of posted data.
+	MimeType string `json:"mimeType"`
+	//  List of posted parameters (in case of URL encoded parameters).
+	Params []PostParam `json:"params"`
+	//  Plain text posted data
+	Text string `json:"text"`
+	// optional (new in 1.2) A comment provided by the user or the
+	// application.
+	Comment string `json:"comment,omitempty"`
+}
+
+func (d PostData) B64Decoded() (bool, []byte, string) {
+	// there is a weird gap in HAR spec 1.2, that does not define encoding for binary POST bodies
+	// we have own convention of putting `base64` into comment field to handle it similar to response `Content`
+	return b64Decoded(d.Comment, d.Text)
+}
+
+// PostParam is a list of posted parameters, if any (embedded in <postData> object).
+type PostParam struct {
+	// name of a posted parameter.
+	Name string `json:"name"`
+	// optional value of a posted parameter or content of a posted file.
+	Value string `json:"value,omitempty"`
+	// optional name of a posted file.
+	FileName string `json:"fileName,omitempty"`
+	// optional content type of a posted file.
+	ContentType string `json:"contentType,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// Content describes details about response content (embedded in <response> object).
+type Content struct {
+	// Length of the returned content in bytes. Should be equal to
+	// response.bodySize if there is no compression and bigger when the content
+	// has been compressed.
+	Size int `json:"size"`
+	// optional Number of bytes saved. Leave out this field if the information
+	// is not available.
+	Compression int `json:"compression,omitempty"`
+	// MIME type of the response text (value of the Content-Type response
+	// header). The charset attribute of the MIME type is included (if
+	// available).
+	MimeType string `json:"mimeType"`
+	// optional Response body sent from the server or loaded from the browser
+	// cache. This field is populated with textual content only. The text field
+	// is either HTTP decoded text or a encoded (e.g. "base64") representation of
+	// the response body. Leave out this field if the information is not
+	// available.
+	Text string `json:"text,omitempty"`
+	// optional (new in 1.2) Encoding used for response text field e.g
+	// "base64". Leave out this field if the text field is HTTP decoded
+	// (decompressed & unchunked), than trans-coded from its original character
+	// set into UTF-8.
+	Encoding string `json:"encoding,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+func (c Content) B64Decoded() (bool, []byte, string) {
+	return b64Decoded(c.Encoding, c.Text)
+}
+
+func b64Decoded(enc string, text string) (isBinary bool, asBytes []byte, asString string) {
+	if enc == "base64" {
+		decoded, err := base64.StdEncoding.DecodeString(text)
+		if err != nil {
+			logger.Log.Warningf("Failed to decode content as base64: %s", text)
+			return false, []byte(text), text
+		}
+		valid := utf8.Valid(decoded)
+		return !valid, decoded, string(decoded)
+	} else {
+		return false, nil, text
+	}
+}
+
+// Cache contains info about a request coming from browser cache.
+type Cache struct {
+	// optional State of a cache entry before the request. Leave out this field
+	// if the information is not available.
+	BeforeRequest CacheObject `json:"beforeRequest,omitempty"`
+	// optional State of a cache entry after the request. Leave out this field if
+	// the information is not available.
+	AfterRequest CacheObject `json:"afterRequest,omitempty"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// CacheObject is used by both beforeRequest and afterRequest
+type CacheObject struct {
+	// optional - Expiration time of the cache entry.
+	Expires string `json:"expires,omitempty"`
+	// The last time the cache entry was opened.
+	LastAccess string `json:"lastAccess"`
+	// Etag
+	ETag string `json:"eTag"`
+	// The number of times the cache entry has been opened.
+	HitCount int `json:"hitCount"`
+	// optional (new in 1.2) A comment provided by the user or the application.
+	Comment string `json:"comment,omitempty"`
+}
+
+// PageTimings describes various phases within request-response round trip.
+// All times are specified in milliseconds.
+type PageTimings struct {
+	Blocked int `json:"blocked,omitempty"`
+	// optional - Time spent in a queue waiting for a network connection. Use -1
+	// if the timing does not apply to the current request.
+	DNS int `json:"dns,omitempty"`
+	// optional - DNS resolution time. The time required to resolve a host name.
+	// Use -1 if the timing does not apply to the current request.
+	Connect int `json:"connect,omitempty"`
+	// optional - Time required to create TCP connection. Use -1 if the timing
+	// does not apply to the current request.
+	Send int `json:"send"`
+	// Time required to send HTTP request to the server.
+	Wait int `json:"wait"`
+	// Waiting for a response from the server.
+	Receive int `json:"receive"`
+	// Time required to read entire response from the server (or cache).
+	Ssl int `json:"ssl,omitempty"`
+	// optional (new in 1.2) - Time required for SSL/TLS negotiation. If this
+	// field is defined then the time is also included in the connect field (to
+	// ensure backward compatibility with HAR 1.1). Use -1 if the timing does not
+	// apply to the current request.
+	Comment string `json:"comment,omitempty"`
+	// optional (new in 1.2) - A comment provided by the user or the application.
+}
+
+// TestResult contains results for an individual HTTP request
+type TestResult struct {
+	URL       string    `json:"url"`
+	Status    int       `json:"status"` // 200, 500, etc.
+	StartTime time.Time `json:"startTime"`
+	EndTime   time.Time `json:"endTime"`
+	Latency   int       `json:"latency"` // milliseconds
+	Method    string    `json:"method"`
+	HarFile   string    `json:"harfile"`
+}
+
+// aliases for martian lib compatibility
+
+type Header = NVP
+type QueryString = NVP
+type Param = PostParam
+type Timings = PageTimings

agent/pkg/har/types_test.go

@@ -0,0 +1,38 @@
+package har
+
+import "testing"
+
+func TestContentEncoded(t *testing.T) {
+	testCases := []struct {
+		text        string
+		isBinary    bool
+		expectedStr string
+		binaryLen   int
+	}{
+		{"not-base64", false, "not-base64", 10},
+		{"dGVzdA==", false, "test", 4},
+		{"test", true, "\f@A", 3}, // valid UTF-8 with some non-printable chars
+		{"IsDggPCAgPiAgID8gICAgN/vv/e/v/u/v7/9v7+/vyIKIu+3kO+3ke+3ku+3k++3lO+3le+3lu+3l++3mO+3me+3mu+3m++3nO+3ne+3nu+3n++3oO+3oe+3ou+3o++3pO+3pe+3pu+3p++3qO+3qe+3qu+3q++3rO+3re+3ru+3ryIK", true, "test", 132}, // invalid UTF-8 (thus binary), taken from https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
+	}
+
+	for _, tc := range testCases {
+		c := Content{
+			Encoding: "base64",
+			Text:     tc.text,
+		}
+		isBinary, asBytes, asString := c.B64Decoded()
+		_ = asBytes
+
+		if tc.isBinary != isBinary {
+			t.Errorf("Binary flag mismatch: %t != %t", tc.isBinary, isBinary)
+		}
+
+		if !isBinary && tc.expectedStr != asString {
+			t.Errorf("Decode value mismatch: %s != %s", tc.expectedStr, asString)
+		}
+
+		if tc.binaryLen != len(asBytes) {
+			t.Errorf("Binary len mismatch: %d != %d", tc.binaryLen, len(asBytes))
+		}
+	}
+}

agent/pkg/har/utils.go

@@ -0,0 +1,215 @@
+package har
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/up9inc/mizu/logger"
+)
+
+func BuildHeaders(rawHeaders map[string]interface{}) ([]Header, string, string, string, string, string) {
+	var host, scheme, authority, path, status string
+	headers := make([]Header, 0, len(rawHeaders))
+
+	for key, value := range rawHeaders {
+		headers = append(headers, Header{
+			Name:  key,
+			Value: value.(string),
+		})
+
+		if key == "Host" {
+			host = value.(string)
+		}
+		if key == ":authority" {
+			authority = value.(string)
+		}
+		if key == ":scheme" {
+			scheme = value.(string)
+		}
+		if key == ":path" {
+			path = value.(string)
+		}
+		if key == ":status" {
+			status = value.(string)
+		}
+	}
+
+	return headers, host, scheme, authority, path, status
+}
+
+func BuildPostParams(rawParams []interface{}) []Param {
+	params := make([]Param, 0, len(rawParams))
+	for _, param := range rawParams {
+		p := param.(map[string]interface{})
+		name := ""
+		if p["name"] != nil {
+			name = p["name"].(string)
+		}
+		value := ""
+		if p["value"] != nil {
+			value = p["value"].(string)
+		}
+		fileName := ""
+		if p["fileName"] != nil {
+			fileName = p["fileName"].(string)
+		}
+		contentType := ""
+		if p["contentType"] != nil {
+			contentType = p["contentType"].(string)
+		}
+
+		params = append(params, Param{
+			Name:        name,
+			Value:       value,
+			FileName:    fileName,
+			ContentType: contentType,
+		})
+	}
+
+	return params
+}
+
+func NewRequest(request map[string]interface{}) (harRequest *Request, err error) {
+	headers, host, scheme, authority, path, _ := BuildHeaders(request["headers"].(map[string]interface{}))
+	cookies := make([]Cookie, 0)
+
+	postData, _ := request["postData"].(map[string]interface{})
+	mimeType := postData["mimeType"]
+	if mimeType == nil {
+		mimeType = ""
+	}
+	text := postData["text"]
+	postDataText := ""
+	if text != nil {
+		postDataText = text.(string)
+	}
+
+	queryString := make([]QueryString, 0)
+	for key, value := range request["queryString"].(map[string]interface{}) {
+		if valuesInterface, ok := value.([]interface{}); ok {
+			for _, valueInterface := range valuesInterface {
+				queryString = append(queryString, QueryString{
+					Name:  key,
+					Value: valueInterface.(string),
+				})
+			}
+		} else {
+			queryString = append(queryString, QueryString{
+				Name:  key,
+				Value: value.(string),
+			})
+		}
+	}
+
+	url := fmt.Sprintf("http://%s%s", host, request["url"].(string))
+	if strings.HasPrefix(mimeType.(string), "application/grpc") {
+		url = fmt.Sprintf("%s://%s%s", scheme, authority, path)
+	}
+
+	harParams := make([]Param, 0)
+	if postData["params"] != nil {
+		harParams = BuildPostParams(postData["params"].([]interface{}))
+	}
+
+	harRequest = &Request{
+		Method:      request["method"].(string),
+		URL:         url,
+		HTTPVersion: request["httpVersion"].(string),
+		HeaderSize:  -1,
+		BodySize:    bytes.NewBufferString(postDataText).Len(),
+		QueryString: queryString,
+		Headers:     headers,
+		Cookies:     cookies,
+		PostData: PostData{
+			MimeType: mimeType.(string),
+			Params:   harParams,
+			Text:     postDataText,
+		},
+	}
+
+	return
+}
+
+func NewResponse(response map[string]interface{}) (harResponse *Response, err error) {
+	headers, _, _, _, _, _status := BuildHeaders(response["headers"].(map[string]interface{}))
+	cookies := make([]Cookie, 0)
+
+	content, _ := response["content"].(map[string]interface{})
+	mimeType := content["mimeType"]
+	if mimeType == nil {
+		mimeType = ""
+	}
+	encoding := content["encoding"]
+	text := content["text"]
+	bodyText := ""
+	if text != nil {
+		bodyText = text.(string)
+	}
+
+	harContent := &Content{
+		Encoding: encoding.(string),
+		MimeType: mimeType.(string),
+		Text:     bodyText,
+		Size:     len(bodyText),
+	}
+
+	status := int(response["status"].(float64))
+	if strings.HasPrefix(mimeType.(string), "application/grpc") {
+		if _status != "" {
+			status, err = strconv.Atoi(_status)
+		}
+		if err != nil {
+			logger.Log.Errorf("Failed converting status to int %s (%v,%+v)", err, err, err)
+			return nil, errors.New("failed converting response status to int for HAR")
+		}
+	}
+
+	harResponse = &Response{
+		HTTPVersion: response["httpVersion"].(string),
+		Status:      status,
+		StatusText:  response["statusText"].(string),
+		HeadersSize: -1,
+		BodySize:    bytes.NewBufferString(bodyText).Len(),
+		Headers:     headers,
+		Cookies:     cookies,
+		Content:     *harContent,
+	}
+	return
+}
+
+func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*Entry, error) {
+	harRequest, err := NewRequest(request)
+	if err != nil {
+		logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
+		return nil, errors.New("failed converting request to HAR")
+	}
+
+	harResponse, err := NewResponse(response)
+	if err != nil {
+		logger.Log.Errorf("Failed converting response to HAR %s (%v,%+v)", err, err, err)
+		return nil, errors.New("failed converting response to HAR")
+	}
+
+	if elapsedTime < 1 {
+		elapsedTime = 1
+	}
+
+	harEntry := Entry{
+		StartedDateTime: startTime.Format(time.RFC3339),
+		Time:            int(elapsedTime),
+		Request:         *harRequest,
+		Response:        *harResponse,
+		Cache:           Cache{},
+		PageTimings: PageTimings{
+			Send:    -1,
+			Wait:    -1,
+			Receive: int(elapsedTime),
+		},
+	}
+
+	return &harEntry, nil
+}

agent/pkg/holder/main.go

@@ -0,0 +1,13 @@
+package holder
+
+import "github.com/up9inc/mizu/agent/pkg/resolver"
+
+var k8sResolver *resolver.Resolver
+
+func SetResolver(param *resolver.Resolver) {
+	k8sResolver = param
+}
+
+func GetResolver() *resolver.Resolver {
+	return k8sResolver
+}

agent/pkg/middlewares/cors.go

@@ -0,0 +1,19 @@
+package middlewares
+
+import "github.com/gin-gonic/gin"
+
+func CORSMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
+		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
+		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With, x-session-token")
+		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE")
+
+		if c.Request.Method == "OPTIONS" {
+			c.AbortWithStatus(204)
+			return
+		}
+
+		c.Next()
+	}
+}

agent/pkg/models/models.go

@@ -0,0 +1,144 @@
+package models
+
+import (
+	"encoding/json"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
+	tapApi "github.com/up9inc/mizu/tap/api"
+
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/shared"
+)
+
+type EntriesRequest struct {
+	LeftOff   string `form:"leftOff" validate:"required"`
+	Direction int    `form:"direction" validate:"required,oneof='1' '-1'"`
+	Query     string `form:"query"`
+	Limit     int    `form:"limit" validate:"required,min=1"`
+	TimeoutMs int    `form:"timeoutMs" validate:"min=1"`
+}
+
+type SingleEntryRequest struct {
+	Query string `form:"query"`
+}
+
+type EntriesResponse struct {
+	Data []interface{}      `json:"data"`
+	Meta *basenine.Metadata `json:"meta"`
+}
+
+type WebSocketEntryMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *tapApi.BaseEntry `json:"data,omitempty"`
+}
+
+type WebSocketFullEntryMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *tapApi.Entry `json:"data,omitempty"`
+}
+
+type WebSocketTappedEntryMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *tapApi.OutputChannelItem
+}
+
+type ToastMessage struct {
+	Type      string `json:"type"`
+	AutoClose uint   `json:"autoClose"`
+	Text      string `json:"text"`
+}
+
+type WebSocketToastMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *ToastMessage `json:"data,omitempty"`
+}
+
+type WebSocketQueryMetadataMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *basenine.Metadata `json:"data,omitempty"`
+}
+
+type WebSocketStartTimeMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data int64 `json:"data"`
+}
+
+func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) {
+	message := &WebSocketEntryMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeEntry,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+func CreateFullEntryWebSocketMessage(entry *tapApi.Entry) ([]byte, error) {
+	message := &WebSocketFullEntryMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeFullEntry,
+		},
+		Data: entry,
+	}
+	return json.Marshal(message)
+}
+
+func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte, error) {
+	message := &WebSocketTappedEntryMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeTappedEntry,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+func CreateWebsocketToastMessage(base *ToastMessage) ([]byte, error) {
+	message := &WebSocketToastMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeToast,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+func CreateWebsocketQueryMetadataMessage(base *basenine.Metadata) ([]byte, error) {
+	message := &WebSocketQueryMetadataMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeQueryMetadata,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+func CreateWebsocketStartTimeMessage(base int64) ([]byte, error) {
+	message := &WebSocketStartTimeMessage{
+		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
+			MessageType: shared.WebSocketMessageTypeStartTime,
+		},
+		Data: base,
+	}
+	return json.Marshal(message)
+}
+
+// ExtendedHAR is the top level object of a HAR log.
+type ExtendedHAR struct {
+	Log *ExtendedLog `json:"log"`
+}
+
+// ExtendedLog is the HAR HTTP request and response log.
+type ExtendedLog struct {
+	// Version number of the HAR format.
+	Version string `json:"version"`
+	// Creator holds information about the log creator application.
+	Creator *ExtendedCreator `json:"creator"`
+	// Entries is a list containing requests and responses.
+	Entries []*har.Entry `json:"entries"`
+}
+
+type ExtendedCreator struct {
+	*har.Creator
+	Source *string `json:"_source"`
+}

agent/pkg/oas/counters.go

@@ -0,0 +1,119 @@
+package oas
+
+import (
+	"fmt"
+	"github.com/chanced/openapi"
+	"math"
+	"strings"
+)
+
+type Counter struct {
+	Entries     int     `json:"entries"`
+	Failures    int     `json:"failures"`
+	FirstSeen   float64 `json:"firstSeen"`
+	LastSeen    float64 `json:"lastSeen"`
+	SumRT       float64 `json:"sumRT"`
+	SumDuration float64 `json:"sumDuration"`
+}
+
+func (c *Counter) addEntry(ts float64, rt float64, succ bool, dur float64) {
+	if dur < 0 {
+		panic("Duration cannot be negative")
+	}
+
+	c.Entries += 1
+	c.SumRT += rt
+	c.SumDuration += dur
+	if !succ {
+		c.Failures += 1
+	}
+
+	if c.FirstSeen == 0 {
+		c.FirstSeen = ts
+	} else {
+		c.FirstSeen = math.Min(c.FirstSeen, ts)
+	}
+
+	c.LastSeen = math.Max(c.LastSeen, ts)
+}
+
+func (c *Counter) addOther(other *Counter) {
+	c.Entries += other.Entries
+	c.SumRT += other.SumRT
+	c.Failures += other.Failures
+	c.SumDuration += other.SumDuration
+
+	if c.FirstSeen == 0 {
+		c.FirstSeen = other.FirstSeen
+	} else {
+		c.FirstSeen = math.Min(c.FirstSeen, other.FirstSeen)
+	}
+
+	c.LastSeen = math.Max(c.LastSeen, other.LastSeen)
+}
+
+type CounterMap map[string]*Counter
+
+func (m *CounterMap) addOther(other *CounterMap) {
+	for src, cmap := range *other {
+		if existing, ok := (*m)[src]; ok {
+			existing.addOther(cmap)
+		} else {
+			copied := *cmap
+			(*m)[src] = &copied
+		}
+	}
+}
+
+func setCounterMsgIfOk(oldStr string, cnt *Counter) string {
+	tpl := "Mizu observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds"
+	if oldStr == "" || (strings.HasPrefix(oldStr, "Mizu ") && strings.HasSuffix(oldStr, " seconds")) {
+		return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries))
+	}
+	return oldStr
+}
+
+type CounterMaps struct {
+	counterTotal    Counter
+	counterMapTotal CounterMap
+}
+
+func (m *CounterMaps) processOp(opObj *openapi.Operation) error {
+	if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
+		counter := new(Counter)
+		err := opObj.Extensions.DecodeExtension(CountersTotal, counter)
+		if err != nil {
+			return err
+		}
+		m.counterTotal.addOther(counter)
+
+		opObj.Description = setCounterMsgIfOk(opObj.Description, counter)
+	}
+
+	if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
+		counterMap := new(CounterMap)
+		err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap)
+		if err != nil {
+			return err
+		}
+		m.counterMapTotal.addOther(counterMap)
+	}
+	return nil
+}
+
+func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error {
+	if oas.Extensions == nil {
+		oas.Extensions = openapi.Extensions{}
+	}
+
+	err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal)
+	if err != nil {
+		return err
+	}
+
+	err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal)
+	if err != nil {
+		return nil
+	}
+	return nil
+}

agent/pkg/oas/feeder_test.go

@@ -0,0 +1,212 @@
+package oas
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
+
+	"github.com/up9inc/mizu/logger"
+)
+
+func getFiles(baseDir string) (result []string, err error) {
+	result = make([]string, 0)
+	logger.Log.Infof("Reading files from tree: %s", baseDir)
+
+	inputs := []string{baseDir}
+
+	// https://yourbasic.org/golang/list-files-in-directory/
+	visitor := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.Mode()&os.ModeSymlink != 0 {
+			path, _ = os.Readlink(path)
+			inputs = append(inputs, path)
+			return nil
+		}
+
+		ext := strings.ToLower(filepath.Ext(path))
+		if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
+			result = append(result, path)
+		}
+
+		return nil
+	}
+
+	for len(inputs) > 0 {
+		path := inputs[0]
+		inputs = inputs[1:]
+		err = filepath.Walk(path, visitor)
+	}
+
+	sort.SliceStable(result, func(i, j int) bool {
+		return fileSize(result[i]) < fileSize(result[j])
+	})
+
+	logger.Log.Infof("Got files: %d", len(result))
+	return result, err
+}
+
+func fileSize(fname string) int64 {
+	fi, err := os.Stat(fname)
+	if err != nil {
+		panic(err)
+	}
+
+	return fi.Size()
+}
+
+func feedEntries(fromFiles []string, isSync bool, gen *defaultOasGenerator) (count uint, err error) {
+	badFiles := make([]string, 0)
+	cnt := uint(0)
+	for _, file := range fromFiles {
+		logger.Log.Info("Processing file: " + file)
+		ext := strings.ToLower(filepath.Ext(file))
+		eCnt := uint(0)
+		switch ext {
+		case ".har":
+			eCnt, err = feedFromHAR(file, isSync, gen)
+			if err != nil {
+				logger.Log.Warning("Failed processing file: " + err.Error())
+				badFiles = append(badFiles, file)
+				continue
+			}
+		case ".ldjson":
+			eCnt, err = feedFromLDJSON(file, isSync, gen)
+			if err != nil {
+				logger.Log.Warning("Failed processing file: " + err.Error())
+				badFiles = append(badFiles, file)
+				continue
+			}
+		default:
+			return 0, errors.New("Unsupported file extension: " + ext)
+		}
+		cnt += eCnt
+	}
+
+	for _, f := range badFiles {
+		logger.Log.Infof("Bad file: %s", f)
+	}
+
+	return cnt, nil
+}
+
+func feedFromHAR(file string, isSync bool, gen *defaultOasGenerator) (uint, error) {
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		return 0, err
+	}
+
+	var harDoc har.HAR
+	err = json.Unmarshal(data, &harDoc)
+	if err != nil {
+		return 0, err
+	}
+
+	cnt := uint(0)
+	for _, entry := range harDoc.Log.Entries {
+		cnt += 1
+		feedEntry(&entry, "", file, gen, fmt.Sprintf("%024d", cnt))
+	}
+
+	return cnt, nil
+}
+
+func feedEntry(entry *har.Entry, source string, file string, gen *defaultOasGenerator, cnt string) {
+	entry.Comment = file
+	if entry.Response.Status == 302 {
+		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
+	}
+
+	if strings.Contains(entry.Request.URL, "some") { // for debugging
+		logger.Log.Debugf("Interesting: %s", entry.Request.URL)
+	}
+
+	u, err := url.Parse(entry.Request.URL)
+	if err != nil {
+		logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
+	}
+
+	ews := EntryWithSource{Entry: *entry, Source: source, Destination: u.Host, Id: cnt}
+	gen.handleHARWithSource(&ews)
+}
+
+func feedFromLDJSON(file string, isSync bool, gen *defaultOasGenerator) (uint, error) {
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	reader := bufio.NewReader(fd)
+
+	var meta map[string]interface{}
+	buf := strings.Builder{}
+	cnt := uint(0)
+	source := ""
+	for {
+		substr, isPrefix, err := reader.ReadLine()
+		if err == io.EOF {
+			break
+		}
+
+		buf.WriteString(string(substr))
+		if isPrefix {
+			continue
+		}
+
+		line := buf.String()
+		buf.Reset()
+
+		if meta == nil {
+			err := json.Unmarshal([]byte(line), &meta)
+			if err != nil {
+				return 0, err
+			}
+			if s, ok := meta["_source"]; ok && s != nil {
+				source = s.(string)
+			}
+		} else {
+			var entry har.Entry
+			err := json.Unmarshal([]byte(line), &entry)
+			if err != nil {
+				logger.Log.Warningf("Failed decoding entry: %s", line)
+			} else {
+				cnt += 1
+				feedEntry(&entry, source, file, gen, fmt.Sprintf("%024d", cnt))
+			}
+		}
+	}
+
+	return cnt, nil
+}
+
+func TestFilesList(t *testing.T) {
+	res, err := getFiles("./test_artifacts/")
+	t.Log(len(res))
+	t.Log(res)
+	if err != nil || len(res) != 3 {
+		t.Logf("Should return 2 files but returned %d", len(res))
+		t.FailNow()
+	}
+}

agent/pkg/oas/gibberish.go

@@ -0,0 +1,185 @@
+package oas
+
+import (
+	"math"
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+var (
+	patUuid4    = regexp.MustCompile(`(?i)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
+	patEmail    = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
+	patLongNum  = regexp.MustCompile(`^\d{3,}$`)
+	patLongNumB = regexp.MustCompile(`[^\d]\d{3,}`)
+	patLongNumA = regexp.MustCompile(`\d{3,}[^\d]`)
+)
+
+func IsGibberish(str string) bool {
+	if IsVersionString(str) {
+		return false
+	}
+
+	if patEmail.MatchString(str) {
+		return true
+	}
+
+	if patUuid4.MatchString(str) {
+		return true
+	}
+
+	if patLongNum.MatchString(str) || patLongNumB.MatchString(str) || patLongNumA.MatchString(str) {
+		return true
+	}
+
+	//alNum := cleanStr(str, isAlNumRune)
+	//alpha := cleanStr(str, isAlphaRune)
+	// noiseAll := isNoisy(alNum)
+	//triAll := isTrigramBad(strings.ToLower(alpha))
+	// _ = noiseAll
+
+	isNotAlNum := func(r rune) bool { return !isAlNumRune(r) }
+	chunks := strings.FieldsFunc(str, isNotAlNum)
+	noisyLen := 0
+	alnumLen := 0
+	for _, chunk := range chunks {
+		alnumLen += len(chunk)
+		noise := isNoisy(chunk)
+		tri := isTrigramBad(strings.ToLower(chunk))
+		if noise || tri {
+			noisyLen += len(chunk)
+		}
+	}
+
+	return float64(noisyLen) > 0
+
+	//if float64(noisyLen) > 0 {
+	//	return true
+	//}
+
+	//if len(chunks) > 0 && float64(noisyLen) >= float64(alnumLen)/3.0 {
+	//	return true
+	//}
+
+	//if triAll {
+	//return true
+	//}
+
+	// return false
+}
+
+func noiseLevel(str string) (score float64) {
+	// opinionated algo of certain char pairs marking the non-human strings
+	prev := *new(rune)
+	cnt := 0.0
+	for _, char := range str {
+		cnt += 1
+		if prev > 0 {
+			switch {
+			// continued class of upper/lower/digit adds no noise
+			case unicode.IsUpper(prev) && unicode.IsUpper(char):
+			case unicode.IsLower(prev) && unicode.IsLower(char):
+			case unicode.IsDigit(prev) && unicode.IsDigit(char):
+
+			// upper =>
+			case unicode.IsUpper(prev) && unicode.IsLower(char):
+				score += 0.10
+			case unicode.IsUpper(prev) && unicode.IsDigit(char):
+				score += 0.5
+
+			// lower =>
+			case unicode.IsLower(prev) && unicode.IsUpper(char):
+				score += 0.75
+			case unicode.IsLower(prev) && unicode.IsDigit(char):
+				score += 0.5
+
+			// digit =>
+			case unicode.IsDigit(prev) && unicode.IsUpper(char):
+				score += 0.75
+			case unicode.IsDigit(prev) && unicode.IsLower(char):
+				score += 1.0
+
+			// the rest is 100% noise
+			default:
+				score += 1
+			}
+		}
+		prev = char
+	}
+
+	return score
+}
+
+func IsVersionString(component string) bool {
+	if component == "" {
+		return false
+	}
+
+	hasV := false
+	if strings.HasPrefix(component, "v") {
+		component = component[1:]
+		hasV = true
+	}
+
+	for _, c := range component {
+		if string(c) != "." && !unicode.IsDigit(c) {
+			return false
+		}
+	}
+
+	if !hasV && !strings.Contains(component, ".") {
+		return false
+	}
+
+	return true
+}
+
+func trigramScore(str string) (float64, int) {
+	tgScore := 0.0
+	trigrams := ngrams(str, 3)
+	if len(trigrams) > 0 {
+		for _, trigram := range trigrams {
+			score, found := corpus_trigrams[trigram]
+			if found {
+				tgScore += score
+			}
+		}
+	}
+
+	return tgScore, len(trigrams)
+}
+
+func isTrigramBad(s string) bool {
+	tgScore, cnt := trigramScore(s)
+
+	if cnt > 0 {
+		val := math.Sqrt(tgScore) / float64(cnt)
+		val2 := tgScore / float64(cnt)
+		threshold := 0.005
+		bad := val < threshold
+		threshold2 := math.Log(float64(cnt)-2) * 0.1
+		bad2 := val2 < threshold2
+		return bad && bad2 // TODO: improve this logic to be clearer
+	}
+	return false
+}
+
+func isNoisy(s string) bool {
+	noise := noiseLevel(s)
+
+	if len(s) > 0 {
+		val := (noise * noise) / float64(len(s))
+		threshold := 0.6
+		bad := val > threshold
+		return bad
+	}
+	return false
+}
+
+func ngrams(s string, n int) []string {
+	result := make([]string, 0)
+	for i := 0; i < len(s)-n+1; i++ {
+		result = append(result, s[i:i+n])
+	}
+	return result
+}

agent/pkg/oas/gibberish_test.go

@@ -0,0 +1,212 @@
+package oas
+
+import (
+	"testing"
+)
+
+func TestNegative(t *testing.T) {
+	cases := []string{
+		"",
+		"{}",
+		"0.0.29",
+		"0.1",
+		"1.0",
+		"1.0.0",
+		"2.1.73",
+		"abTestV2",
+		"actionText,setName,setAttribute,save,ignore,onEnd,getContext,end,get",
+		"AddUserGroupLink",
+		"advert-management.adBlockerMessage.html",
+		"agents.author.1.json",
+		"animated-gif",
+		"b", // can be valid hexadecimal
+		"big-danger-coronavirus-panic-greater-crisis",
+		"breakout-box",
+		"callback",
+		"core.algorithm_execution.view",
+		"core.devices.view",
+		"data.json",
+		"dialog.overlay.infinity.json",
+		"domain-input",
+		"embeddable", // lul, it's a valid HEX!
+		"embeddable_blip",
+		"E PLURIBUS UNUM",
+		"etc",
+		"eu-central-1a",
+		"fcgi-bin",
+		"footer.include.html",
+		"fullHashes:find",
+		"generate-feed",
+		"GetAds",
+		"GetCart",
+		"GetUniversalVariableUser",
+		"github-audit-exports",
+		"g.js",
+		"g.pixel",
+		".html",
+		"Hugo Michiels",
+		"image.sbix",
+		"index.html",
+		"iPad",
+		"Joanna Mazewski",
+		"LibGit2Sharp",
+		"Michael_Vaughan1.png",
+		"New RSS feed has been generated",
+		"nick-clegg",
+		"opt-out",
+		"pixel_details.html",
+		"post.json",
+		"profile-method-info",
+		"project-id",
+		"publisha.1.json",
+		"publish_and_moderate",
+		"Ronna McDaniel",
+		"rtb-h",
+		"runs",
+		"sign-up",
+		"some-uuid-maybe",
+		"stable-4.0-version.json",
+		"StartUpCheckout",
+		"Steve Flunk",
+		"sync_a9",
+		"Ted Cruz",
+		"test.png",
+		"token",
+		"ToList",
+		"v2.1.3",
+		"VersionCheck.php",
+		"v Rusiji",
+		"Walnut St",
+		"web_widget",
+		"zoom_in.cur",
+		"xray",
+		"web",
+		"vipbets1",
+		"trcc",
+		"fbpixel",
+
+		// TODO below
+		// "tcfv2",
+		// "Matt-cartoon-255x206px-small.png",
+		// "TheTelegraph_portal_white-320-small.png",
+		// "testdata-10kB.js",
+	}
+
+	for _, str := range cases {
+		if IsGibberish(str) {
+			t.Errorf("Mistakenly true: %s", str)
+		}
+	}
+}
+
+func TestPositive(t *testing.T) {
+	cases := []string{
+		"0a0d0174-b338-4520-a1c3-24f7e3d5ec50.html",
+		"1024807212418223",
+		"11ca096cbc224a67360493d44a9903",
+		"1553183382779",
+		"1554507871",
+		"19180481",
+		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
+		"456795af-b48f-4a8d-9b37-3e932622c2f0",
+		"601a2bdcc5b69137248ddbbf",
+		"60fe9aaeaefe2400012df94f",
+		"610bc3fd5a77a7fa25033fb0",
+		"610bd0315a77a7fa25034368",
+		"610bd0315a77a7fa25034368zh",
+		"6120c057c7a97b03f6986f1b",
+		"710a462e",
+		"730970532670-compute@developer.gserviceaccount.com",
+		"819db2242a648b305395537022523d65",
+		"952bea17-3776-11ea-9341-42010a84012a",
+		"a3226860758.html",
+		"AAAA028295945",
+		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4",
+		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
+		"bnjksfd897345nl098asd53412kl98",
+		"c738338322370b47a79251f7510dd",                        // prefixed hex
+		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
+		"css/login.0f48c49a34eb53ea4623.min.css",
+		"d_fLLxlhzDilixeBEimaZ5",
+		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
+		"e8782afc112720300c049ff124434b79",
+		"fb6cjraf9cejut2a",
+		"i-0236530c66ed02200",
+		"JEHJW4BKVFDRTMTUQLHKK5WVAU",
+		"john.dow.1981@protonmail.com",
+		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
+		"MNUTGVFMGLEMFTBH0XSE5E02F6J2DS",
+		"n63nd45qsj",
+		"n9z9QGNiz",
+		"NC4WTmcy",
+		"proxy.3d2100fd7107262ecb55ce6847f01fa5.html",
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
+		"qwerqwerasdfqwer@protonmai.com",
+		"r-ext-5579e00a95c90",
+		"r-ext-5579e8b12f11e",
+		"r-v4-5c92513c9e04a",
+		"r-v4-5c92513c9e04a-0-c.monitoring.json",
+		"segments-1563566437171.639994",
+		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
+		"sp_dxJTfx11_576742227280287872",
+		"sp_NnUPB5wj_601a2bdcc5b69137248ddbbf",
+		"sp_NxITuoE4_premiumchron-article-14302157_c_ryGQBs_r_yIWvwP",
+		"t_52d94268-8810-4a7e-ba87-ffd657a6752f",
+		"timeouts-1563566437171.639994",
+		"u_YPF3GsGKMo02",
+
+		"0000000000 65535 f",
+		"0000000178 00000 n",
+		"0-10000",
+		"01526123,",
+		"0,18168,183955,3,4,1151616,5663,731,223,5104,207,3204,10,1051,175,364,1435,4,60,576,241,383,246,5,1102",
+		"05/10/2020",
+		"14336456724940333",
+		"fb6cjraf9cejut2a",
+		"JEHJW4BKVFDRTMTUQLHKK5WVAU",
+
+		// TODO
+		/*
+			"0,20",
+			"0.001",
+			"YISAtiX1",
+			"Fxvd1timk", // questionable
+			"B4GCSkORAJs",
+			"D_4EDAqenHQ",
+			"EICJp29EGOk",
+			"Fxvd1timk",
+			"GTqMZELYfQQ",
+			"GZPTpLPEGmwHGWPC",
+			"_HChnE9NDPY",
+			"NwhjgIWHgGg",
+			"production/tsbqksph4xswqjexfbec",
+			"p/u/bguhrxupr23mw3nwxcrw",
+			"nRSNapbJZnc",
+			"zgfpbtolciznub5egzxk",
+			"zufnu7aimadua9wrgwwo",
+			"zznto1jzch9yjsbtbrul",
+		*/
+	}
+
+	for _, str := range cases {
+		if !IsGibberish(str) {
+			t.Errorf("Mistakenly false: %s", str)
+		}
+	}
+}
+
+func TestVersionStrings(t *testing.T) {
+	cases := []string{
+		"1.0",
+		"1.0.0",
+		"v2.1.3",
+		"2.1.73",
+	}
+
+	for _, str := range cases {
+		if !IsVersionString(str) {
+			t.Errorf("Mistakenly false: %s", str)
+		}
+	}
+}

agent/pkg/oas/ignores.go

@@ -0,0 +1,77 @@
+package oas
+
+import "strings"
+
+var ignoredExtensions = []string{"gif", "svg", "css", "png", "ico", "js", "woff2", "woff", "jpg", "jpeg", "swf", "ttf", "map", "webp", "otf", "mp3"}
+
+var ignoredCtypePrefixes = []string{"image/", "font/", "video/", "audio/", "text/javascript"}
+var ignoredCtypes = []string{"application/javascript", "application/x-javascript", "text/css", "application/font-woff2", "application/font-woff", "application/x-font-woff"}
+
+var ignoredHeaders = []string{
+	"a-im", "accept",
+	"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-range", "content-type", "cookie",
+	"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
+	"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
+	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests", "x-download-options",
+	"server", "user-agent", "via", "warning", "strict-transport-security", "x-permitted-cross-domain-policies",
+	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data", "x-dns-prefetch-control",
+	"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
+	"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
+	"x-wap-profile", "x-scheme", "status", "x-cache", "x-application-context", "retry-after",
+	"newrelic", "x-cloud-trace-context", "sentry-trace", "x-cache-hits", "x-served-by", "x-span-name",
+	"expires", "set-cookie", "p3p", "content-security-policy", "content-security-policy-report-only",
+	"last-modified", "content-language", "x-varnish", "true-client-ip", "akamai-origin-hop",
+	"keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by",
+	"age", "allow", "www-authenticate", "expect-ct", "timing-allow-origin", "referrer-policy",
+	"x-aspnet-version", "x-aspnetmvc-version", "x-timer", "x-abuse-info", "x-mod-pagespeed",
+	"duration_ms", // UP9 custom
+}
+
+var ignoredHeaderPrefixes = []string{
+	":", "accept-", "access-control-", "if-", "sec-", "grpc-",
+	"x-forwarded-", "x-original-", "cf-",
+	"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
+	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-", "x-goog-",
+}
+
+func isCtypeIgnored(ctype string) bool {
+	for _, prefix := range ignoredCtypePrefixes {
+		if strings.HasPrefix(ctype, prefix) {
+			return true
+		}
+	}
+
+	for _, toIgnore := range ignoredCtypes {
+		if ctype == toIgnore {
+			return true
+		}
+	}
+	return false
+}
+
+func isExtIgnored(path string) bool {
+	for _, extIgn := range ignoredExtensions {
+		if strings.HasSuffix(path, "."+extIgn) {
+			return true
+		}
+	}
+	return false
+}
+
+func isHeaderIgnored(name string) bool {
+	name = strings.ToLower(name)
+
+	for _, ignore := range ignoredHeaders {
+		if name == ignore {
+			return true
+		}
+	}
+
+	for _, prefix := range ignoredHeaderPrefixes {
+		if strings.HasPrefix(name, prefix) {
+			return true
+		}
+	}
+
+	return false
+}

agent/pkg/oas/oas-samples/carts.json

@@ -0,0 +1,96 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "http://carts",
+    "description": "Mizu observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
+    "version": "1.0"
+  },
+  "servers": [
+    {
+      "url": "http://carts"
+    }
+  ],
+  "paths": {
+    "/carts/{cartId}/items": {
+      "get": {
+        "summary": "/carts/{cartId}/items",
+        "description": "Mizu observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
+        "operationId": "84c9b926-1f73-4ab4-b381-3c124528959f",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": [
+                  {
+                    "id": "60fe98fb86c0fc000869a90c",
+                    "itemId": "3395a43e-2d88-40de-b95f-e00e1502085b",
+                    "quantity": 1,
+                    "unitPrice": 18
+                  }
+                ],
+                "x-sample-entry": "000000000000000000000010"
+              }
+            },
+            "x-sample-entry": "000000000000000000000010"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 3,
+            "failures": 0,
+            "firstSeen": 1627298058.3798368,
+            "lastSeen": 1627298065.2397773,
+            "sumRT": 0.05,
+            "sumDuration": 6.859940528869629
+          }
+        },
+        "x-counters-total": {
+          "entries": 3,
+          "failures": 0,
+          "firstSeen": 1627298058.3798368,
+          "lastSeen": 1627298065.2397773,
+          "sumRT": 0.05,
+          "sumDuration": 6.859940528869629
+        },
+        "x-last-seen-ts": 1627298065.2397773,
+        "x-sample-entry": "000000000000000000000010"
+      },
+      "parameters": [
+        {
+          "name": "cartId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "mHK0P7zTktmV1zv57iWAvCTd43FFMHap"
+            }
+          },
+          "x-sample-entry": "000000000000000000000010"
+        }
+      ]
+    }
+  },
+  "x-counters-per-source": {
+    "some-source": {
+      "entries": 3,
+      "failures": 0,
+      "firstSeen": 1627298058.3798368,
+      "lastSeen": 1627298065.2397773,
+      "sumRT": 0.05,
+      "sumDuration": 6.859940528869629
+    }
+  },
+  "x-counters-total": {
+    "entries": 3,
+    "failures": 0,
+    "firstSeen": 1627298058.3798368,
+    "lastSeen": 1627298065.2397773,
+    "sumRT": 0.05,
+    "sumDuration": 6.859940528869629
+  }
+}

agent/pkg/oas/oas-samples/catalogue.json

@@ -0,0 +1,485 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "Preloaded",
+    "description": "Test file for loading pre-existing OAS",
+    "version": "0.1"
+  },
+  "paths": {
+    "/catalogue": {
+      "get": {
+        "tags": [
+          "catalogue"
+        ],
+        "summary": "/catalogue",
+        "description": "Mizu observed 3 entries (0 failed), at 2.647 hits/s, average response time is 0.008 seconds",
+        "operationId": "dd6c3dbe-6b6b-4ddd-baed-757e237ddb8a",
+        "parameters": [
+          {
+            "name": "page",
+            "in": "query",
+            "required": false,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "1"
+              }
+            },
+            "x-sample-entry": "000000000000000000000002"
+          },
+          {
+            "name": "size",
+            "in": "query",
+            "required": true,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "6"
+              },
+              "example #1": {
+                "value": "3"
+              },
+              "example #2": {
+                "value": "5"
+              }
+            },
+            "x-sample-entry": "000000000000000000000011"
+          },
+          {
+            "name": "tags",
+            "in": "query",
+            "required": false,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": ""
+              },
+              "example #1": {
+                "value": "blue"
+              }
+            },
+            "x-sample-entry": "000000000000000000000007"
+          },
+          {
+            "name": "sort",
+            "in": "query",
+            "required": false,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "id"
+              }
+            },
+            "x-sample-entry": "000000000000000000000007"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": [
+                  {
+                    "count": 1,
+                    "description": "Socks fit for a Messiah. You too can experience walking in water with these special edition beauties. Each hole is lovingly proggled to leave smooth edges. The only sock approved by a higher power.",
+                    "id": "03fef6ac-1896-4ce8-bd69-b798f85c6e0b",
+                    "imageUrl": [
+                      "/catalogue/images/holy_1.jpeg",
+                      "/catalogue/images/holy_2.jpeg"
+                    ],
+                    "name": "Holy",
+                    "price": 99.99,
+                    "tag": [
+                      "action",
+                      "magic"
+                    ]
+                  },
+                  {
+                    "count": 438,
+                    "description": "proident occaecat irure et excepteur labore minim nisi amet irure",
+                    "id": "3395a43e-2d88-40de-b95f-e00e1502085b",
+                    "imageUrl": [
+                      "/catalogue/images/colourful_socks.jpg",
+                      "/catalogue/images/colourful_socks.jpg"
+                    ],
+                    "name": "Colourful",
+                    "price": 18,
+                    "tag": [
+                      "brown",
+                      "blue"
+                    ]
+                  },
+                  {
+                    "count": 820,
+                    "description": "Ready for action. Engineers: be ready to smash that next bug! Be ready, with these super-action-sport-masterpieces. This particular engineer was chased away from the office with a stick.",
+                    "id": "510a0d7e-8e83-4193-b483-e27e09ddc34d",
+                    "imageUrl": [
+                      "/catalogue/images/puma_1.jpeg",
+                      "/catalogue/images/puma_2.jpeg"
+                    ],
+                    "name": "SuperSport XL",
+                    "price": 15,
+                    "tag": [
+                      "sport",
+                      "formal",
+                      "black"
+                    ]
+                  },
+                  {
+                    "count": 738,
+                    "description": "A mature sock, crossed, with an air of nonchalance.",
+                    "id": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38",
+                    "imageUrl": [
+                      "/catalogue/images/cross_1.jpeg",
+                      "/catalogue/images/cross_2.jpeg"
+                    ],
+                    "name": "Crossed",
+                    "price": 17.32,
+                    "tag": [
+                      "blue",
+                      "action",
+                      "red",
+                      "formal"
+                    ]
+                  },
+                  {
+                    "count": 808,
+                    "description": "enim officia aliqua excepteur esse deserunt quis aliquip nostrud anim",
+                    "id": "819e1fbf-8b7e-4f6d-811f-693534916a8b",
+                    "imageUrl": [
+                      "/catalogue/images/WAT.jpg",
+                      "/catalogue/images/WAT2.jpg"
+                    ],
+                    "name": "Figueroa",
+                    "price": 14,
+                    "tag": [
+                      "green",
+                      "formal",
+                      "blue"
+                    ]
+                  },
+                  {
+                    "count": 175,
+                    "description": "consequat amet cupidatat minim laborum tempor elit ex consequat in",
+                    "id": "837ab141-399e-4c1f-9abc-bace40296bac",
+                    "imageUrl": [
+                      "/catalogue/images/catsocks.jpg",
+                      "/catalogue/images/catsocks2.jpg"
+                    ],
+                    "name": "Cat socks",
+                    "price": 15,
+                    "tag": [
+                      "brown",
+                      "formal",
+                      "green"
+                    ]
+                  }
+                ],
+                "x-sample-entry": "000000000000000000000011"
+              }
+            },
+            "x-sample-entry": "000000000000000000000011"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 3,
+            "failures": 0,
+            "firstSeen": 1627298057.7849188,
+            "lastSeen": 1627298065.7258668,
+            "sumRT": 0.024999999999999998,
+            "sumDuration": 7.940948009490967
+          }
+        },
+        "x-counters-total": {
+          "entries": 3,
+          "failures": 0,
+          "firstSeen": 1627298057.7849188,
+          "lastSeen": 1627298065.7258668,
+          "sumRT": 0.024999999999999998,
+          "sumDuration": 7.940948009490967
+        },
+        "x-last-seen-ts": 1627298065.7258668,
+        "x-sample-entry": "000000000000000000000011"
+      }
+    },
+    "/catalogue/size": {
+      "get": {
+        "tags": [
+          "catalogue"
+        ],
+        "summary": "/catalogue/size",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.013 seconds",
+        "operationId": "2315e69d-9d66-48cf-b3d3-fec9c30bd28b",
+        "parameters": [
+          {
+            "name": "tags",
+            "in": "query",
+            "required": true,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": ""
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
+          },
+          {
+            "name": "x-some",
+            "in": "header",
+            "required": true,
+            "style": "simple",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "demo val"
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": {
+                  "err": null,
+                  "size": 9
+                },
+                "x-sample-entry": "000000000000000000000001"
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1627298057.7841518,
+            "lastSeen": 1627298057.7841518,
+            "sumRT": 0.013,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1627298057.7841518,
+          "lastSeen": 1627298057.7841518,
+          "sumRT": 0.013,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1627298057.7841518,
+        "x-sample-entry": "000000000000000000000001"
+      }
+    },
+    "/catalogue/{id}": {
+      "get": {
+        "tags": [
+          "catalogue"
+        ],
+        "summary": "/catalogue/{id}",
+        "description": "Mizu observed 4 entries (0 failed), at 1.899 hits/s, average response time is 0.003 seconds",
+        "parameters": [
+          {
+            "name": "non-required-header",
+            "in": "header",
+            "required": false,
+            "style": "simple",
+            "schema": {
+              "type": "string"
+            },
+            "example": "some-uuid-maybe"
+          },
+          {
+            "name": "x-some",
+            "in": "header",
+            "required": false,
+            "style": "simple",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "demoval"
+              }
+            },
+            "x-sample-entry": "000000000000000000000004"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": {
+                  "count": 438,
+                  "description": "proident occaecat irure et excepteur labore minim nisi amet irure",
+                  "id": "3395a43e-2d88-40de-b95f-e00e1502085b",
+                  "imageUrl": [
+                    "/catalogue/images/colourful_socks.jpg",
+                    "/catalogue/images/colourful_socks.jpg"
+                  ],
+                  "name": "Colourful",
+                  "price": 18,
+                  "tag": [
+                    "brown",
+                    "blue"
+                  ]
+                },
+                "x-sample-entry": "000000000000000000000012"
+              }
+            },
+            "x-sample-entry": "000000000000000000000012"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 4,
+            "failures": 0,
+            "firstSeen": 1627298058.1315014,
+            "lastSeen": 1627298065.7293031,
+            "sumRT": 0.013999999999999999,
+            "sumDuration": 7.597801685333252
+          }
+        },
+        "x-counters-total": {
+          "entries": 4,
+          "failures": 0,
+          "firstSeen": 1627298058.1315014,
+          "lastSeen": 1627298065.7293031,
+          "sumRT": 0.013999999999999999,
+          "sumDuration": 7.597801685333252
+        },
+        "x-last-seen-ts": 1627298065.7293031,
+        "x-sample-entry": "000000000000000000000012"
+      },
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "3395a43e-2d88-40de-b95f-e00e1502085b"
+            },
+            "example #1": {
+              "value": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38"
+            }
+          },
+          "example": "some-uuid-maybe",
+          "x-sample-entry": "000000000000000000000012"
+        }
+      ]
+    },
+    "/catalogue/{id}/details": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ]
+    },
+    "/tags": {
+      "get": {
+        "summary": "/tags",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.007 seconds",
+        "operationId": "c4d7d0ed-1a78-4370-a049-efe3abc631a6",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": {
+                  "err": null,
+                  "tags": [
+                    "brown",
+                    "geek",
+                    "formal",
+                    "blue",
+                    "skin",
+                    "red",
+                    "action",
+                    "sport",
+                    "black",
+                    "magic",
+                    "green"
+                  ]
+                },
+                "x-sample-entry": "000000000000000000000003"
+              }
+            },
+            "x-sample-entry": "000000000000000000000003"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1627298057.7841816,
+            "lastSeen": 1627298057.7841816,
+            "sumRT": 0.007,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1627298057.7841816,
+          "lastSeen": 1627298057.7841816,
+          "sumRT": 0.007,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1627298057.7841816,
+        "x-sample-entry": "000000000000000000000003"
+      }
+    }
+  },
+  "x-counters-per-source": {
+    "some-source": {
+      "entries": 9,
+      "failures": 0,
+      "firstSeen": 1627298057.7841518,
+      "lastSeen": 1627298065.7293031,
+      "sumRT": 0.05899999999999999,
+      "sumDuration": 15.538749694824219
+    }
+  },
+  "x-counters-total": {
+    "entries": 9,
+    "failures": 0,
+    "firstSeen": 1627298057.7841518,
+    "lastSeen": 1627298065.7293031,
+    "sumRT": 0.05899999999999999,
+    "sumDuration": 15.538749694824219
+  }
+}

agent/pkg/oas/oas-samples/httpbin.org.json

@@ -0,0 +1,897 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "https://httpbin.org",
+    "description": "Mizu observed 19 entries (0 failed), at 0.106 hits/s, average response time is 0.172 seconds",
+    "version": "1.0"
+  },
+  "servers": [
+    {
+      "url": "https://httpbin.org"
+    }
+  ],
+  "paths": {
+    "/appears-once": {
+      "get": {
+        "summary": "/appears-once",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "2d34623e-fde8-4720-8390-9a7439051755",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000004"
+              }
+            },
+            "x-sample-entry": "000000000000000000000004"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750580.0471218,
+            "lastSeen": 1567750580.0471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750580.0471218,
+          "lastSeen": 1567750580.0471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750580.0471218,
+        "x-sample-entry": "000000000000000000000004"
+      }
+    },
+    "/appears-twice": {
+      "get": {
+        "summary": "/appears-twice",
+        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds",
+        "operationId": "9c5330f3-8062-468b-b5a3-df1ad82b4846",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000006"
+              }
+            },
+            "x-sample-entry": "000000000000000000000006"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750580.7471218,
+            "lastSeen": 1567750581.7471218,
+            "sumRT": 1.26,
+            "sumDuration": 1
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750580.7471218,
+          "lastSeen": 1567750581.7471218,
+          "sumRT": 1.26,
+          "sumDuration": 1
+        },
+        "x-last-seen-ts": 1567750581.7471218,
+        "x-sample-entry": "000000000000000000000006"
+      }
+    },
+    "/body-optional": {
+      "post": {
+        "summary": "/body-optional",
+        "description": "Mizu observed 3 entries (0 failed), at 0.003 hits/s, average response time is 0.001 seconds",
+        "operationId": "34f3d66c-b1f7-4dca-9cab-987fcc8ae472",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000012"
+              }
+            },
+            "x-sample-entry": "000000000000000000000012"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 3,
+            "failures": 0,
+            "firstSeen": 1567750581.7471218,
+            "lastSeen": 1567750581.757122,
+            "sumRT": 0.003,
+            "sumDuration": 0.010000228881835938
+          }
+        },
+        "x-counters-total": {
+          "entries": 3,
+          "failures": 0,
+          "firstSeen": 1567750581.7471218,
+          "lastSeen": 1567750581.757122,
+          "sumRT": 0.003,
+          "sumDuration": 0.010000228881835938
+        },
+        "x-last-seen-ts": 1567750581.757122,
+        "x-sample-entry": "000000000000000000000012",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "application/json": {
+              "example": "{\"key\", \"val\"}",
+              "x-sample-entry": "000000000000000000000011"
+            }
+          },
+          "x-sample-entry": "000000000000000000000012"
+        }
+      }
+    },
+    "/body-required": {
+      "post": {
+        "summary": "/body-required",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "ff6add53-ab1c-4d4e-b590-0835fa318276",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000013"
+              }
+            },
+            "x-sample-entry": "000000000000000000000013"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750581.757122,
+            "lastSeen": 1567750581.757122,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750581.757122,
+          "lastSeen": 1567750581.757122,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750581.757122,
+        "x-sample-entry": "000000000000000000000013",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "": {
+              "example": "body exists",
+              "x-sample-entry": "000000000000000000000013"
+            }
+          },
+          "required": true,
+          "x-sample-entry": "000000000000000000000013"
+        }
+      }
+    },
+    "/form-multipart": {
+      "post": {
+        "summary": "/form-multipart",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "153f0925-9fc7-4e9f-9d33-f1470f25f0f7",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "example": {},
+                "x-sample-entry": "000000000000000000000009"
+              }
+            },
+            "x-sample-entry": "000000000000000000000009"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.7471218,
+            "lastSeen": 1567750582.7471218,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.7471218,
+          "lastSeen": 1567750582.7471218,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.7471218,
+        "x-sample-entry": "000000000000000000000009",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "multipart/form-data": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "file",
+                  "path"
+                ],
+                "properties": {
+                  "file": {
+                    "type": "string",
+                    "contentMediaType": "application/json",
+                    "examples": [
+                      "{\"functions\": 123}"
+                    ]
+                  },
+                  "path": {
+                    "type": "string",
+                    "examples": [
+                      "/content/components"
+                    ]
+                  }
+                }
+              },
+              "example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n",
+              "x-sample-entry": "000000000000000000000009"
+            }
+          },
+          "required": true,
+          "x-sample-entry": "000000000000000000000009"
+        }
+      }
+    },
+    "/form-urlencoded": {
+      "post": {
+        "summary": "/form-urlencoded",
+        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds",
+        "operationId": "c92189f5-5636-46eb-ac71-92b17941a568",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000008"
+              }
+            },
+            "x-sample-entry": "000000000000000000000008"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750580.7471218,
+            "lastSeen": 1567750581.7471218,
+            "sumRT": 0.002,
+            "sumDuration": 1
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750580.7471218,
+          "lastSeen": 1567750581.7471218,
+          "sumRT": 0.002,
+          "sumDuration": 1
+        },
+        "x-last-seen-ts": 1567750581.7471218,
+        "x-sample-entry": "000000000000000000000008",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "application/x-www-form-urlencoded": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "agent-id",
+                  "callback-url",
+                  "token"
+                ],
+                "properties": {
+                  "agent-id": {
+                    "type": "string",
+                    "examples": [
+                      "ade"
+                    ]
+                  },
+                  "callback-url": {
+                    "type": "string",
+                    "examples": [
+                      ""
+                    ]
+                  },
+                  "optional": {
+                    "type": "string",
+                    "examples": [
+                      "another"
+                    ]
+                  },
+                  "token": {
+                    "type": "string",
+                    "examples": [
+                      "sometoken",
+                      "sometoken-second-val"
+                    ]
+                  }
+                }
+              },
+              "example": "agent-id=ade\u0026callback-url=\u0026token=sometoken",
+              "x-sample-entry": "000000000000000000000008"
+            }
+          },
+          "required": true,
+          "x-sample-entry": "000000000000000000000008"
+        }
+      }
+    },
+    "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "85270437-7aae-4a5b-b988-3662092463d0",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000014"
+              }
+            },
+            "x-sample-entry": "000000000000000000000014"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582,
+            "lastSeen": 1567750582,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582,
+          "lastSeen": 1567750582,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582,
+        "x-sample-entry": "000000000000000000000014"
+      },
+      "parameters": [
+        {
+          "name": "prefixgibberishfineId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "234324"
+            }
+          },
+          "x-sample-entry": "000000000000000000000014"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}",
+        "description": "Mizu observed 2 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "da597734-1cf5-4d3b-917b-6b02dacf7b7b",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000018"
+              }
+            },
+            "x-sample-entry": "000000000000000000000018"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750582.000003,
+            "lastSeen": 1567750582.000004,
+            "sumRT": 0.002,
+            "sumDuration": 9.5367431640625e-7
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750582.000003,
+          "lastSeen": 1567750582.000004,
+          "sumRT": 0.002,
+          "sumDuration": 9.5367431640625e-7
+        },
+        "x-last-seen-ts": 1567750582.000004,
+        "x-sample-entry": "000000000000000000000018"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/1": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/1",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "e965a245-9cfc-48ed-94e1-f765eadb3960",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000015"
+              }
+            },
+            "x-sample-entry": "000000000000000000000015"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.000001,
+            "lastSeen": 1567750582.000001,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.000001,
+          "lastSeen": 1567750582.000001,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.000001,
+        "x-sample-entry": "000000000000000000000015"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/static": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/static",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "7af420dc-f8b7-450f-8f6f-18b039aa3cde",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000016"
+              }
+            },
+            "x-sample-entry": "000000000000000000000016"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.000002,
+            "lastSeen": 1567750582.000002,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.000002,
+          "lastSeen": 1567750582.000002,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.000002,
+        "x-sample-entry": "000000000000000000000016"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/{param1}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/{param1}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "02a1771d-2d50-4a8c-8be2-29c7e59b8435",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000019"
+              }
+            },
+            "x-sample-entry": "000000000000000000000019"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.000002,
+            "lastSeen": 1567750582.000002,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.000002,
+          "lastSeen": 1567750582.000002,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.000002,
+        "x-sample-entry": "000000000000000000000019"
+      },
+      "parameters": [
+        {
+          "name": "param1",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "23421"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        },
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/{Id}": {
+      "get": {
+        "summary": "/{Id}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "77ec4910-d47a-46a5-8234-fb80a11034b4",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000003"
+              }
+            },
+            "x-sample-entry": "000000000000000000000003"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750579.7471218,
+            "lastSeen": 1567750579.7471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750579.7471218,
+          "lastSeen": 1567750579.7471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750579.7471218,
+        "x-sample-entry": "000000000000000000000003"
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          },
+          "x-sample-entry": "000000000000000000000003"
+        }
+      ]
+    },
+    "/{Id}/sub1": {
+      "get": {
+        "summary": "/{Id}/sub1",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds",
+        "operationId": "198675eb-9faf-407b-83fa-0483a730bbbe",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "text/html": {
+                "x-sample-entry": "000000000000000000000001"
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750483.864529,
+            "lastSeen": 1567750483.864529,
+            "sumRT": 0.111,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750483.864529,
+          "lastSeen": 1567750483.864529,
+          "sumRT": 0.111,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750483.864529,
+        "x-sample-entry": "000000000000000000000001"
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          },
+          "x-sample-entry": "000000000000000000000003"
+        }
+      ]
+    },
+    "/{Id}/sub2": {
+      "get": {
+        "summary": "/{Id}/sub2",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "31d880f1-152f-4dd6-84a7-463e13b694a5",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000002"
+              }
+            },
+            "x-sample-entry": "000000000000000000000002"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750578.7471218,
+            "lastSeen": 1567750578.7471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750578.7471218,
+          "lastSeen": 1567750578.7471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750578.7471218,
+        "x-sample-entry": "000000000000000000000002"
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          },
+          "x-sample-entry": "000000000000000000000003"
+        }
+      ]
+    }
+  },
+  "x-counters-per-source": {
+    "": {
+      "entries": 19,
+      "failures": 0,
+      "firstSeen": 1567750483.864529,
+      "lastSeen": 1567750582.7471218,
+      "sumRT": 3.273999999999999,
+      "sumDuration": 2.0100011825561523
+    }
+  },
+  "x-counters-total": {
+    "entries": 19,
+    "failures": 0,
+    "firstSeen": 1567750483.864529,
+    "lastSeen": 1567750582.7471218,
+    "sumRT": 3.273999999999999,
+    "sumDuration": 2.0100011825561523
+  }
+}

agent/pkg/oas/oas_generator.go

@@ -0,0 +1,142 @@
+package oas
+
+import (
+	"encoding/json"
+	"net/url"
+	"sync"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/tap/api"
+)
+
+var (
+	syncOnce sync.Once
+	instance *defaultOasGenerator
+)
+
+type OasGeneratorSink interface {
+	HandleEntry(mizuEntry *api.Entry)
+}
+
+type OasGenerator interface {
+	Start()
+	Stop()
+	IsStarted() bool
+	GetServiceSpecs() *sync.Map
+}
+
+type defaultOasGenerator struct {
+	started       bool
+	serviceSpecs  *sync.Map
+	maxExampleLen int
+}
+
+func GetDefaultOasGeneratorInstance(maxExampleLen int) *defaultOasGenerator {
+	syncOnce.Do(func() {
+		instance = NewDefaultOasGenerator(maxExampleLen)
+		logger.Log.Debug("OAS Generator Initialized")
+	})
+	return instance
+}
+
+func (g *defaultOasGenerator) Start() {
+	g.started = true
+}
+
+func (g *defaultOasGenerator) Stop() {
+	if !g.started {
+		return
+	}
+
+	g.started = false
+
+	g.reset()
+}
+
+func (g *defaultOasGenerator) IsStarted() bool {
+	return g.started
+}
+
+func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry) {
+	if !g.started {
+		return
+	}
+
+	if mizuEntry.Protocol.Name == "http" {
+		dest := mizuEntry.Destination.Name
+		if dest == "" {
+			logger.Log.Debugf("OAS: Unresolved entry %d", mizuEntry.Id)
+			return
+		}
+
+		entry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
+		if err != nil {
+			logger.Log.Warningf("Failed to turn MizuEntry %d into HAR Entry: %s", mizuEntry.Id, err)
+			return
+		}
+
+		entryWSource := &EntryWithSource{
+			Entry:       *entry,
+			Source:      mizuEntry.Source.Name,
+			Destination: dest,
+			Id:          mizuEntry.Id,
+		}
+
+		g.handleHARWithSource(entryWSource)
+	} else {
+		logger.Log.Debugf("OAS: Unsupported protocol in entry %d: %s", mizuEntry.Id, mizuEntry.Protocol.Name)
+	}
+}
+
+func (g *defaultOasGenerator) handleHARWithSource(entryWSource *EntryWithSource) {
+	entry := entryWSource.Entry
+	gen := g.getGen(entryWSource.Destination, entry.Request.URL)
+
+	opId, err := gen.feedEntry(entryWSource)
+	if err != nil {
+		txt, suberr := json.Marshal(entry)
+		if suberr == nil {
+			logger.Log.Debugf("Problematic entry: %s", txt)
+		}
+
+		logger.Log.Warningf("Failed processing entry %d: %s", entryWSource.Id, err)
+		return
+	}
+
+	logger.Log.Debugf("Handled entry %s as opId: %s", entryWSource.Id, opId) // TODO: set opId back to entry?
+}
+
+func (g *defaultOasGenerator) getGen(dest string, urlStr string) *SpecGen {
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", urlStr, err)
+	}
+
+	val, found := g.serviceSpecs.Load(dest)
+	var gen *SpecGen
+	if !found {
+		gen = NewGen(u.Scheme + "://" + dest)
+		gen.MaxExampleLen = g.maxExampleLen
+		g.serviceSpecs.Store(dest, gen)
+	} else {
+		gen = val.(*SpecGen)
+	}
+	return gen
+}
+
+func (g *defaultOasGenerator) reset() {
+	g.serviceSpecs = &sync.Map{}
+}
+
+func (g *defaultOasGenerator) GetServiceSpecs() *sync.Map {
+	return g.serviceSpecs
+}
+
+func NewDefaultOasGenerator(maxExampleLen int) *defaultOasGenerator {
+	return &defaultOasGenerator{
+		started:       false,
+		serviceSpecs:  &sync.Map{},
+		maxExampleLen: maxExampleLen,
+	}
+}

agent/pkg/oas/oas_generator_test.go

@@ -0,0 +1,45 @@
+package oas
+
+import (
+	"encoding/json"
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"testing"
+	"time"
+)
+
+func TestOASGen(t *testing.T) {
+	gen := GetDefaultOasGeneratorInstance(-1)
+
+	e := new(har.Entry)
+	err := json.Unmarshal([]byte(`{"startedDateTime": "20000101","request": {"url": "https://host/path", "method": "GET"}, "response": {"status": 200}}`), e)
+	if err != nil {
+		panic(err)
+	}
+
+	ews := &EntryWithSource{
+		Destination: "some",
+		Entry:       *e,
+	}
+
+	gen.Start()
+	gen.handleHARWithSource(ews)
+	g, ok := gen.serviceSpecs.Load("some")
+	if !ok {
+		panic("Failed")
+	}
+	sg := g.(*SpecGen)
+	spec, err := sg.GetSpec()
+	if err != nil {
+		panic(err)
+	}
+	specText, _ := json.Marshal(spec)
+	t.Log(string(specText))
+
+	if !gen.IsStarted() {
+		t.Errorf("Should be started")
+	}
+
+	time.Sleep(100 * time.Millisecond)
+
+	gen.Stop()
+}

agent/pkg/oas/specgen.go

@@ -0,0 +1,748 @@
+package oas
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"io/ioutil"
+	"mime"
+	"mime/multipart"
+	"net/textproto"
+	"net/url"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/chanced/openapi"
+	"github.com/google/uuid"
+	"github.com/nav-inc/datetime"
+	"github.com/up9inc/mizu/logger"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
+
+	"time"
+)
+
+const LastSeenTS = "x-last-seen-ts"
+const CountersTotal = "x-counters-total"
+const CountersPerSource = "x-counters-per-source"
+const SampleId = "x-sample-entry"
+
+type EntryWithSource struct {
+	Source      string
+	Destination string
+	Entry       har.Entry
+	Id          string
+}
+
+type reqResp struct { // hello, generics in Go
+	Req  *har.Request
+	Resp *har.Response
+}
+
+type SpecGen struct {
+	MaxExampleLen int // -1 unlimited, 0 and above sets limit
+
+	oas  *openapi.OpenAPI
+	tree *Node
+	lock sync.Mutex
+}
+
+func NewGen(server string) *SpecGen {
+	spec := new(openapi.OpenAPI)
+	spec.Version = "3.1.0"
+
+	info := openapi.Info{Title: server}
+	info.Version = "1.0"
+	spec.Info = &info
+	spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
+
+	spec.Servers = make([]*openapi.Server, 0)
+	spec.Servers = append(spec.Servers, &openapi.Server{URL: server})
+
+	gen := SpecGen{
+		oas:           spec,
+		tree:          new(Node),
+		MaxExampleLen: -1,
+	}
+	return &gen
+}
+
+func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
+	g.oas = oas
+	g.tree = new(Node)
+	for pathStr, pathObj := range oas.Paths.Items {
+		pathSplit := strings.Split(string(pathStr), "/")
+		g.tree.getOrSet(pathSplit, pathObj, "")
+
+		// clean "last entry timestamp" markers from the past
+		for _, pathAndOp := range g.tree.listOps() {
+			delete(pathAndOp.op.Extensions, LastSeenTS)
+		}
+	}
+}
+
+func (g *SpecGen) feedEntry(entryWithSource *EntryWithSource) (string, error) {
+	g.lock.Lock()
+	defer g.lock.Unlock()
+
+	opId, err := g.handlePathObj(entryWithSource)
+	if err != nil {
+		return "", err
+	}
+
+	// NOTE: opId can be empty for some failed entries
+	return opId, err
+}
+
+func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
+	g.lock.Lock()
+	defer g.lock.Unlock()
+
+	g.tree.compact()
+
+	counters := CounterMaps{counterTotal: Counter{}, counterMapTotal: CounterMap{}}
+
+	for _, pathAndOp := range g.tree.listOps() {
+		opObj := pathAndOp.op
+		if opObj.Summary == "" {
+			opObj.Summary = pathAndOp.path
+		}
+
+		err := counters.processOp(opObj)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	err := counters.processOas(g.oas)
+	if err != nil {
+		return nil, err
+	}
+
+	// put paths back from tree into OAS
+	g.oas.Paths = g.tree.listPaths()
+
+	suggestTags(g.oas)
+
+	g.oas.Info.Description = setCounterMsgIfOk(g.oas.Info.Description, &counters.counterTotal)
+
+	// to make a deep copy, no better idea than marshal+unmarshal
+	specText, err := json.MarshalIndent(g.oas, "", "\t")
+	if err != nil {
+		return nil, err
+	}
+
+	spec := new(openapi.OpenAPI)
+	err = json.Unmarshal(specText, spec)
+	if err != nil {
+		return nil, err
+	}
+
+	return spec, err
+}
+
+func suggestTags(oas *openapi.OpenAPI) {
+	paths := getPathsKeys(oas.Paths.Items)
+	sort.Strings(paths) // make it stable in case of multiple candidates
+	for len(paths) > 0 {
+		group := make([]string, 0)
+		group = append(group, paths[0])
+		paths = paths[1:]
+
+		pathsClone := append(paths[:0:0], paths...)
+		for _, path := range pathsClone {
+			if getSimilarPrefix([]string{group[0], path}) != "" {
+				group = append(group, path)
+				paths = deleteFromSlice(paths, path)
+			}
+		}
+
+		common := getSimilarPrefix(group)
+
+		if len(group) > 1 {
+			for _, path := range group {
+				pathObj := oas.Paths.Items[openapi.PathValue(path)]
+				for _, op := range getOps(pathObj) {
+					if op.Tags == nil {
+						op.Tags = make([]string, 0)
+					}
+					// only add tags if not present
+					if len(op.Tags) == 0 {
+						op.Tags = append(op.Tags, common)
+					}
+				}
+			}
+		}
+	}
+}
+
+func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
+	keys := make([]string, len(mymap))
+
+	i := 0
+	for k := range mymap {
+		keys[i] = string(k)
+		i++
+	}
+	return keys
+}
+
+func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error) {
+	entry := entryWithSource.Entry
+	urlParsed, err := url.Parse(entry.Request.URL)
+	if err != nil {
+		return "", err
+	}
+
+	if isExtIgnored(urlParsed.Path) {
+		logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
+		return "", nil
+	}
+
+	if entry.Request.Method == "OPTIONS" {
+		logger.Log.Debugf("Dropped traffic entry due to its method: %s %s", entry.Request.Method, urlParsed.Path)
+		return "", nil
+	}
+
+	ctype := getRespCtype(&entry.Response)
+	if isCtypeIgnored(ctype) {
+		logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
+		return "", nil
+	}
+
+	if entry.Response.Status < 100 {
+		logger.Log.Debugf("Dropped traffic entry due to status<100: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	if entry.Response.Status == 301 || entry.Response.Status == 308 {
+		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	if entry.Response.Status == 502 || entry.Response.Status == 503 || entry.Response.Status == 504 {
+		logger.Log.Debugf("Dropped traffic entry due to temporary server error: %s", entry.StartedDateTime)
+		return "", nil
+	}
+
+	var split []string
+	if urlParsed.RawPath != "" {
+		split = strings.Split(urlParsed.RawPath, "/")
+	} else {
+		split = strings.Split(urlParsed.Path, "/")
+	}
+	node := g.tree.getOrSet(split, new(openapi.PathObj), entryWithSource.Id)
+	opObj, err := handleOpObj(entryWithSource, node.pathObj, g.MaxExampleLen)
+
+	if opObj != nil {
+		return opObj.OperationID, err
+	}
+
+	return "", err
+}
+
+func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj, limit int) (*openapi.Operation, error) {
+	entry := entryWithSource.Entry
+	isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
+	opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
+	if err != nil {
+		return nil, err
+	}
+
+	if !isSuccess && wasMissing {
+		logger.Log.Debugf("Dropped traffic entry due to failed status and no known endpoint at: %s", entry.StartedDateTime)
+		return nil, nil
+	}
+
+	err = handleRequest(&entry.Request, opObj, isSuccess, entryWithSource.Id, limit)
+	if err != nil {
+		return nil, err
+	}
+
+	err = handleResponse(&entry.Response, opObj, isSuccess, entryWithSource.Id, limit)
+	if err != nil {
+		return nil, err
+	}
+
+	err = handleCounters(opObj, isSuccess, entryWithSource)
+	if err != nil {
+		return nil, err
+	}
+
+	setSampleID(&opObj.Extensions, entryWithSource.Id)
+
+	return opObj, nil
+}
+
+func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *EntryWithSource) error {
+	// TODO: if performance around DecodeExtension+SetExtension is bad, store counters as separate maps
+	counter := Counter{}
+	counterMap := CounterMap{}
+	prevTs := 0.0
+	if opObj.Extensions == nil {
+		opObj.Extensions = openapi.Extensions{}
+	} else {
+		if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
+			err := opObj.Extensions.DecodeExtension(CountersTotal, &counter)
+			if err != nil {
+				return err
+			}
+		}
+
+		if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
+			err := opObj.Extensions.DecodeExtension(CountersPerSource, &counterMap)
+			if err != nil {
+				return err
+			}
+		}
+
+		if _, ok := opObj.Extensions.Extension(LastSeenTS); ok {
+			err := opObj.Extensions.DecodeExtension(LastSeenTS, &prevTs)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	var counterPerSource *Counter
+	if existing, ok := counterMap[entryWithSource.Source]; ok {
+		counterPerSource = existing
+	} else {
+		counterPerSource = new(Counter)
+		counterMap[entryWithSource.Source] = counterPerSource
+	}
+
+	started, err := datetime.Parse(entryWithSource.Entry.StartedDateTime, time.UTC)
+	if err != nil {
+		return err
+	}
+
+	ts := float64(started.UnixNano()) / float64(time.Millisecond) / 1000
+	rt := float64(entryWithSource.Entry.Time) / 1000
+
+	dur := 0.0
+	if prevTs != 0 && ts >= prevTs {
+		dur = ts - prevTs
+	}
+
+	counter.addEntry(ts, rt, success, dur)
+	counterPerSource.addEntry(ts, rt, success, dur)
+
+	err = opObj.Extensions.SetExtension(LastSeenTS, ts)
+	if err != nil {
+		return err
+	}
+
+	err = opObj.Extensions.SetExtension(CountersTotal, counter)
+	if err != nil {
+		return err
+	}
+
+	err = opObj.Extensions.SetExtension(CountersPerSource, counterMap)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, sampleId string, limit int) error {
+	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
+	urlParsed, err := url.Parse(req.URL)
+	if err != nil {
+		return err
+	}
+
+	qs := make([]har.NVP, 0)
+	for name, vals := range urlParsed.Query() {
+		for _, val := range vals {
+			qs = append(qs, har.NVP{Name: name, Value: val})
+		}
+	}
+
+	if len(qs) != len(req.QueryString) {
+		logger.Log.Warningf("QStr params in HAR do not match URL: %s", req.URL)
+	}
+
+	qstrGW := nvParams{
+		In:             openapi.InQuery,
+		Pairs:          qs,
+		IsIgnored:      func(name string) bool { return false },
+		GeneralizeName: func(name string) string { return name },
+	}
+	handleNameVals(qstrGW, &opObj.Parameters, false, sampleId)
+
+	hdrGW := nvParams{
+		In:             openapi.InHeader,
+		Pairs:          req.Headers,
+		IsIgnored:      isHeaderIgnored,
+		GeneralizeName: strings.ToLower,
+	}
+	handleNameVals(hdrGW, &opObj.Parameters, true, sampleId)
+
+	if isSuccess {
+		reqBody, err := getRequestBody(req, opObj)
+		if err != nil {
+			return err
+		}
+
+		if reqBody != nil {
+			setSampleID(&reqBody.Extensions, sampleId)
+
+			if req.PostData.Text == "" {
+				reqBody.Required = false
+			} else {
+
+				reqCtype, _ := getReqCtype(req)
+				reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, sampleId, limit)
+				if err != nil {
+					return err
+				}
+
+				_ = reqMedia
+			}
+		}
+	}
+	return nil
+}
+
+func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool, sampleId string, limit int) error {
+	// TODO: we don't support "default" response
+	respObj, err := getResponseObj(resp, opObj, isSuccess)
+	if err != nil {
+		return err
+	}
+
+	setSampleID(&respObj.Extensions, sampleId)
+
+	handleRespHeaders(resp.Headers, respObj, sampleId)
+
+	respCtype := getRespCtype(resp)
+	respContent := respObj.Content
+	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, sampleId, limit)
+	if err != nil {
+		return err
+	}
+	_ = respMedia
+	return nil
+}
+
+func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sampleId string) {
+	visited := map[string]*openapi.HeaderObj{}
+	for _, pair := range reqHeaders {
+		if isHeaderIgnored(pair.Name) {
+			continue
+		}
+
+		nameGeneral := strings.ToLower(pair.Name)
+
+		initHeaders(respObj)
+		objHeaders := respObj.Headers
+		param := findHeaderByName(&respObj.Headers, pair.Name)
+		if param == nil {
+			param = createHeader(openapi.TypeString)
+			objHeaders[nameGeneral] = param
+		}
+		exmp := &param.Examples
+		err := fillParamExample(&exmp, pair.Value)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+		visited[nameGeneral] = param
+
+		setSampleID(&param.Extensions, sampleId)
+	}
+
+	// maintain "required" flag
+	if respObj.Headers != nil {
+		for name, param := range respObj.Headers {
+			paramObj, err := param.ResolveHeader(headerResolver)
+			if err != nil {
+				logger.Log.Warningf("Failed to resolve param: %s", err)
+				continue
+			}
+
+			_, ok := visited[strings.ToLower(name)]
+			if !ok {
+				flag := false
+				paramObj.Required = &flag
+			}
+		}
+	}
+}
+
+func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, sampleId string, limit int) (*openapi.MediaType, error) {
+	content, found := respContent[ctype]
+	if !found {
+		respContent[ctype] = &openapi.MediaType{}
+		content = respContent[ctype]
+	}
+
+	setSampleID(&content.Extensions, sampleId)
+
+	var text string
+	var isBinary bool
+	if reqResp.Req != nil {
+		isBinary, _, text = reqResp.Req.PostData.B64Decoded()
+	} else {
+		isBinary, _, text = reqResp.Resp.Content.B64Decoded()
+	}
+
+	if !isBinary && text != "" {
+		var exampleMsg []byte
+		// try treating it as json
+		anyVal, isJSON := anyJSON(text)
+		if isJSON {
+			// re-marshal with forced indent
+			if msg, err := json.MarshalIndent(anyVal, "", "\t"); err != nil {
+				panic("Failed to re-marshal value, super-strange")
+			} else {
+				exampleMsg = msg
+			}
+		} else {
+			if msg, err := json.Marshal(text); err != nil {
+				return nil, err
+			} else {
+				exampleMsg = msg
+			}
+		}
+
+		if ctype == "application/x-www-form-urlencoded" && reqResp.Req != nil {
+			handleFormDataUrlencoded(text, content)
+		} else if strings.HasPrefix(ctype, "multipart/form-data") && reqResp.Req != nil {
+			_, params := getReqCtype(reqResp.Req)
+			handleFormDataMultipart(text, content, params)
+		}
+
+		if len(exampleMsg) > len(content.Example) && (limit < 0 || len(exampleMsg) <= limit) {
+			content.Example = exampleMsg
+		}
+	}
+
+	return respContent[ctype], nil
+}
+
+func handleFormDataUrlencoded(text string, content *openapi.MediaType) {
+	formData, err := url.ParseQuery(text)
+	if err != nil {
+		logger.Log.Warningf("Could not decode urlencoded: %s", err)
+		return
+	}
+
+	parts := make([]PartWithBody, 0)
+	for name, vals := range formData {
+		for _, val := range vals {
+			part := new(multipart.Part)
+			part.Header = textproto.MIMEHeader{}
+			part.Header.Add("Content-Disposition", "form-data; name=\""+name+"\";")
+			parts = append(parts, PartWithBody{part: part, body: []byte(val)})
+		}
+	}
+	handleFormData(content, parts)
+}
+
+func handleFormData(content *openapi.MediaType, parts []PartWithBody) {
+	hadSchema := true
+	if content.Schema == nil {
+		hadSchema = false // will use it for required flags
+		content.Schema = new(openapi.SchemaObj)
+		content.Schema.Type = openapi.Types{openapi.TypeObject}
+		content.Schema.Properties = openapi.Schemas{}
+	}
+
+	props := &content.Schema.Properties
+	seenNames := map[string]struct{}{} // set equivalent in Go, yikes
+	for _, pwb := range parts {
+		name := pwb.part.FormName()
+		seenNames[name] = struct{}{}
+		existing, found := (*props)[name]
+		if !found {
+			existing = new(openapi.SchemaObj)
+			existing.Type = openapi.Types{openapi.TypeString}
+			(*props)[name] = existing
+
+			ctype := pwb.part.Header.Get("content-type")
+			if ctype != "" {
+				if existing.Keywords == nil {
+					existing.Keywords = map[string]json.RawMessage{}
+				}
+				existing.Keywords["contentMediaType"], _ = json.Marshal(ctype)
+			}
+		}
+
+		addSchemaExample(existing, string(pwb.body))
+	}
+
+	// handle required flag
+	if content.Schema.Required == nil {
+		if !hadSchema {
+			content.Schema.Required = make([]string, 0)
+			for name := range seenNames {
+				content.Schema.Required = append(content.Schema.Required, name)
+			}
+			sort.Strings(content.Schema.Required)
+		} // else it's a known schema with no required fields
+	} else {
+		content.Schema.Required = intersectSliceWithMap(content.Schema.Required, seenNames)
+		sort.Strings(content.Schema.Required)
+	}
+}
+
+type PartWithBody struct {
+	part *multipart.Part
+	body []byte
+}
+
+func handleFormDataMultipart(text string, content *openapi.MediaType, ctypeParams map[string]string) {
+	boundary, ok := ctypeParams["boundary"]
+	if !ok {
+		logger.Log.Errorf("Multipart header has no boundary")
+		return
+	}
+	mpr := multipart.NewReader(strings.NewReader(text), boundary)
+
+	parts := make([]PartWithBody, 0)
+	for {
+		part, err := mpr.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			logger.Log.Errorf("Cannot parse multipart body: %v", err)
+			break
+		}
+		defer part.Close()
+
+		body, err := ioutil.ReadAll(part)
+		if err != nil {
+			logger.Log.Errorf("Error reading multipart Part %s: %v", part.Header, err)
+		}
+
+		parts = append(parts, PartWithBody{part: part, body: body})
+	}
+
+	handleFormData(content, parts)
+}
+
+func getRespCtype(resp *har.Response) string {
+	var ctype string
+	ctype = resp.Content.MimeType
+	for _, hdr := range resp.Headers {
+		if strings.ToLower(hdr.Name) == "content-type" {
+			ctype = hdr.Value
+		}
+	}
+
+	mediaType, _, err := mime.ParseMediaType(ctype)
+	if err != nil {
+		return ""
+	}
+	return mediaType
+}
+
+func getReqCtype(req *har.Request) (ctype string, params map[string]string) {
+	ctype = req.PostData.MimeType
+	for _, hdr := range req.Headers {
+		if strings.ToLower(hdr.Name) == "content-type" {
+			ctype = hdr.Value
+		}
+	}
+
+	if ctype == "" {
+		return "", map[string]string{}
+	}
+
+	mediaType, params, err := mime.ParseMediaType(ctype)
+	if err != nil {
+		logger.Log.Errorf("Cannot parse Content-Type header %q: %v", ctype, err)
+		return "", map[string]string{}
+	}
+	return mediaType, params
+}
+
+func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool) (*openapi.ResponseObj, error) {
+	statusStr := strconv.Itoa(resp.Status)
+
+	var response openapi.Response
+	response, found := opObj.Responses[statusStr]
+	if !found {
+		if opObj.Responses == nil {
+			opObj.Responses = map[string]openapi.Response{}
+		}
+
+		opObj.Responses[statusStr] = &openapi.ResponseObj{Content: map[string]*openapi.MediaType{}}
+		response = opObj.Responses[statusStr]
+	}
+
+	resResponse, err := response.ResolveResponse(responseResolver)
+	if err != nil {
+		return nil, err
+	}
+
+	if isSuccess {
+		resResponse.Description = "Successful call with status " + statusStr
+	} else {
+		resResponse.Description = "Failed call with status " + statusStr
+	}
+	return resResponse, nil
+}
+
+func getRequestBody(req *har.Request, opObj *openapi.Operation) (*openapi.RequestBodyObj, error) {
+	if opObj.RequestBody == nil {
+		// create if there is body in request
+		if req.PostData.Text != "" {
+			opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
+		} else {
+			return nil, nil
+		}
+	}
+
+	reqBody, err := opObj.RequestBody.ResolveRequestBody(reqBodyResolver)
+	if err != nil {
+		return nil, err
+	}
+
+	return reqBody, nil
+}
+
+func getOpObj(pathObj *openapi.PathObj, method string, createIfNone bool) (*openapi.Operation, bool, error) {
+	method = strings.ToLower(method)
+	var op **openapi.Operation
+
+	switch method {
+	case "get":
+		op = &pathObj.Get
+	case "put":
+		op = &pathObj.Put
+	case "post":
+		op = &pathObj.Post
+	case "delete":
+		op = &pathObj.Delete
+	case "options":
+		op = &pathObj.Options
+	case "head":
+		op = &pathObj.Head
+	case "patch":
+		op = &pathObj.Patch
+	case "trace":
+		op = &pathObj.Trace
+	default:
+		return nil, false, errors.New("unsupported HTTP method: " + method)
+	}
+
+	isMissing := false
+	if *op == nil {
+		isMissing = true
+		if createIfNone {
+			*op = &openapi.Operation{Responses: map[string]openapi.Response{}}
+			newUUID := uuid.New().String()
+			(**op).OperationID = newUUID
+		} else {
+			return nil, isMissing, nil
+		}
+	}
+
+	return *op, isMissing, nil
+}

agent/pkg/oas/specgen_test.go

@@ -0,0 +1,267 @@
+package oas
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/chanced/openapi"
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/logger"
+	"github.com/wI2L/jsondiff"
+)
+
+// if started via env, write file into subdir
+func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string {
+	content, err := json.MarshalIndent(spec, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+
+	if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
+		path := "./oas-samples"
+		err := os.MkdirAll(path, 0o755)
+		if err != nil {
+			panic(err)
+		}
+		err = ioutil.WriteFile(path+"/"+label+".json", content, 0644)
+		if err != nil {
+			panic(err)
+		}
+		t.Logf("Written: %s", label)
+	} else {
+		t.Logf("%s", string(content))
+	}
+	return string(content)
+}
+
+func TestEntries(t *testing.T) {
+	//logger.InitLoggerStd(logging.INFO) causes race condition
+	files, err := getFiles("./test_artifacts/")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	gen := NewDefaultOasGenerator(-1)
+	gen.serviceSpecs = new(sync.Map)
+	loadStartingOAS("test_artifacts/catalogue.json", "catalogue", gen.serviceSpecs)
+	loadStartingOAS("test_artifacts/trcc.json", "trcc-api-service", gen.serviceSpecs)
+
+	go func() {
+		for {
+			time.Sleep(1 * time.Second)
+			gen.serviceSpecs.Range(func(key, val interface{}) bool {
+				svc := key.(string)
+				t.Logf("Getting spec for %s", svc)
+				gen := val.(*SpecGen)
+				_, err := gen.GetSpec()
+				if err != nil {
+					t.Error(err)
+				}
+				return true
+			})
+		}
+	}()
+
+	cnt, err := feedEntries(files, true, gen)
+	if err != nil {
+		t.Log(err)
+		t.Fail()
+	}
+
+	svcs := strings.Builder{}
+	gen.serviceSpecs.Range(func(key, val interface{}) bool {
+		gen := val.(*SpecGen)
+		svc := key.(string)
+		svcs.WriteString(svc + ",")
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+			return false
+		}
+
+		err = spec.Validate()
+		if err != nil {
+			specText, _ := json.MarshalIndent(spec, "", "\t")
+			t.Log(string(specText))
+			t.Log(err)
+			t.FailNow()
+		}
+
+		return true
+	})
+
+	gen.serviceSpecs.Range(func(key, val interface{}) bool {
+		svc := key.(string)
+		gen := val.(*SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		outputSpec(svc, spec, t)
+
+		err = spec.Validate()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		return true
+	})
+
+	logger.Log.Infof("Total entries: %d", cnt)
+}
+
+func TestFileSingle(t *testing.T) {
+	gen := NewDefaultOasGenerator(-1)
+	gen.serviceSpecs = new(sync.Map)
+	// loadStartingOAS()
+	file := "test_artifacts/params.har"
+	files := []string{file}
+	cnt, err := feedEntries(files, true, gen)
+	if err != nil {
+		logger.Log.Warning("Failed processing file: " + err.Error())
+		t.Fail()
+	}
+
+	gen.serviceSpecs.Range(func(key, val interface{}) bool {
+		svc := key.(string)
+		gen := val.(*SpecGen)
+		spec, err := gen.GetSpec()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		specText := outputSpec(svc, spec, t)
+
+		err = spec.Validate()
+		if err != nil {
+			t.Log(err)
+			t.FailNow()
+		}
+
+		expected, err := ioutil.ReadFile(file + ".spec.json")
+		if err != nil {
+			t.Errorf(err.Error())
+			t.FailNow()
+		}
+
+		patFloatPrecision := regexp.MustCompile(`(\d+\.\d{1,2})(\d*)`)
+
+		expected = []byte(patUuid4.ReplaceAllString(string(expected), "<UUID4>"))
+		specText = patUuid4.ReplaceAllString(specText, "<UUID4>")
+		expected = []byte(patFloatPrecision.ReplaceAllString(string(expected), "$1"))
+		specText = patFloatPrecision.ReplaceAllString(specText, "$1")
+
+		diff, err := jsondiff.CompareJSON(expected, []byte(specText))
+		if err != nil {
+			t.Errorf(err.Error())
+			t.FailNow()
+		}
+
+		if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
+			err = ioutil.WriteFile(file+".spec.json", []byte(specText), 0644)
+			if err != nil {
+				panic(err)
+			}
+		}
+
+		if len(diff) > 0 {
+			t.Errorf("Generated spec does not match expected:\n%s", diff.String())
+		}
+
+		return true
+	})
+
+	logger.Log.Infof("Processed entries: %d", cnt)
+}
+
+func loadStartingOAS(file string, label string, specs *sync.Map) {
+	fd, err := os.Open(file)
+	if err != nil {
+		panic(err)
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		panic(err)
+	}
+
+	var doc *openapi.OpenAPI
+	err = json.Unmarshal(data, &doc)
+	if err != nil {
+		panic(err)
+	}
+
+	gen := NewGen(label)
+	gen.StartFromSpec(doc)
+
+	specs.Store(label, gen)
+}
+
+func TestEntriesNegative(t *testing.T) {
+	gen := NewDefaultOasGenerator(-1)
+	gen.serviceSpecs = new(sync.Map)
+	files := []string{"invalid"}
+	_, err := feedEntries(files, false, gen)
+	if err == nil {
+		t.Logf("Should have failed")
+		t.Fail()
+	}
+}
+
+func TestEntriesPositive(t *testing.T) {
+	gen := NewDefaultOasGenerator(-1)
+	gen.serviceSpecs = new(sync.Map)
+	files := []string{"test_artifacts/params.har"}
+	_, err := feedEntries(files, false, gen)
+	if err != nil {
+		t.Logf("Failed")
+		t.Fail()
+	}
+}
+
+func TestLoadValidHAR(t *testing.T) {
+	inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}`
+	var entry *har.Entry
+	var err = json.Unmarshal([]byte(inp), &entry)
+	if err != nil {
+		t.Logf("Failed to decode entry: %s", err)
+		t.FailNow() // demonstrates the problem of `martian` HAR library
+	}
+}
+
+func TestLoadValid3_1(t *testing.T) {
+	fd, err := os.Open("test_artifacts/catalogue.json")
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	defer fd.Close()
+
+	data, err := ioutil.ReadAll(fd)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+
+	var oas openapi.OpenAPI
+	err = json.Unmarshal(data, &oas)
+	if err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+}

agent/pkg/oas/test_artifacts/catalogue.json

@@ -0,0 +1,51 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "Preloaded",
+    "version": "0.1",
+    "description": "Test file for loading pre-existing OAS"
+  },
+  "paths": {
+    "/catalogue/{id}": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ],
+      "get": {
+        "parameters": [        {
+          "name": "non-required-header",
+          "in": "header",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+        ]
+      }
+    },
+    "/catalogue/{id}/details": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "style": "simple",
+          "required": true,
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ]
+    }
+  }
+}

agent/pkg/oas/test_artifacts/example.ldjson

@@ -0,0 +1,13 @@
+{"messageType": "http", "_source": "some-source", ",firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78415179Z", "time": 13, "request": {"method": "GET", "url": "http://catalogue/catalogue/size?tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "x-some", "value": "demo val"},{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [{"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "22"}], "content": {"size": 22, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInNpemUiOjl9", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 22}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 13}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.784918698Z", "time": 19, "request": {"method": "GET", "url": "http://catalogue/catalogue?page=1&size=6&tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "page", "value": "1"}, {"name": "size", "value": "6"}, {"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "1927"}], "content": {"size": 1927, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19LHsiaWQiOiI4MzdhYjE0MS0zOTllLTRjMWYtOWFiYy1iYWNlNDAyOTZiYWMiLCJuYW1lIjoiQ2F0IHNvY2tzIiwiZGVzY3JpcHRpb24iOiJjb25zZXF1YXQgYW1ldCBjdXBpZGF0YXQgbWluaW0gbGFib3J1bSB0ZW1wb3IgZWxpdCBleCBjb25zZXF1YXQgaW4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2NrczIuanBnIl0sInByaWNlIjoxNSwiY291bnQiOjE3NSwidGFnIjpbImJyb3duIiwiZm9ybWFsIiwiZ3JlZW4iXX1dCg==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1927}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 19}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78418182Z", "time": 7, "request": {"method": "GET", "url": "http://catalogue/tags", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "107"}], "content": {"size": 107, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInRhZ3MiOlsiYnJvd24iLCJnZWVrIiwiZm9ybWFsIiwiYmx1ZSIsInNraW4iLCJyZWQiLCJhY3Rpb24iLCJzcG9ydCIsImJsYWNrIiwibWFnaWMiLCJncmVlbiJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 107}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 7}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:18.131501482Z", "time": 5, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}, {"name": "x-some", "value": "demoval"}], ",queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 5}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:18.379836908Z", "time": 14, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 14}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.920540124Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/808a2de1-1aaa-4c25-a9b9-6612e8f29a38", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "275"}], "content": {"size": 275, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NzM4LCJkZXNjcmlwdGlvbiI6IkEgbWF0dXJlIHNvY2ssIGNyb3NzZWQsIHdpdGggYW4gYWlyIG9mIG5vbmNoYWxhbmNlLiIsImlkIjoiODA4YTJkZTEtMWFhYS00YzI1LWE5YjktNjYxMmU4ZjI5YTM4IiwiaW1hZ2VVcmwiOlsiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMS5qcGVnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMi5qcGVnIl0sIm5hbWUiOiJDcm9zc2VkIiwicHJpY2UiOjE3LjMyLCJ0YWciOlsiYmx1ZSIsInJlZCIsImFjdGlvbiIsImZvcm1hbCJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 275}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.921609501Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?sort=id&size=3&tags=blue", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "3"}, {"name": "tags", "value": "blue"}, {"name": "sort", "value": "id"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "789"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}], "content": {"size": 789, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJuYW1lIjoiQ29sb3VyZnVsIiwiZGVzY3JpcHRpb24iOiJwcm9pZGVudCBvY2NhZWNhdCBpcnVyZSBldCBleGNlcHRldXIgbGFib3JlIG1pbmltIG5pc2kgYW1ldCBpcnVyZSIsImltYWdlVXJsIjpbIi9jYXRhbG9ndWUvaW1hZ2VzL2NvbG91cmZ1bF9zb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIl0sInByaWNlIjoxOCwiY291bnQiOjQzOCwidGFnIjpbImJsdWUiXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiYmx1ZSJdfV0K", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 789}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:22.923197848Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:23.175549218Z", "time": 26, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 26}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.239777333Z", "time": 10, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Transfer-Encoding", "value": "chunked"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "X-Application-Context", "value": "carts:80"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 10}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.725866772Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?size=5", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "5"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "1643"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}], "content": {"size": 1643, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19XQo=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1643}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
+{"_id": "", "startedDateTime": "2021-07-26T11:14:25.729303217Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}

agent/pkg/oas/test_artifacts/params.har

@@ -0,0 +1,790 @@
+{
+  "log": {
+    "version": "1.2",
+    "creator": {
+      "name": "mitmproxy har_dump",
+      "version": "0.1",
+      "comment": "mitmproxy version mitmproxy 4.0.4"
+    },
+    "entries": [
+      {
+        "startedDateTime": "2019-09-06T06:14:43.864529+00:00",
+        "time": 111,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/e21f7112-3d3b-4632-9da3-a4af2e0e9166/sub1",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "headersSize": 1542,
+          "bodySize": 0,
+          "queryString": []
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "mimeType": "text/html",
+            "text": "",
+            "size": 0
+          },
+          "redirectURL": "",
+          "headersSize": 245,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 22,
+          "receive": 2,
+          "wait": 87,
+          "connect": -1,
+          "ssl": -1
+        },
+        "serverIPAddress": "54.210.29.33"
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:18.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a/sub2",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:19.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a;mparam=matrixparam",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:20.047122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/appears-once",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:20.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/appears-twice",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/appears-twice",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:20.747122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/form-urlencoded",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "Content-Type",
+              "value": "application/x-www-form-urlencoded"
+            }
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "agent-id=ade&callback-url=&token=sometoken"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/form-urlencoded",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "Content-Type",
+              "value": "application/x-www-form-urlencoded"
+            }
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "agent-id=ade&callback-url=&token=sometoken-second-val&optional=another"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:22.747122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/form-multipart",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "Content-Type",
+              "value": "multipart/form-data; boundary=BOUNDARY"
+            }
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 62,
+            "mimeType": "",
+            "text": "{}"
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 62
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/body-optional",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/body-optional",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "Content-Type",
+              "value": "application/json"
+            }
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "{\"key\", \"val\"}"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/body-optional",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
+        "time": 1,
+        "request": {
+          "method": "POST",
+          "url": "https://httpbin.org/body-required",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": "body exists"
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:22.000000+00:00",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/param-patterns/prefix-gibberish-fine/234324",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:22.000001+00:00",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/param-patterns/prefix-gibberish-sfdlasdfkadf87sd93284q24r/1",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+     {
+        "startedDateTime": "2019-09-06T06:16:22.000002+00:00",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/param-patterns/prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf/static",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+     {
+        "startedDateTime": "2019-09-06T06:16:22.000003+00:00",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/param-patterns/prefix-gibberish-4jk5l2345h2452l4352435jlk45",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+     {
+        "startedDateTime": "2019-09-06T06:16:22.000004+00:00",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/param-patterns/prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:22.000002+00:00",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/param-patterns/prefix-gibberish-afterwards/23421",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      }
+    ]
+  }
+}

agent/pkg/oas/test_artifacts/params.har.spec.json

@@ -0,0 +1,897 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "https://httpbin.org",
+    "description": "Mizu observed 19 entries (0 failed), at 0.10 hits/s, average response time is 0.17 seconds",
+    "version": "1.0"
+  },
+  "servers": [
+    {
+      "url": "https://httpbin.org"
+    }
+  ],
+  "paths": {
+    "/appears-once": {
+      "get": {
+        "summary": "/appears-once",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000004"
+              }
+            },
+            "x-sample-entry": "000000000000000000000004"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750580.04,
+            "lastSeen": 1567750580.04,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750580.04,
+          "lastSeen": 1567750580.04,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750580.04,
+        "x-sample-entry": "000000000000000000000004"
+      }
+    },
+    "/appears-twice": {
+      "get": {
+        "summary": "/appears-twice",
+        "description": "Mizu observed 2 entries (0 failed), at 0.50 hits/s, average response time is 0.63 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000006"
+              }
+            },
+            "x-sample-entry": "000000000000000000000006"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750580.74,
+            "lastSeen": 1567750581.74,
+            "sumRT": 1.26,
+            "sumDuration": 1
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750580.74,
+          "lastSeen": 1567750581.74,
+          "sumRT": 1.26,
+          "sumDuration": 1
+        },
+        "x-last-seen-ts": 1567750581.74,
+        "x-sample-entry": "000000000000000000000006"
+      }
+    },
+    "/body-optional": {
+      "post": {
+        "summary": "/body-optional",
+        "description": "Mizu observed 3 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000012"
+              }
+            },
+            "x-sample-entry": "000000000000000000000012"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 3,
+            "failures": 0,
+            "firstSeen": 1567750581.74,
+            "lastSeen": 1567750581.75,
+            "sumRT": 0.00,
+            "sumDuration": 0.01
+          }
+        },
+        "x-counters-total": {
+          "entries": 3,
+          "failures": 0,
+          "firstSeen": 1567750581.74,
+          "lastSeen": 1567750581.75,
+          "sumRT": 0.00,
+          "sumDuration": 0.01
+        },
+        "x-last-seen-ts": 1567750581.75,
+        "x-sample-entry": "000000000000000000000012",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "application/json": {
+              "example": "{\"key\", \"val\"}",
+              "x-sample-entry": "000000000000000000000011"
+            }
+          },
+          "x-sample-entry": "000000000000000000000012"
+        }
+      }
+    },
+    "/body-required": {
+      "post": {
+        "summary": "/body-required",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000013"
+              }
+            },
+            "x-sample-entry": "000000000000000000000013"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750581.75,
+            "lastSeen": 1567750581.75,
+            "sumRT": 0.00,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750581.75,
+          "lastSeen": 1567750581.75,
+          "sumRT": 0.00,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750581.75,
+        "x-sample-entry": "000000000000000000000013",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "": {
+              "example": "body exists",
+              "x-sample-entry": "000000000000000000000013"
+            }
+          },
+          "required": true,
+          "x-sample-entry": "000000000000000000000013"
+        }
+      }
+    },
+    "/form-multipart": {
+      "post": {
+        "summary": "/form-multipart",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "example": {},
+                "x-sample-entry": "000000000000000000000009"
+              }
+            },
+            "x-sample-entry": "000000000000000000000009"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.74,
+            "lastSeen": 1567750582.74,
+            "sumRT": 0.00,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.74,
+          "lastSeen": 1567750582.74,
+          "sumRT": 0.00,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.74,
+        "x-sample-entry": "000000000000000000000009",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "multipart/form-data": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "file",
+                  "path"
+                ],
+                "properties": {
+                  "file": {
+                    "type": "string",
+                    "contentMediaType": "application/json",
+                    "examples": [
+                      "{\"functions\": 123}"
+                    ]
+                  },
+                  "path": {
+                    "type": "string",
+                    "examples": [
+                      "/content/components"
+                    ]
+                  }
+                }
+              },
+              "example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n",
+              "x-sample-entry": "000000000000000000000009"
+            }
+          },
+          "required": true,
+          "x-sample-entry": "000000000000000000000009"
+        }
+      }
+    },
+    "/form-urlencoded": {
+      "post": {
+        "summary": "/form-urlencoded",
+        "description": "Mizu observed 2 entries (0 failed), at 0.50 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000008"
+              }
+            },
+            "x-sample-entry": "000000000000000000000008"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750580.74,
+            "lastSeen": 1567750581.74,
+            "sumRT": 0.00,
+            "sumDuration": 1
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750580.74,
+          "lastSeen": 1567750581.74,
+          "sumRT": 0.00,
+          "sumDuration": 1
+        },
+        "x-last-seen-ts": 1567750581.74,
+        "x-sample-entry": "000000000000000000000008",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "application/x-www-form-urlencoded": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "agent-id",
+                  "callback-url",
+                  "token"
+                ],
+                "properties": {
+                  "agent-id": {
+                    "type": "string",
+                    "examples": [
+                      "ade"
+                    ]
+                  },
+                  "callback-url": {
+                    "type": "string",
+                    "examples": [
+                      ""
+                    ]
+                  },
+                  "optional": {
+                    "type": "string",
+                    "examples": [
+                      "another"
+                    ]
+                  },
+                  "token": {
+                    "type": "string",
+                    "examples": [
+                      "sometoken",
+                      "sometoken-second-val"
+                    ]
+                  }
+                }
+              },
+              "example": "agent-id=ade\u0026callback-url=\u0026token=sometoken-second-val\u0026optional=another",
+              "x-sample-entry": "000000000000000000000008"
+            }
+          },
+          "required": true,
+          "x-sample-entry": "000000000000000000000008"
+        }
+      }
+    },
+    "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000014"
+              }
+            },
+            "x-sample-entry": "000000000000000000000014"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582,
+            "lastSeen": 1567750582,
+            "sumRT": 0.00,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582,
+          "lastSeen": 1567750582,
+          "sumRT": 0.00,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582,
+        "x-sample-entry": "000000000000000000000014"
+      },
+      "parameters": [
+        {
+          "name": "prefixgibberishfineId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "234324"
+            }
+          },
+          "x-sample-entry": "000000000000000000000014"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}",
+        "description": "Mizu observed 2 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000018"
+              }
+            },
+            "x-sample-entry": "000000000000000000000018"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750582.00,
+            "lastSeen": 1567750582.00,
+            "sumRT": 0.00,
+            "sumDuration": 9.53e-7
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750582.00,
+          "lastSeen": 1567750582.00,
+          "sumRT": 0.00,
+          "sumDuration": 9.53e-7
+        },
+        "x-last-seen-ts": 1567750582.00,
+        "x-sample-entry": "000000000000000000000018"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/1": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/1",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000015"
+              }
+            },
+            "x-sample-entry": "000000000000000000000015"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.00,
+            "lastSeen": 1567750582.00,
+            "sumRT": 0.00,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.00,
+          "lastSeen": 1567750582.00,
+          "sumRT": 0.00,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.00,
+        "x-sample-entry": "000000000000000000000015"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/static": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/static",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000016"
+              }
+            },
+            "x-sample-entry": "000000000000000000000016"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.00,
+            "lastSeen": 1567750582.00,
+            "sumRT": 0.00,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.00,
+          "lastSeen": 1567750582.00,
+          "sumRT": 0.00,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.00,
+        "x-sample-entry": "000000000000000000000016"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/{param1}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/{param1}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000019"
+              }
+            },
+            "x-sample-entry": "000000000000000000000019"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.00,
+            "lastSeen": 1567750582.00,
+            "sumRT": 0.00,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.00,
+          "lastSeen": 1567750582.00,
+          "sumRT": 0.00,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.00,
+        "x-sample-entry": "000000000000000000000019"
+      },
+      "parameters": [
+        {
+          "name": "param1",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "23421"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        },
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/{Id}": {
+      "get": {
+        "summary": "/{Id}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000003"
+              }
+            },
+            "x-sample-entry": "000000000000000000000003"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750579.74,
+            "lastSeen": 1567750579.74,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750579.74,
+          "lastSeen": 1567750579.74,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750579.74,
+        "x-sample-entry": "000000000000000000000003"
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "<UUID4>"
+            },
+            "example #1": {
+              "value": "<UUID4>"
+            }
+          },
+          "x-sample-entry": "000000000000000000000003"
+        }
+      ]
+    },
+    "/{Id}/sub1": {
+      "get": {
+        "summary": "/{Id}/sub1",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.11 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "text/html": {
+                "x-sample-entry": "000000000000000000000001"
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750483.86,
+            "lastSeen": 1567750483.86,
+            "sumRT": 0.11,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750483.86,
+          "lastSeen": 1567750483.86,
+          "sumRT": 0.11,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750483.86,
+        "x-sample-entry": "000000000000000000000001"
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "<UUID4>"
+            },
+            "example #1": {
+              "value": "<UUID4>"
+            }
+          },
+          "x-sample-entry": "000000000000000000000003"
+        }
+      ]
+    },
+    "/{Id}/sub2": {
+      "get": {
+        "summary": "/{Id}/sub2",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000002"
+              }
+            },
+            "x-sample-entry": "000000000000000000000002"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750578.74,
+            "lastSeen": 1567750578.74,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750578.74,
+          "lastSeen": 1567750578.74,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750578.74,
+        "x-sample-entry": "000000000000000000000002"
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "<UUID4>"
+            },
+            "example #1": {
+              "value": "<UUID4>"
+            }
+          },
+          "x-sample-entry": "000000000000000000000003"
+        }
+      ]
+    }
+  },
+  "x-counters-per-source": {
+    "": {
+      "entries": 19,
+      "failures": 0,
+      "firstSeen": 1567750483.86,
+      "lastSeen": 1567750582.74,
+      "sumRT": 3.27,
+      "sumDuration": 2.01
+    }
+  },
+  "x-counters-total": {
+    "entries": 19,
+    "failures": 0,
+    "firstSeen": 1567750483.86,
+    "lastSeen": 1567750582.74,
+    "sumRT": 3.27,
+    "sumDuration": 2.01
+  }
+}

agent/pkg/oas/test_artifacts/trcc.json

@@ -0,0 +1,50 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "Preloaded TRCC",
+    "version": "0.1",
+    "description": "Test file for loading pre-existing OAS"
+  },
+  "paths": {
+    "/models/{id}": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": ".+(_|-|\\.).+"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ]
+    },
+    "/models/{id}/{id2}": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": ".+(_|-|\\.).+"
+          },
+          "example": "some-uuid-maybe"
+        },
+        {
+          "name": "id2",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "\\d+"
+          }
+        }
+      ]
+    }
+  }
+}

agent/pkg/oas/tree.go

@@ -0,0 +1,313 @@
+package oas
+
+import (
+	"encoding/json"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/chanced/openapi"
+	"github.com/up9inc/mizu/logger"
+)
+
+type NodePath = []string
+
+type Node struct {
+	constant  *string
+	pathParam *openapi.ParameterObj
+	pathObj   *openapi.PathObj
+	parent    *Node
+	children  []*Node
+}
+
+func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj, sampleId string) (node *Node) {
+	if existingPathObj == nil {
+		panic("Invalid function call")
+	}
+
+	pathChunk := path[0]
+	potentialMatrix := strings.SplitN(pathChunk, ";", 2)
+	if len(potentialMatrix) > 1 {
+		pathChunk = potentialMatrix[0]
+		logger.Log.Warningf("URI matrix params are not supported: %s", potentialMatrix[1])
+	}
+
+	chunkIsParam := strings.HasPrefix(pathChunk, "{") && strings.HasSuffix(pathChunk, "}")
+	pathChunk, err := url.PathUnescape(pathChunk)
+	if err != nil {
+		logger.Log.Warningf("URI segment is not correctly encoded: %s", pathChunk)
+		// any side effects on continuing?
+	}
+
+	chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
+
+	var paramObj *openapi.ParameterObj
+	if chunkIsParam && existingPathObj != nil && existingPathObj.Parameters != nil {
+		_, paramObj = findParamByName(existingPathObj.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
+	}
+
+	if paramObj == nil {
+		node = n.searchInConstants(pathChunk)
+	}
+
+	if node == nil && pathChunk != "" {
+		node = n.searchInParams(paramObj, pathChunk, chunkIsGibberish)
+	}
+
+	// still no node found, should create it
+	if node == nil {
+		node = new(Node)
+		node.parent = n
+		n.children = append(n.children, node)
+
+		if paramObj != nil {
+			node.pathParam = paramObj
+		} else if chunkIsGibberish {
+			newParam := n.createParam()
+			node.pathParam = newParam
+		} else {
+			node.constant = &pathChunk
+		}
+	}
+
+	if node.pathParam != nil {
+		setSampleID(&node.pathParam.Extensions, sampleId)
+	}
+
+	// add example if it's a gibberish chunk
+	if node.pathParam != nil && !chunkIsParam {
+		exmp := &node.pathParam.Examples
+		err := fillParamExample(&exmp, pathChunk)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+
+		if len(*exmp) >= 3 && node.pathParam.Schema.Pattern == nil { // is it enough to decide on 2 samples?
+			node.pathParam.Schema.Pattern = getPatternFromExamples(exmp)
+		}
+	}
+
+	// TODO: eat up trailing slash, in a smart way: node.pathObj!=nil && path[1]==""
+	if len(path) > 1 {
+		return node.getOrSet(path[1:], existingPathObj, sampleId)
+	} else if node.pathObj == nil {
+		node.pathObj = existingPathObj
+	}
+
+	return node
+}
+
+func getPatternFromExamples(exmp *openapi.Examples) *openapi.Regexp {
+	allInts := true
+	strs := make([]string, 0)
+	for _, example := range *exmp {
+		exampleObj, err := example.ResolveExample(exampleResolver)
+		if err != nil {
+			continue
+		}
+
+		var value string
+		err = json.Unmarshal(exampleObj.Value, &value)
+		if err != nil {
+			logger.Log.Warningf("Failed decoding parameter example into string: %s", err)
+			continue
+		}
+		strs = append(strs, value)
+
+		if _, err := strconv.Atoi(value); err != nil {
+			allInts = false
+		}
+	}
+
+	if allInts {
+		re := new(openapi.Regexp)
+		re.Regexp = regexp.MustCompile(`\d+`)
+		return re
+	} else {
+		prefix := longestCommonXfixStr(strs, true)
+		suffix := longestCommonXfixStr(strs, false)
+
+		pat := ""
+		separators := "-._/:|*,+" // TODO: we could also cut prefix till the last separator
+		if len(prefix) > 0 && strings.Contains(separators, string(prefix[len(prefix)-1])) {
+			pat = "^" + regexp.QuoteMeta(prefix)
+		}
+
+		pat += ".+"
+
+		if len(suffix) > 0 && strings.Contains(separators, string(suffix[0])) {
+			pat += regexp.QuoteMeta(suffix) + "$"
+		}
+
+		if pat != ".+" {
+			re := new(openapi.Regexp)
+			re.Regexp = regexp.MustCompile(pat)
+			return re
+		}
+	}
+	return nil
+}
+
+func (n *Node) createParam() *openapi.ParameterObj {
+	name := "param"
+
+	if n.constant != nil { // the node is already a param
+		// REST assumption, not always correct
+		if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
+			name = *n.constant
+			name = name[:len(name)-2] + "Id"
+		} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
+			name = *n.constant
+			name = name[:len(name)-1] + "Id"
+		} else {
+			name = *n.constant + "Id"
+		}
+
+		name = cleanStr(name, isAlNumRune)
+		if !isAlphaRune(rune(name[0])) {
+			name = "_" + name
+		}
+	}
+
+	newParam := createSimpleParam(name, "path", "string")
+	x := n.countParentParams()
+	if x > 0 {
+		newParam.Name = newParam.Name + strconv.Itoa(x)
+	}
+
+	return newParam
+}
+
+func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunk string, chunkIsGibberish bool) *Node {
+	// look among params
+	for _, subnode := range n.children {
+		if subnode.constant != nil {
+			continue
+		}
+
+		if paramObj != nil {
+			// TODO: mergeParam(subnode.pathParam, paramObj)
+			return subnode
+		} else if subnode.pathParam.Schema.Pattern != nil { // it has defined param pattern, have to respect it
+			// TODO: and not in exceptions
+			if subnode.pathParam.Schema.Pattern.Match([]byte(chunk)) {
+				return subnode
+			} else if chunkIsGibberish {
+				// TODO: what to do if gibberish chunk does not match the pattern and not in exceptions?
+				return nil
+			} else {
+				return nil
+			}
+		} else if chunkIsGibberish {
+			return subnode
+		}
+
+	}
+	return nil
+}
+
+func (n *Node) searchInConstants(pathChunk string) *Node {
+	// look among constants
+	for _, subnode := range n.children {
+		if subnode.constant == nil {
+			continue
+		}
+
+		if *subnode.constant == pathChunk {
+			return subnode
+		}
+	}
+	return nil
+}
+
+func (n *Node) compact() {
+	// TODO
+}
+
+func (n *Node) listPaths() *openapi.Paths {
+	paths := &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
+
+	var strChunk string
+	if n.constant != nil {
+		strChunk = *n.constant
+	} else if n.pathParam != nil {
+		strChunk = "{" + n.pathParam.Name + "}"
+	} // else -> this is the root node
+
+	// add self
+	if n.pathObj != nil {
+		fillPathParams(n, n.pathObj)
+		paths.Items[openapi.PathValue(strChunk)] = n.pathObj
+	}
+
+	// recurse into children
+	for _, child := range n.children {
+		subPaths := child.listPaths()
+		for path, pathObj := range subPaths.Items {
+			var concat string
+			if n.parent == nil {
+				concat = string(path)
+			} else {
+				concat = strChunk + "/" + string(path)
+			}
+			paths.Items[openapi.PathValue(concat)] = pathObj
+		}
+	}
+
+	return paths
+}
+
+func fillPathParams(n *Node, pathObj *openapi.PathObj) {
+	// collect all path parameters from parent hierarchy
+	node := n
+	for {
+		if node.pathParam != nil {
+			initParams(&pathObj.Parameters)
+
+			idx, paramObj := findParamByName(pathObj.Parameters, openapi.InPath, node.pathParam.Name)
+			if paramObj == nil {
+				appended := append(*pathObj.Parameters, node.pathParam)
+				pathObj.Parameters = &appended
+			} else {
+				(*pathObj.Parameters)[idx] = paramObj
+			}
+		}
+
+		node = node.parent
+		if node == nil {
+			break
+		}
+	}
+}
+
+type PathAndOp struct {
+	path string
+	op   *openapi.Operation
+}
+
+func (n *Node) listOps() []PathAndOp {
+	res := make([]PathAndOp, 0)
+	for path, pathObj := range n.listPaths().Items {
+		for _, op := range getOps(pathObj) {
+			res = append(res, PathAndOp{path: string(path), op: op})
+		}
+	}
+	return res
+}
+
+func (n *Node) countParentParams() int {
+	res := 0
+	node := n
+	for {
+		if node.pathParam != nil {
+			res++
+		}
+
+		if node.parent == nil {
+			break
+		}
+		node = node.parent
+	}
+	return res
+}

agent/pkg/oas/tree_test.go

@@ -0,0 +1,39 @@
+package oas
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/chanced/openapi"
+)
+
+func TestTree(t *testing.T) {
+	testCases := []struct {
+		inp       string
+		numParams int
+		label     string
+	}{
+		{"/", 0, ""},
+		{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation", 1, "vendor,init,conversation"},
+		{"/v1.0.0/config/launcher/sp_nKNHCzsN/{f34efcae-6583-11eb-908a-00b0fcb9d4f6}/vendor,init,conversation", 0, "vendor,init,conversation"},
+		{"/getSvgs/size/small/brand/SFLY/layoutId/170943/layoutVersion/1/sizeId/742/surface/0/isLandscape/true/childSkus/%7B%7D", 1, "{}"},
+	}
+
+	tree := new(Node)
+	for i, tc := range testCases {
+		split := strings.Split(tc.inp, "/")
+		pathObj := new(openapi.PathObj)
+		node := tree.getOrSet(split, pathObj, fmt.Sprintf("%024d", i))
+
+		fillPathParams(node, pathObj)
+
+		if node.constant != nil && *node.constant != tc.label {
+			t.Errorf("Constant does not match: %s != %s", *node.constant, tc.label)
+		}
+
+		if tc.numParams > 0 && (pathObj.Parameters == nil || len(*pathObj.Parameters) < tc.numParams) {
+			t.Errorf("Wrong num of params, expected: %d", tc.numParams)
+		}
+	}
+}

agent/pkg/oas/utils.go

@@ -0,0 +1,490 @@
+package oas
+
+import (
+	"encoding/json"
+	"errors"
+	"strconv"
+	"strings"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
+
+	"github.com/chanced/openapi"
+	"github.com/up9inc/mizu/logger"
+)
+
+func exampleResolver(ref string) (*openapi.ExampleObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func responseResolver(ref string) (*openapi.ResponseObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func reqBodyResolver(ref string) (*openapi.RequestBodyObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func paramResolver(ref string) (*openapi.ParameterObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func headerResolver(ref string) (*openapi.HeaderObj, error) {
+	return nil, errors.New("JSON references are not supported at the moment: " + ref)
+}
+
+func initParams(obj **openapi.ParameterList) {
+	if *obj == nil {
+		var params openapi.ParameterList = make([]openapi.Parameter, 0)
+		*obj = &params
+	}
+}
+
+func initHeaders(respObj *openapi.ResponseObj) {
+	if respObj.Headers == nil {
+		var created openapi.Headers = map[string]openapi.Header{}
+		respObj.Headers = created
+	}
+}
+
+func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *openapi.ParameterObj {
+	if name == "" {
+		panic("Cannot create parameter with empty name")
+	}
+	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
+	schema := new(openapi.SchemaObj)
+	schema.Type = openapi.Types{ptype}
+
+	style := openapi.StyleSimple
+	if in == openapi.InQuery {
+		style = openapi.StyleForm
+	}
+
+	newParam := openapi.ParameterObj{
+		Name:     name,
+		In:       in,
+		Style:    string(style),
+		Examples: map[string]openapi.Example{},
+		Schema:   schema,
+		Required: &required,
+	}
+	return &newParam
+}
+
+func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (idx int, pathParam *openapi.ParameterObj) {
+	caseInsensitive := in == openapi.InHeader
+	for i, param := range *params {
+		idx = i
+		paramObj, err := param.ResolveParameter(paramResolver)
+		if err != nil {
+			logger.Log.Warningf("Failed to resolve reference: %s", err)
+			continue
+		}
+
+		if paramObj.In != in {
+			continue
+		}
+
+		if paramObj.Name == name || (caseInsensitive && strings.EqualFold(paramObj.Name, name)) {
+			pathParam = paramObj
+			break
+		}
+	}
+
+	return idx, pathParam
+}
+
+func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj {
+	for hname, param := range *headers {
+		hdrObj, err := param.ResolveHeader(headerResolver)
+		if err != nil {
+			logger.Log.Warningf("Failed to resolve reference: %s", err)
+			continue
+		}
+
+		if strings.EqualFold(hname, name) {
+			return hdrObj
+		}
+	}
+	return nil
+}
+
+type nvParams struct {
+	In             openapi.In
+	Pairs          []har.NVP
+	IsIgnored      func(name string) bool
+	GeneralizeName func(name string) string
+}
+
+func handleNameVals(gw nvParams, params **openapi.ParameterList, checkIgnore bool, sampleId string) {
+	visited := map[string]*openapi.ParameterObj{}
+	for _, pair := range gw.Pairs {
+		if (checkIgnore && gw.IsIgnored(pair.Name)) || pair.Name == "" {
+			continue
+		}
+
+		nameGeneral := gw.GeneralizeName(pair.Name)
+
+		initParams(params)
+		_, param := findParamByName(*params, gw.In, pair.Name)
+		if param == nil {
+			param = createSimpleParam(nameGeneral, gw.In, openapi.TypeString)
+			appended := append(**params, param)
+			*params = &appended
+		}
+		exmp := &param.Examples
+		err := fillParamExample(&exmp, pair.Value)
+		if err != nil {
+			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
+		}
+		visited[nameGeneral] = param
+
+		setSampleID(&param.Extensions, sampleId)
+	}
+
+	// maintain "required" flag
+	if *params != nil {
+		for _, param := range **params {
+			paramObj, err := param.ResolveParameter(paramResolver)
+			if err != nil {
+				logger.Log.Warningf("Failed to resolve param: %s", err)
+				continue
+			}
+			if paramObj.In != gw.In {
+				continue
+			}
+
+			_, ok := visited[strings.ToLower(paramObj.Name)]
+			if !ok {
+				flag := false
+				paramObj.Required = &flag
+			}
+		}
+	}
+}
+
+func createHeader(ptype openapi.SchemaType) *openapi.HeaderObj {
+	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
+	schema := new(openapi.SchemaObj)
+	schema.Type = make(openapi.Types, 0)
+	schema.Type = append(schema.Type, ptype)
+
+	style := openapi.StyleSimple
+	newParam := openapi.HeaderObj{
+		Style:    string(style),
+		Examples: map[string]openapi.Example{},
+		Schema:   schema,
+		Required: &required,
+	}
+	return &newParam
+}
+
+func fillParamExample(param **openapi.Examples, exampleValue string) error {
+	if **param == nil {
+		**param = map[string]openapi.Example{}
+	}
+
+	cnt := 0
+	for _, example := range **param {
+		cnt++
+		exampleObj, err := example.ResolveExample(exampleResolver)
+		if err != nil {
+			continue
+		}
+
+		var value string
+		err = json.Unmarshal(exampleObj.Value, &value)
+		if err != nil {
+			logger.Log.Warningf("Failed decoding parameter example into string: %s", err)
+			continue
+		}
+
+		if value == exampleValue || cnt >= 5 { // 5 examples is enough
+			return nil
+		}
+	}
+
+	valMsg, err := json.Marshal(exampleValue)
+	if err != nil {
+		return err
+	}
+
+	themap := **param
+	themap["example #"+strconv.Itoa(cnt)] = &openapi.ExampleObj{Value: valMsg}
+
+	return nil
+}
+
+// TODO: somehow generalize the two example setting functions, plus add body example handling
+
+func addSchemaExample(existing *openapi.SchemaObj, bodyStr string) {
+	if len(existing.Examples) < 5 {
+		found := false
+		for _, eVal := range existing.Examples {
+			existingExample := ""
+			err := json.Unmarshal(eVal, &existingExample)
+			if err != nil {
+				logger.Log.Debugf("Failed to unmarshal example: %v", eVal)
+				continue
+			}
+
+			if existingExample == bodyStr {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			example, err := json.Marshal(bodyStr)
+			if err != nil {
+				logger.Log.Debugf("Failed to marshal example: %v", bodyStr)
+				return
+			}
+			existing.Examples = append(existing.Examples, example)
+		}
+	}
+}
+
+func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.com/jpillora/longestcommon
+	empty := make([]string, 0)
+	//short-circuit empty list
+	if len(strs) == 0 {
+		return empty
+	}
+	xfix := strs[0]
+	//short-circuit single-element list
+	if len(strs) == 1 {
+		return xfix
+	}
+	//compare first to rest
+	for _, str := range strs[1:] {
+		xfixl := len(xfix)
+		strl := len(str)
+		//short-circuit empty strings
+		if xfixl == 0 || strl == 0 {
+			return empty
+		}
+		//maximum possible length
+		maxl := xfixl
+		if strl < maxl {
+			maxl = strl
+		}
+		//compare letters
+		if pre {
+			//prefix, iterate left to right
+			for i := 0; i < maxl; i++ {
+				if xfix[i] != str[i] {
+					xfix = xfix[:i]
+					break
+				}
+			}
+		} else {
+			//suffix, iternate right to left
+			for i := 0; i < maxl; i++ {
+				xi := xfixl - i - 1
+				si := strl - i - 1
+				if xfix[xi] != str[si] {
+					xfix = xfix[xi+1:]
+					break
+				}
+			}
+		}
+	}
+	return xfix
+}
+
+func longestCommonXfixStr(strs []string, pre bool) string { // https://github.com/jpillora/longestcommon
+	//short-circuit empty list
+	if len(strs) == 0 {
+		return ""
+	}
+	xfix := strs[0]
+	//short-circuit single-element list
+	if len(strs) == 1 {
+		return xfix
+	}
+	//compare first to rest
+	for _, str := range strs[1:] {
+		xfixl := len(xfix)
+		strl := len(str)
+		//short-circuit empty strings
+		if xfixl == 0 || strl == 0 {
+			return ""
+		}
+		//maximum possible length
+		maxl := xfixl
+		if strl < maxl {
+			maxl = strl
+		}
+		//compare letters
+		if pre {
+			//prefix, iterate left to right
+			for i := 0; i < maxl; i++ {
+				if xfix[i] != str[i] {
+					xfix = xfix[:i]
+					break
+				}
+			}
+		} else {
+			//suffix, iternate right to left
+			for i := 0; i < maxl; i++ {
+				xi := xfixl - i - 1
+				si := strl - i - 1
+				if xfix[xi] != str[si] {
+					xfix = xfix[xi+1:]
+					break
+				}
+			}
+		}
+	}
+	return xfix
+}
+
+func getSimilarPrefix(strs []string) string {
+	chunked := make([][]string, 0)
+	for _, item := range strs {
+		chunked = append(chunked, strings.Split(item, "/"))
+	}
+
+	cmn := longestCommonXfix(chunked, true)
+	res := make([]string, 0)
+	for _, chunk := range cmn {
+		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
+			res = append(res, chunk)
+		}
+	}
+	return strings.Join(res[1:], ".")
+}
+
+// returns all non-nil ops in PathObj
+func getOps(pathObj *openapi.PathObj) []*openapi.Operation {
+	ops := []**openapi.Operation{&pathObj.Get, &pathObj.Patch, &pathObj.Put, &pathObj.Options, &pathObj.Post, &pathObj.Trace, &pathObj.Head, &pathObj.Delete}
+	res := make([]*openapi.Operation, 0)
+	for _, opp := range ops {
+		if *opp == nil {
+			continue
+		}
+		res = append(res, *opp)
+	}
+	return res
+}
+
+// parses JSON into any possible value
+func anyJSON(text string) (anyVal interface{}, isJSON bool) {
+	isJSON = true
+	asMap := map[string]interface{}{}
+	err := json.Unmarshal([]byte(text), &asMap)
+	if err == nil && asMap != nil {
+		return asMap, isJSON
+	}
+
+	asArray := make([]interface{}, 0)
+	err = json.Unmarshal([]byte(text), &asArray)
+	if err == nil && asArray != nil {
+		return asArray, isJSON
+	}
+
+	asString := ""
+	sPtr := &asString
+	err = json.Unmarshal([]byte(text), &sPtr)
+	if err == nil && sPtr != nil {
+		return asString, isJSON
+	}
+
+	asInt := 0
+	intPtr := &asInt
+	err = json.Unmarshal([]byte(text), &intPtr)
+	if err == nil && intPtr != nil {
+		return asInt, isJSON
+	}
+
+	asFloat := 0.0
+	floatPtr := &asFloat
+	err = json.Unmarshal([]byte(text), &floatPtr)
+	if err == nil && floatPtr != nil {
+		return asFloat, isJSON
+	}
+
+	asBool := false
+	boolPtr := &asBool
+	err = json.Unmarshal([]byte(text), &boolPtr)
+	if err == nil && boolPtr != nil {
+		return asBool, isJSON
+	}
+
+	if text == "null" {
+		return nil, isJSON
+	}
+
+	return nil, false
+}
+
+func cleanStr(str string, criterion func(r rune) bool) string {
+	s := []byte(str)
+	j := 0
+	for _, b := range s {
+		if criterion(rune(b)) {
+			s[j] = b
+			j++
+		}
+	}
+	return string(s[:j])
+}
+
+/*
+func isAlpha(s string) bool {
+	for _, r := range s {
+		if isAlphaRune(r) {
+			return false
+		}
+	}
+	return true
+}
+*/
+
+func isAlphaRune(r rune) bool {
+	return !((r < 'a' || r > 'z') && (r < 'A' || r > 'Z'))
+}
+
+func isAlNumRune(b rune) bool {
+	return isAlphaRune(b) || ('0' <= b && b <= '9')
+}
+
+func deleteFromSlice(s []string, val string) []string {
+	temp := s[:0]
+	for _, x := range s {
+		if x != val {
+			temp = append(temp, x)
+		}
+	}
+	return temp
+}
+
+func sliceContains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}
+
+func intersectSliceWithMap(required []string, names map[string]struct{}) []string {
+	for name := range names {
+		if !sliceContains(required, name) {
+			required = deleteFromSlice(required, name)
+		}
+	}
+	return required
+}
+
+func setSampleID(extensions *openapi.Extensions, id string) {
+	if id != "" {
+		if *extensions == nil {
+			*extensions = openapi.Extensions{}
+		}
+		err := (extensions).SetExtension(SampleId, id)
+		if err != nil {
+			logger.Log.Warningf("Failed to set sample ID: %s", err)
+		}
+	}
+}

agent/pkg/oas/utils_test.go

@@ -0,0 +1,59 @@
+package oas
+
+import (
+	"testing"
+)
+
+func TestAnyJSON(t *testing.T) {
+	testCases := []struct {
+		inp    string
+		isJSON bool
+		out    interface{}
+	}{
+		{`{"key": 1, "keyNull": null}`, true, nil},
+		{`[{"key": "val"}, ["subarray"], "string", 1, 2.2, true, null]`, true, nil},
+		{`"somestring"`, true, "somestring"},
+		{"0", true, 0},
+		{"0.5", true, 0.5},
+		{"true", true, true},
+		{"null", true, nil},
+		{"sabbra cadabra", false, nil},
+		{"0.1.2.3", false, nil},
+	}
+	for _, tc := range testCases {
+		any, isJSON := anyJSON(tc.inp)
+		if isJSON != tc.isJSON {
+			t.Errorf("Parse flag mismatch: %t != %t", tc.isJSON, isJSON)
+		} else if isJSON && tc.out != nil && tc.out != any {
+			t.Errorf("%s != %s", any, tc.out)
+		} else if tc.inp == "null" && any != nil {
+			t.Errorf("null has to parse as nil (but got %s)", any)
+		} else {
+			t.Logf("%s => %v", tc.inp, any)
+		}
+	}
+}
+
+func TestStrRunes(t *testing.T) {
+	if isAlphaRune('5') {
+		t.Logf("Failed")
+	}
+	if !isAlphaRune('a') {
+		t.Logf("Failed")
+	}
+
+	if !isAlNumRune('5') {
+		t.Logf("Failed")
+	}
+	if isAlNumRune(' ') {
+		t.Logf("Failed")
+	}
+
+	if cleanStr("-abc_567", isAlphaRune) != "abc" {
+		t.Logf("Failed")
+	}
+
+	if cleanStr("-abc_567", isAlNumRune) != "abc567" {
+		t.Logf("Failed")
+	}
+}

agent/pkg/providers/stats_provider.go

@@ -0,0 +1,357 @@
+package providers
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"reflect"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/jinzhu/copier"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/tap/api"
+)
+
+type GeneralStats struct {
+	EntriesCount        int
+	EntriesVolumeInGB   float64
+	FirstEntryTimestamp int
+	LastEntryTimestamp  int
+}
+
+type BucketStats []*TimeFrameStatsValue
+
+type TimeFrameStatsValue struct {
+	BucketTime    time.Time                `json:"timestamp"`
+	ProtocolStats map[string]ProtocolStats `json:"protocols"`
+}
+
+type ProtocolStats struct {
+	MethodsStats map[string]*SizeAndEntriesCount `json:"methods"`
+}
+
+type SizeAndEntriesCount struct {
+	EntriesCount  int `json:"entriesCount"`
+	VolumeInBytes int `json:"volumeInBytes"`
+}
+
+type AccumulativeStatsCounter struct {
+	Name            string `json:"name"`
+	Color           string `json:"color"`
+	EntriesCount    int    `json:"entriesCount"`
+	VolumeSizeBytes int    `json:"volumeSizeBytes"`
+}
+
+type AccumulativeStatsProtocol struct {
+	AccumulativeStatsCounter
+	Methods []*AccumulativeStatsCounter `json:"methods"`
+}
+
+type AccumulativeStatsProtocolTime struct {
+	ProtocolsData []*AccumulativeStatsProtocol `json:"protocols"`
+	Time          int64                        `json:"timestamp"`
+}
+
+type TrafficStatsResponse struct {
+	Protocols     []string                         `json:"protocols"`
+	PieStats      []*AccumulativeStatsProtocol     `json:"pie"`
+	TimelineStats []*AccumulativeStatsProtocolTime `json:"timeline"`
+}
+
+var (
+	generalStats      = GeneralStats{}
+	bucketsStats      = BucketStats{}
+	bucketStatsLocker = sync.Mutex{}
+	protocolToColor   = map[string]string{}
+)
+
+const (
+	InternalBucketThreshold = time.Minute * 1
+	MaxNumberOfBars         = 30
+)
+
+func ResetGeneralStats() {
+	generalStats = GeneralStats{}
+}
+
+func GetGeneralStats() *GeneralStats {
+	return &generalStats
+}
+
+func InitProtocolToColor(protocolMap map[string]*api.Protocol) {
+	for item, value := range protocolMap {
+		splitted := strings.SplitN(item, "/", 3)
+		protocolToColor[splitted[len(splitted)-1]] = value.BackgroundColor
+	}
+}
+
+func GetTrafficStats() *TrafficStatsResponse {
+	bucketsStatsCopy := getBucketStatsCopy()
+
+	return &TrafficStatsResponse{
+		Protocols:     getAvailableProtocols(bucketsStatsCopy),
+		PieStats:      getAccumulativeStats(bucketsStatsCopy),
+		TimelineStats: getAccumulativeStatsTiming(bucketsStatsCopy),
+	}
+}
+
+func EntryAdded(size int, summery *api.BaseEntry) {
+	generalStats.EntriesCount++
+	generalStats.EntriesVolumeInGB += float64(size) / (1 << 30)
+
+	currentTimestamp := int(time.Now().Unix())
+
+	if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) {
+		generalStats.FirstEntryTimestamp = currentTimestamp
+	}
+
+	addToBucketStats(size, summery)
+
+	generalStats.LastEntryTimestamp = currentTimestamp
+}
+
+func calculateInterval(firstTimestamp int64, lastTimestamp int64) time.Duration {
+	validDurations := []time.Duration{
+		time.Minute,
+		time.Minute * 2,
+		time.Minute * 3,
+		time.Minute * 5,
+		time.Minute * 10,
+		time.Minute * 15,
+		time.Minute * 20,
+		time.Minute * 30,
+		time.Minute * 45,
+		time.Minute * 60,
+		time.Minute * 75,
+		time.Minute * 90,   // 1.5 minutes
+		time.Minute * 120,  // 2 hours
+		time.Minute * 150,  // 2.5 hours
+		time.Minute * 180,  // 3 hours
+		time.Minute * 240,  // 4 hours
+		time.Minute * 300,  // 5 hours
+		time.Minute * 360,  // 6 hours
+		time.Minute * 420,  // 7 hours
+		time.Minute * 480,  // 8 hours
+		time.Minute * 540,  // 9 hours
+		time.Minute * 600,  // 10 hours
+		time.Minute * 660,  // 11 hours
+		time.Minute * 720,  // 12 hours
+		time.Minute * 1440, // 24 hours
+	}
+	duration := time.Duration(lastTimestamp-firstTimestamp) * time.Second / time.Duration(MaxNumberOfBars)
+	for _, validDuration := range validDurations {
+		if validDuration-duration >= 0 {
+			return validDuration
+		}
+	}
+	return duration.Round(validDurations[len(validDurations)-1])
+
+}
+
+func getAccumulativeStats(stats BucketStats) []*AccumulativeStatsProtocol {
+	if len(stats) == 0 {
+		return make([]*AccumulativeStatsProtocol, 0)
+	}
+
+	methodsPerProtocolAggregated := getAggregatedStats(stats)
+
+	return convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated)
+}
+
+func getAccumulativeStatsTiming(stats BucketStats) []*AccumulativeStatsProtocolTime {
+	if len(stats) == 0 {
+		return make([]*AccumulativeStatsProtocolTime, 0)
+	}
+
+	interval := calculateInterval(stats[0].BucketTime.Unix(), stats[len(stats)-1].BucketTime.Unix()) // in seconds
+	methodsPerProtocolPerTimeAggregated := getAggregatedResultTiming(stats, interval)
+
+	return convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggregated)
+}
+
+func addToBucketStats(size int, summery *api.BaseEntry) {
+	entryTimeBucketRounded := getBucketFromTimeStamp(summery.Timestamp)
+
+	if len(bucketsStats) == 0 {
+		bucketsStats = append(bucketsStats, &TimeFrameStatsValue{
+			BucketTime:    entryTimeBucketRounded,
+			ProtocolStats: map[string]ProtocolStats{},
+		})
+	}
+	bucketOfEntry := bucketsStats[len(bucketsStats)-1]
+	if bucketOfEntry.BucketTime != entryTimeBucketRounded {
+		bucketOfEntry = &TimeFrameStatsValue{
+			BucketTime:    entryTimeBucketRounded,
+			ProtocolStats: map[string]ProtocolStats{},
+		}
+		bucketsStats = append(bucketsStats, bucketOfEntry)
+	}
+	if _, found := bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation]; !found {
+		bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation] = ProtocolStats{
+			MethodsStats: map[string]*SizeAndEntriesCount{},
+		}
+	}
+	if _, found := bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation].MethodsStats[summery.Method]; !found {
+		bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation].MethodsStats[summery.Method] = &SizeAndEntriesCount{
+			VolumeInBytes: 0,
+			EntriesCount:  0,
+		}
+	}
+
+	bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation].MethodsStats[summery.Method].EntriesCount += 1
+	bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation].MethodsStats[summery.Method].VolumeInBytes += size
+}
+
+func getBucketFromTimeStamp(timestamp int64) time.Time {
+	entryTimeStampAsTime := time.UnixMilli(timestamp)
+	return entryTimeStampAsTime.Add(-1 * InternalBucketThreshold / 2).Round(InternalBucketThreshold)
+}
+
+func convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggregated map[time.Time]map[string]map[string]*AccumulativeStatsCounter) []*AccumulativeStatsProtocolTime {
+	finalResult := make([]*AccumulativeStatsProtocolTime, 0)
+	for timeKey, item := range methodsPerProtocolPerTimeAggregated {
+		protocolsData := make([]*AccumulativeStatsProtocol, 0)
+		for protocolName, value := range item {
+			entriesCount := 0
+			volumeSizeBytes := 0
+			methods := make([]*AccumulativeStatsCounter, 0)
+			for _, methodAccData := range value {
+				entriesCount += methodAccData.EntriesCount
+				volumeSizeBytes += methodAccData.VolumeSizeBytes
+				methods = append(methods, methodAccData)
+			}
+			protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
+				AccumulativeStatsCounter: AccumulativeStatsCounter{
+					Name:            protocolName,
+					Color:           protocolToColor[protocolName],
+					EntriesCount:    entriesCount,
+					VolumeSizeBytes: volumeSizeBytes,
+				},
+				Methods: methods,
+			})
+		}
+		finalResult = append(finalResult, &AccumulativeStatsProtocolTime{
+			Time:          timeKey.UnixMilli(),
+			ProtocolsData: protocolsData,
+		})
+	}
+	return finalResult
+}
+
+func convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated map[string]map[string]*AccumulativeStatsCounter) []*AccumulativeStatsProtocol {
+	protocolsData := make([]*AccumulativeStatsProtocol, 0)
+	for protocolName, value := range methodsPerProtocolAggregated {
+		entriesCount := 0
+		volumeSizeBytes := 0
+		methods := make([]*AccumulativeStatsCounter, 0)
+		for _, methodAccData := range value {
+			entriesCount += methodAccData.EntriesCount
+			volumeSizeBytes += methodAccData.VolumeSizeBytes
+			methods = append(methods, methodAccData)
+		}
+		protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
+			AccumulativeStatsCounter: AccumulativeStatsCounter{
+				Name:            protocolName,
+				Color:           protocolToColor[protocolName],
+				EntriesCount:    entriesCount,
+				VolumeSizeBytes: volumeSizeBytes,
+			},
+			Methods: methods,
+		})
+	}
+	return protocolsData
+}
+
+func getBucketStatsCopy() BucketStats {
+	bucketStatsCopy := BucketStats{}
+	bucketStatsLocker.Lock()
+	if err := copier.Copy(&bucketStatsCopy, bucketsStats); err != nil {
+		logger.Log.Errorf("Error while copying src stats into temporary copied object")
+		return nil
+	}
+	bucketStatsLocker.Unlock()
+	return bucketStatsCopy
+}
+
+func getAggregatedResultTiming(stats BucketStats, interval time.Duration) map[time.Time]map[string]map[string]*AccumulativeStatsCounter {
+	methodsPerProtocolPerTimeAggregated := map[time.Time]map[string]map[string]*AccumulativeStatsCounter{}
+
+	bucketStatsIndex := len(stats) - 1
+	for bucketStatsIndex >= 0 {
+		currentBucketTime := stats[bucketStatsIndex].BucketTime
+		resultBucketRoundedKey := currentBucketTime.Add(-1 * interval / 2).Round(interval)
+
+		for protocolName, data := range stats[bucketStatsIndex].ProtocolStats {
+			for methodName, dataOfMethod := range data.MethodsStats {
+
+				if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey]; !ok {
+					methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey] = map[string]map[string]*AccumulativeStatsCounter{}
+				}
+				if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName]; !ok {
+					methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName] = map[string]*AccumulativeStatsCounter{}
+				}
+				if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName]; !ok {
+					methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName] = &AccumulativeStatsCounter{
+						Name:            methodName,
+						Color:           getColorForMethod(protocolName, methodName),
+						EntriesCount:    0,
+						VolumeSizeBytes: 0,
+					}
+				}
+				methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName].EntriesCount += dataOfMethod.EntriesCount
+				methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName].VolumeSizeBytes += dataOfMethod.VolumeInBytes
+			}
+		}
+
+		bucketStatsIndex--
+	}
+	return methodsPerProtocolPerTimeAggregated
+}
+
+func getAggregatedStats(stats BucketStats) map[string]map[string]*AccumulativeStatsCounter {
+	methodsPerProtocolAggregated := make(map[string]map[string]*AccumulativeStatsCounter, 0)
+	for _, countersOfTimeFrame := range stats {
+		for protocolName, value := range countersOfTimeFrame.ProtocolStats {
+			for method, countersValue := range value.MethodsStats {
+				if _, found := methodsPerProtocolAggregated[protocolName]; !found {
+					methodsPerProtocolAggregated[protocolName] = map[string]*AccumulativeStatsCounter{}
+				}
+				if _, found := methodsPerProtocolAggregated[protocolName][method]; !found {
+					methodsPerProtocolAggregated[protocolName][method] = &AccumulativeStatsCounter{
+						Name:            method,
+						Color:           getColorForMethod(protocolName, method),
+						EntriesCount:    0,
+						VolumeSizeBytes: 0,
+					}
+				}
+				methodsPerProtocolAggregated[protocolName][method].EntriesCount += countersValue.EntriesCount
+				methodsPerProtocolAggregated[protocolName][method].VolumeSizeBytes += countersValue.VolumeInBytes
+			}
+		}
+	}
+	return methodsPerProtocolAggregated
+}
+
+func getColorForMethod(protocolName string, methodName string) string {
+	hash := md5.Sum([]byte(fmt.Sprintf("%v_%v", protocolName, methodName)))
+	input := hex.EncodeToString(hash[:])
+	return fmt.Sprintf("#%v", input[:6])
+}
+
+func getAvailableProtocols(stats BucketStats) []string {
+	protocols := map[string]bool{}
+	for _, countersOfTimeFrame := range stats {
+		for protocolName := range countersOfTimeFrame.ProtocolStats {
+			protocols[protocolName] = true
+		}
+	}
+
+	result := make([]string, 0)
+	for protocol := range protocols {
+		result = append(result, protocol)
+	}
+	result = append(result, "ALL")
+	return result
+}

agent/pkg/providers/stats_provider_internal_test.go

@@ -0,0 +1,298 @@
+package providers
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestGetBucketOfTimeStamp(t *testing.T) {
+	tests := map[int64]time.Time{
+		time.Date(2022, time.Month(1), 1, 10, 34, 45, 0, time.Local).UnixMilli(): time.Date(2022, time.Month(1), 1, 10, 34, 00, 0, time.Local),
+		time.Date(2022, time.Month(1), 1, 10, 34, 00, 0, time.Local).UnixMilli(): time.Date(2022, time.Month(1), 1, 10, 34, 00, 0, time.Local),
+		time.Date(2022, time.Month(1), 1, 10, 59, 01, 0, time.Local).UnixMilli(): time.Date(2022, time.Month(1), 1, 10, 59, 00, 0, time.Local),
+	}
+
+	for key, value := range tests {
+		t.Run(fmt.Sprintf("%v", key), func(t *testing.T) {
+
+			actual := getBucketFromTimeStamp(key)
+
+			if actual != value {
+				t.Errorf("unexpected result - expected: %v, actual: %v", value, actual)
+			}
+		})
+	}
+}
+
+func TestGetAggregatedStatsAllTime(t *testing.T) {
+	bucketStatsForTest := BucketStats{
+		&TimeFrameStatsValue{
+			BucketTime: time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC),
+			ProtocolStats: map[string]ProtocolStats{
+				"http": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"get": {
+							EntriesCount:  1,
+							VolumeInBytes: 2,
+						},
+						"post": {
+							EntriesCount:  2,
+							VolumeInBytes: 3,
+						},
+					},
+				},
+				"kafka": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"listTopics": {
+							EntriesCount:  5,
+							VolumeInBytes: 6,
+						},
+					},
+				},
+			},
+		},
+		&TimeFrameStatsValue{
+			BucketTime: time.Date(2022, time.Month(1), 1, 10, 01, 00, 0, time.UTC),
+			ProtocolStats: map[string]ProtocolStats{
+				"http": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"get": {
+							EntriesCount:  1,
+							VolumeInBytes: 2,
+						},
+						"post": {
+							EntriesCount:  2,
+							VolumeInBytes: 3,
+						},
+					},
+				},
+				"redis": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"set": {
+							EntriesCount:  5,
+							VolumeInBytes: 6,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	expected := map[string]map[string]*AccumulativeStatsCounter{
+		"http": {
+			"post": {
+				Name:            "post",
+				EntriesCount:    4,
+				VolumeSizeBytes: 6,
+			},
+			"get": {
+				Name:            "get",
+				EntriesCount:    2,
+				VolumeSizeBytes: 4,
+			},
+		},
+		"kafka": {
+			"listTopics": {
+				Name:            "listTopics",
+				EntriesCount:    5,
+				VolumeSizeBytes: 6,
+			},
+		},
+		"redis": {
+			"set": {
+				Name:            "set",
+				EntriesCount:    5,
+				VolumeSizeBytes: 6,
+			},
+		},
+	}
+	actual := getAggregatedStats(bucketStatsForTest)
+
+	if len(actual) != len(expected) {
+		t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
+	}
+}
+
+func TestGetAggregatedStatsFromSpecificTime(t *testing.T) {
+	bucketStatsForTest := BucketStats{
+		&TimeFrameStatsValue{
+			BucketTime: time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC),
+			ProtocolStats: map[string]ProtocolStats{
+				"http": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"get": {
+							EntriesCount:  1,
+							VolumeInBytes: 2,
+						},
+					},
+				},
+				"kafka": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"listTopics": {
+							EntriesCount:  5,
+							VolumeInBytes: 6,
+						},
+					},
+				},
+			},
+		},
+		&TimeFrameStatsValue{
+			BucketTime: time.Date(2022, time.Month(1), 1, 10, 01, 00, 0, time.UTC),
+			ProtocolStats: map[string]ProtocolStats{
+				"http": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"get": {
+							EntriesCount:  1,
+							VolumeInBytes: 2,
+						},
+						"post": {
+							EntriesCount:  2,
+							VolumeInBytes: 3,
+						},
+					},
+				},
+				"redis": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"set": {
+							EntriesCount:  5,
+							VolumeInBytes: 6,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	expected := map[time.Time]map[string]map[string]*AccumulativeStatsCounter{
+		time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC): {
+			"http": {
+				"post": {
+					Name:            "post",
+					EntriesCount:    2,
+					VolumeSizeBytes: 3,
+				},
+				"get": {
+					Name:            "get",
+					EntriesCount:    2,
+					VolumeSizeBytes: 4,
+				},
+			},
+			"kafka": {
+				"listTopics": {
+					Name:            "listTopics",
+					EntriesCount:    5,
+					VolumeSizeBytes: 6,
+				},
+			},
+			"redis": {
+				"set": {
+					Name:            "set",
+					EntriesCount:    5,
+					VolumeSizeBytes: 6,
+				},
+			},
+		},
+	}
+	actual := getAggregatedResultTiming(bucketStatsForTest, time.Minute*5)
+
+	if len(actual) != len(expected) {
+		t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
+	}
+}
+
+func TestGetAggregatedStatsFromSpecificTimeMultipleBuckets(t *testing.T) {
+	bucketStatsForTest := BucketStats{
+		&TimeFrameStatsValue{
+			BucketTime: time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC),
+			ProtocolStats: map[string]ProtocolStats{
+				"http": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"get": {
+							EntriesCount:  1,
+							VolumeInBytes: 2,
+						},
+					},
+				},
+				"kafka": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"listTopics": {
+							EntriesCount:  5,
+							VolumeInBytes: 6,
+						},
+					},
+				},
+			},
+		},
+		&TimeFrameStatsValue{
+			BucketTime: time.Date(2022, time.Month(1), 1, 10, 01, 00, 0, time.UTC),
+			ProtocolStats: map[string]ProtocolStats{
+				"http": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"get": {
+							EntriesCount:  1,
+							VolumeInBytes: 2,
+						},
+						"post": {
+							EntriesCount:  2,
+							VolumeInBytes: 3,
+						},
+					},
+				},
+				"redis": {
+					MethodsStats: map[string]*SizeAndEntriesCount{
+						"set": {
+							EntriesCount:  5,
+							VolumeInBytes: 6,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	expected := map[time.Time]map[string]map[string]*AccumulativeStatsCounter{
+		time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC): {
+			"http": {
+				"get": {
+					Name:            "get",
+					EntriesCount:    1,
+					VolumeSizeBytes: 2,
+				},
+			},
+			"kafka": {
+				"listTopics": {
+					Name:            "listTopics",
+					EntriesCount:    5,
+					VolumeSizeBytes: 6,
+				},
+			},
+		},
+		time.Date(2022, time.Month(1), 1, 10, 01, 00, 0, time.UTC): {
+			"http": {
+				"post": {
+					Name:            "post",
+					EntriesCount:    2,
+					VolumeSizeBytes: 3,
+				},
+				"get": {
+					Name:            "get",
+					EntriesCount:    1,
+					VolumeSizeBytes: 2,
+				},
+			},
+			"redis": {
+				"set": {
+					Name:            "set",
+					EntriesCount:    5,
+					VolumeSizeBytes: 6,
+				},
+			},
+		},
+	}
+	actual := getAggregatedResultTiming(bucketStatsForTest, time.Minute)
+
+	if len(actual) != len(expected) {
+		t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
+	}
+}

agent/pkg/providers/stats_provider_test.go

@@ -0,0 +1,84 @@
+package providers_test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/up9inc/mizu/agent/pkg/providers"
+	"github.com/up9inc/mizu/tap/api"
+)
+
+func TestNoEntryAddedCount(t *testing.T) {
+	entriesStats := providers.GetGeneralStats()
+
+	if entriesStats.EntriesCount != 0 {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesCount)
+	}
+
+	if entriesStats.EntriesVolumeInGB != 0 {
+		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesVolumeInGB)
+	}
+}
+
+func TestEntryAddedCount(t *testing.T) {
+	tests := []int{1, 5, 10, 100, 500, 1000}
+
+	entryBucketKey := time.Date(2021, 1, 1, 10, 0, 0, 0, time.UTC)
+	valueLessThanBucketThreshold := time.Second * 130
+	mockSummery := &api.BaseEntry{Protocol: api.Protocol{ProtocolSummary: api.ProtocolSummary{Name: "mock"}}, Method: "mock-method", Timestamp: entryBucketKey.Add(valueLessThanBucketThreshold).UnixNano()}
+	for _, entriesCount := range tests {
+		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
+			for i := 0; i < entriesCount; i++ {
+				providers.EntryAdded(0, mockSummery)
+			}
+
+			entriesStats := providers.GetGeneralStats()
+
+			if entriesStats.EntriesCount != entriesCount {
+				t.Errorf("unexpected result - expected: %v, actual: %v", entriesCount, entriesStats.EntriesCount)
+			}
+
+			if entriesStats.EntriesVolumeInGB != 0 {
+				t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesVolumeInGB)
+			}
+
+			t.Cleanup(func() {
+				providers.ResetGeneralStats()
+				generalStats := providers.GetGeneralStats()
+				if generalStats.EntriesCount != 0 {
+					t.Errorf("unexpected result - expected: %v, actual: %v", 0, generalStats.EntriesCount)
+				}
+
+			})
+		})
+	}
+}
+
+func TestEntryAddedVolume(t *testing.T) {
+	// 6 bytes + 4 bytes
+	tests := [][]byte{[]byte("volume"), []byte("test")}
+	var expectedEntriesCount int
+	var expectedVolumeInGB float64
+
+	mockSummery := &api.BaseEntry{Protocol: api.Protocol{ProtocolSummary: api.ProtocolSummary{Name: "mock"}}, Method: "mock-method", Timestamp: time.Date(2021, 1, 1, 10, 0, 0, 0, time.UTC).UnixNano()}
+
+	for _, data := range tests {
+		t.Run(fmt.Sprintf("%d", len(data)), func(t *testing.T) {
+			expectedEntriesCount++
+			expectedVolumeInGB += float64(len(data)) / (1 << 30)
+
+			providers.EntryAdded(len(data), mockSummery)
+
+			entriesStats := providers.GetGeneralStats()
+
+			if entriesStats.EntriesCount != expectedEntriesCount {
+				t.Errorf("unexpected result - expected: %v, actual: %v", expectedEntriesCount, entriesStats.EntriesCount)
+			}
+
+			if entriesStats.EntriesVolumeInGB != expectedVolumeInGB {
+				t.Errorf("unexpected result - expected: %v, actual: %v", expectedVolumeInGB, entriesStats.EntriesVolumeInGB)
+			}
+		})
+	}
+}