Develop -> main (version 35.0)

Merge pull request #1182 from up9inc/develop #major

.github/workflows/acceptance_tests.yml

@@ -8,25 +8,47 @@ on:
     branches:
       - 'develop'
 
-concurrency:
-  group: mizu-acceptance-tests-${{ github.ref }}
-  cancel-in-progress: true
+env:
+  MIZU_CI_IMAGE: mizu/ci:0.0
 
 jobs:
   run-acceptance-tests:
     name: Run acceptance tests
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.16
+      - name: Set up Go 1.17
         uses: actions/setup-go@v2
         with:
-          go-version: '^1.16'
+          go-version: '^1.17'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          load: true
+          tags: ${{ env.MIZU_CI_IMAGE }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Setup acceptance test
-        run: source ./acceptanceTests/setup.sh
+        run: ./acceptanceTests/setup.sh
+
+      - name: Create k8s users and change context
+        env:
+          USERNAME_UNRESTRICTED: user-with-clusterwide-access
+          USERNAME_RESTRICTED: user-with-restricted-access
+        run: |
+          ./acceptanceTests/create_user.sh "${USERNAME_UNRESTRICTED}"
+          ./acceptanceTests/create_user.sh "${USERNAME_RESTRICTED}"
+          kubectl apply -f cli/cmd/permissionFiles/permissions-all-namespaces-tap.yaml
+          kubectl config use-context ${USERNAME_UNRESTRICTED}
 
       - name: Test
         run: make acceptance-test
@@ -37,7 +59,7 @@ jobs:
         with:
           status: ${{ job.status }}
           notification_title: 'Mizu {workflow} has {status_message}'
-          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit: <{commit_url}|{commit_sha}>'
+          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit <{commit_url}|{commit_sha}> by ${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}> ```${{ github.event.head_commit.message }}```'
           footer: 'Linked Repo <{repo_url}|{repo}>'
           notify_when: 'failure'
         env:

.github/workflows/acceptance_tests_on_pr.yml

@@ -0,0 +1,54 @@
+name: Acceptance tests on PR
+
+on: push
+
+env:
+  MIZU_CI_IMAGE: mizu/ci:0.0
+
+concurrency:
+  group: acceptance-tests-on-pr-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-tests:
+    name: Run tests
+    runs-on: ubuntu-latest
+    if: ${{ contains(github.event.head_commit.message, '#run_acceptance_tests') }}
+
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: false
+          load: true
+          tags: ${{ env.MIZU_CI_IMAGE }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Setup acceptance test
+        run: ./acceptanceTests/setup.sh
+
+      - name: Create k8s users and change context
+        env:
+          USERNAME_UNRESTRICTED: user-with-clusterwide-access
+          USERNAME_RESTRICTED: user-with-restricted-access
+        run: |
+          ./acceptanceTests/create_user.sh "${USERNAME_UNRESTRICTED}"
+          ./acceptanceTests/create_user.sh "${USERNAME_RESTRICTED}"
+          kubectl apply -f cli/cmd/permissionFiles/permissions-all-namespaces-tap.yaml
+          kubectl config use-context ${USERNAME_UNRESTRICTED}
+
+      - name: Test
+        run: make acceptance-test

.github/workflows/build-custom-branch.yml

@@ -0,0 +1,44 @@
+name: Build Custom Branch
+
+on: push
+
+concurrency:
+  group: custom-branch-build-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Push custom branch image to GCR
+    runs-on: ubuntu-latest
+    if: ${{ contains(github.event.head_commit.message, '#build_and_publish_custom_image') }}
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
+        with:
+          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v0'
+
+      - name: Get base image name
+        shell: bash
+        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
+        id: base_image_step
+
+      - name: Login to GCR
+        uses: docker/login-action@v1
+        with:
+          registry: gcr.io
+          username: _json_key
+          password: ${{ secrets.GCR_JSON_KEY }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.base_image_step.outputs.image }}:latest

.github/workflows/build.yml

@@ -15,15 +15,23 @@ jobs:
     name: CLI executable build
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.16'
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      
+      - name: Check modified files
+        id: modified_files
+        run: devops/check_modified_files.sh cli/
+
+      - name: Set up Go 1.17
+        if: steps.modified_files.outputs.matched == 'true'
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.17'
 
       - name: Build CLI
+        if: steps.modified_files.outputs.matched == 'true'
         run: make cli
 
   build-agent:
@@ -32,6 +40,23 @@ jobs:
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      
+      - name: Check modified files
+        id: modified_files
+        run: devops/check_modified_files.sh agent/ shared/ tap/ ui/ ui-common/ Dockerfile
 
-      - name: Build Agent
-        run: make agent-docker
+      - name: Set up Docker Buildx
+        if: steps.modified_files.outputs.matched == 'true'
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        if: steps.modified_files.outputs.matched == 'true'
+        with:
+          context: .
+          push: false
+          tags: up9inc/mizu:devlatest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/inactive-issues-close.yaml

@@ -13,10 +13,11 @@ jobs:
       - uses: actions/stale@v3
         with:
           days-before-issue-stale: 30
-          days-before-issue-close: 14
+          days-before-issue-close: -1
+          exempt-issue-labels: "enhancement"
           stale-issue-label: "stale"
           stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
-          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+          close-issue-message: ""
           days-before-pr-stale: -1
           days-before-pr-close: -1
           repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -58,6 +58,8 @@ jobs:
             up9inc/mizu
           tags: |
             type=raw,${{ steps.versioning.outputs.version }}
+            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
+            type=raw,value=dev-latest,enable=${{ steps.condval.outputs.value == 'dev' }}
           flavor: |
             latest=auto
             prefix=
@@ -143,6 +145,8 @@ jobs:
             ${{ steps.base_image_step.outputs.image }}
           tags: |
             type=raw,${{ steps.versioning.outputs.version }}
+            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
+            type=raw,value=dev-latest,enable=${{ steps.condval.outputs.value == 'dev' }}
           flavor: |
             latest=auto
             prefix=
@@ -205,7 +209,8 @@ jobs:
             up9inc/mizu
           tags: |
             type=raw,${{ steps.versioning.outputs.version }}
-
+            type=raw,value=latest,enable=${{ steps.condval.outputs.value == 'stable' }}
+            type=raw,value=dev-latest,enable=${{ steps.condval.outputs.value == 'dev' }}
       - name: Login to Docker Hub
         uses: docker/login-action@v1
         with:
@@ -229,10 +234,10 @@ jobs:
     runs-on: ubuntu-latest
     needs: [docker-manifest, gcp-registry]
     steps:
-      - name: Set up Go 1.16
+      - name: Set up Go 1.17
         uses: actions/setup-go@v2
         with:
-          go-version: '1.16'
+          go-version: '1.17'
 
       - name: Check out the repo
         uses: actions/checkout@v2
@@ -285,3 +290,15 @@ jobs:
           tag: ${{ steps.versioning.outputs.version }}
           prerelease: ${{ github.ref != 'refs/heads/main' }}
           bodyFile: 'cli/bin/README.md'
+      
+      - name: Slack notification on failure
+        uses: ravsamhq/notify-slack-action@v1
+        if: always()
+        with:
+          status: ${{ job.status }}
+          notification_title: 'Mizu enterprise {workflow} has {status_message}'
+          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit <{commit_url}|{commit_sha}> by ${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}> ```${{ github.event.head_commit.message }}```'
+          footer: 'Linked Repo <{repo_url}|{repo}>'
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/static_code_analysis.yml

@@ -10,81 +10,193 @@ permissions:
   contents: read
 
 jobs:
-  golangci:
+  go-lint:
     name: Go lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
       - uses: actions/setup-go@v2
         with:
-            go-version: '^1.16'
+            go-version: '^1.17'
 
       - name: Install dependencies
         run: |
           sudo apt update
           sudo apt install -y libpcap-dev
+          ./devops/install-capstone.sh
+
+      - name: Check Agent modified files
+        id: agent_modified_files
+        run: devops/check_modified_files.sh agent/
 
       - name: Go lint - agent
         uses: golangci/golangci-lint-action@v2
+        if: steps.agent_modified_files.outputs.matched == 'true'
         with:
           version: latest
           working-directory: agent
-          args: --timeout=3m
+          args: --timeout=10m
+
+      - name: Check shared modified files
+        id: shared_modified_files
+        run: devops/check_modified_files.sh shared/
 
       - name: Go lint - shared
         uses: golangci/golangci-lint-action@v2
+        if: steps.shared_modified_files.outputs.matched == 'true'
         with:
           version: latest
           working-directory: shared
-          args: --timeout=3m
+          args: --timeout=10m
+
+      - name: Check tap modified files
+        id: tap_modified_files
+        run: devops/check_modified_files.sh tap/
 
       - name: Go lint - tap
         uses: golangci/golangci-lint-action@v2
+        if: steps.tap_modified_files.outputs.matched == 'true'
         with:
           version: latest
           working-directory: tap
-          args: --timeout=3m
+          args: --timeout=10m
+
+      - name: Check cli modified files
+        id: cli_modified_files
+        run: devops/check_modified_files.sh cli/
 
       - name: Go lint - CLI
         uses: golangci/golangci-lint-action@v2
+        if: steps.cli_modified_files.outputs.matched == 'true'
         with:
           version: latest
           working-directory: cli
-          args: --timeout=3m
+          args: --timeout=10m
+
+      - name: Check acceptanceTests modified files
+        id: acceptanceTests_modified_files
+        run: devops/check_modified_files.sh acceptanceTests/
 
       - name: Go lint - acceptanceTests
         uses: golangci/golangci-lint-action@v2
+        if: steps.acceptanceTests_modified_files.outputs.matched == 'true'
         with:
           version: latest
           working-directory: acceptanceTests
-          args: --timeout=3m
+          args: --timeout=10m
+
+      - name: Check tap/api modified files
+        id: tap_api_modified_files
+        run: devops/check_modified_files.sh tap/api/
 
       - name: Go lint - tap/api
         uses: golangci/golangci-lint-action@v2
+        if: steps.tap_api_modified_files.outputs.matched == 'true'
         with:
           version: latest
           working-directory: tap/api
 
+      - name: Check tap/extensions/amqp modified files
+        id: tap_amqp_modified_files
+        run: devops/check_modified_files.sh tap/extensions/amqp/
+
       - name: Go lint - tap/extensions/amqp
         uses: golangci/golangci-lint-action@v2
+        if: steps.tap_amqp_modified_files.outputs.matched == 'true'
         with:
           version: latest
           working-directory: tap/extensions/amqp
+      
+      - name: Check tap/extensions/http modified files
+        id: tap_http_modified_files
+        run: devops/check_modified_files.sh tap/extensions/http/
 
       - name: Go lint - tap/extensions/http
         uses: golangci/golangci-lint-action@v2
+        if: steps.tap_http_modified_files.outputs.matched == 'true'
         with:
           version: latest
           working-directory: tap/extensions/http
 
+      - name: Check tap/extensions/kafka modified files
+        id: tap_kafka_modified_files
+        run: devops/check_modified_files.sh tap/extensions/kafka/
+
       - name: Go lint - tap/extensions/kafka
         uses: golangci/golangci-lint-action@v2
+        if: steps.tap_kafka_modified_files.outputs.matched == 'true'
         with:
           version: latest
           working-directory: tap/extensions/kafka
 
+      - name: Check tap/extensions/redis modified files
+        id: tap_redis_modified_files
+        run: devops/check_modified_files.sh tap/extensions/redis/
+
       - name: Go lint - tap/extensions/redis
         uses: golangci/golangci-lint-action@v2
+        if: steps.tap_redis_modified_files.outputs.matched == 'true'
         with:
           version: latest
           working-directory: tap/extensions/redis
+
+      - name: Check logger modified files
+        id: logger_modified_files
+        run: devops/check_modified_files.sh logger/
+
+      - name: Go lint - logger
+        uses: golangci/golangci-lint-action@v2
+        if: steps.logger_modified_files.outputs.matched == 'true'
+        with:
+          version: latest
+          working-directory: logger
+
+  es-lint:
+    name: ES lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16
+        
+      - name: Check modified UI files
+        id: ui_modified_files
+        run: devops/check_modified_files.sh ui/
+      
+      - name: ESLint prerequisites ui
+        if: steps.ui_modified_files.outputs.matched == 'true'
+        run: |
+          sudo npm install -g eslint
+          cd ui
+          npm run prestart
+          npm i
+
+      - name: ESLint ui
+        if: steps.ui_modified_files.outputs.matched == 'true'
+        run: |
+          cd ui
+          npm run eslint
+      
+      - name: Check modified ui-common files
+        id: ui_common_modified_files
+        run: devops/check_modified_files.sh ui-common/
+      
+      - name: ESLint prerequisites ui-common
+        if: steps.ui_common_modified_files.outputs.matched == 'true'
+        run: |
+          sudo npm install -g eslint
+          cd ui-common
+          npm i
+
+      - name: ESLint ui-common
+        if: steps.ui_common_modified_files.outputs.matched == 'true'
+        run: |
+          cd ui-common
+          npm run eslint

.github/workflows/test.yml

@@ -18,28 +18,54 @@ jobs:
   run-unit-tests:
     name: Unit Tests
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     steps:
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
-        with:
-          go-version: '^1.16'
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
 
-      - shell: bash
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17'
+
+      - name: Install libpcap
+        shell: bash
         run: |
           sudo apt-get install libpcap-dev
 
+      - name: Install Capstone
+        shell: bash
+        run: |
+          ./devops/install-capstone.sh
+
+      - name: Check CLI modified files
+        id: cli_modified_files
+        run: devops/check_modified_files.sh cli/
+
       - name: CLI Test
+        if: github.event_name == 'push' || steps.cli_modified_files.outputs.matched == 'true'
         run: make test-cli
 
+      - name: Check Agent modified files
+        id: agent_modified_files
+        run: devops/check_modified_files.sh agent/
+
       - name: Agent Test
+        if: github.event_name == 'push' || steps.agent_modified_files.outputs.matched == 'true'
         run: make test-agent
-      
+
       - name: Shared Test
         run: make test-shared
+      
+      - name: Check extensions modified files
+        id: ext_modified_files
+        run: devops/check_modified_files.sh tap/extensions/ tap/api/
+
+      - name: Extensions Test
+        if: github.event_name == 'push' || steps.ext_modified_files.outputs.matched == 'true'
+        run: make test-extensions
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
-

.gitignore

@@ -30,7 +30,7 @@ build
 pprof/*
 
 # Database Files
-*.bin
+*.db
 *.gob
 
 # Nohup Files - https://man7.org/linux/man-pages/man1/nohup.1p.html
@@ -44,3 +44,15 @@ cypress.env.json
 */cypress/screenshots
 */cypress/videos
 */cypress/support
+
+# Ignore test data in extensions
+tap/extensions/*/bin
+tap/extensions/*/expect
+
+# UI folders to ignore
+**/node_modules/**
+**/dist/**
+*.editorconfig
+
+# Ignore *.log files
+*.log

Dockerfile

@@ -1,38 +1,70 @@
 ARG BUILDARCH=amd64
 ARG TARGETARCH=amd64
 
+### Front-end common
+FROM node:16 AS front-end-common
+
+WORKDIR /app/ui-build
+COPY ui-common/package.json .
+COPY ui-common/package-lock.json .
+RUN npm i
+COPY ui-common .
+RUN npm pack
+
 ### Front-end
 FROM node:16 AS front-end
 
 WORKDIR /app/ui-build
 
-COPY ui/package.json .
-COPY ui/package-lock.json .
+COPY ui/package.json ui/package-lock.json ./
+COPY --from=front-end-common ["/app/ui-build/up9-mizu-common-0.0.0.tgz", "."]
 RUN npm i
 COPY ui .
 RUN npm run build
-RUN npm run build-ent
 
 ### Base builder image for native builds architecture
-FROM golang:1.16-alpine AS builder-native-base
+FROM golang:1.17-alpine AS builder-native-base
 ENV CGO_ENABLED=1 GOOS=linux
-RUN apk add libpcap-dev g++ perl-utils
+RUN apk add --no-cache \
+    libpcap-dev \
+    g++ \
+    perl-utils \
+    curl \
+    build-base \
+    binutils-gold \
+    bash \
+    clang \
+    llvm \
+    libbpf-dev \
+    linux-headers
+COPY devops/install-capstone.sh .
+RUN ./install-capstone.sh
 
 
 ### Intermediate builder image for x86-64 to x86-64 native builds
 FROM builder-native-base AS builder-from-amd64-to-amd64
 ENV GOARCH=amd64
+ENV BPF_TARGET=amd64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86"
 
 
 ### Intermediate builder image for AArch64 to AArch64 native builds
 FROM builder-native-base AS builder-from-arm64v8-to-arm64v8
 ENV GOARCH=arm64
+ENV BPF_TARGET=arm64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64"
 
 
 ### Builder image for x86-64 to AArch64 cross-compilation
-FROM up9inc/linux-arm64-musl-go-libpcap AS builder-from-amd64-to-arm64v8
+FROM up9inc/linux-arm64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 AS builder-from-amd64-to-arm64v8
 ENV CGO_ENABLED=1 GOOS=linux
-ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap"
+ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap -I/work/capstone/include"
+ENV BPF_TARGET=arm64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64 -I/usr/xcc/aarch64-linux-musl-cross/aarch64-linux-musl/include/"
+
+
+### Builder image for AArch64 to x86-64 cross-compilation
+FROM up9inc/linux-x86_64-musl-go-libpcap-capstone-bpf:capstone-5.0-rc2 AS builder-from-arm64v8-to-amd64
+ENV CGO_ENABLED=1 GOOS=linux
+ENV GOARCH=amd64 CGO_CFLAGS="-I/libpcap -I/capstone/include"
+ENV BPF_TARGET=amd64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86  -I/usr/local/musl/x86_64-unknown-linux-musl/include/"
 
 
 ### Final builder image where the build happens
@@ -40,6 +72,7 @@ ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap"
 # BUILDARCH=amd64 TARGETARCH=amd64
 # BUILDARCH=arm64v8 TARGETARCH=arm64v8
 # BUILDARCH=amd64 TARGETARCH=arm64v8
+# BUILDARCH=arm64v8 TARGETARCH=amd64
 ARG BUILDARCH=amd64
 ARG TARGETARCH=amd64
 FROM builder-from-${BUILDARCH}-to-${TARGETARCH} AS builder
@@ -49,18 +82,19 @@ WORKDIR /app/agent-build
 
 COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
+COPY logger/go.mod logger/go.mod ../logger/
 COPY tap/go.mod tap/go.mod ../tap/
 COPY tap/api/go.mod ../tap/api/
+COPY tap/dbgctl/go.mod ../tap/dbgctl/
 COPY tap/extensions/amqp/go.mod ../tap/extensions/amqp/
 COPY tap/extensions/http/go.mod ../tap/extensions/http/
 COPY tap/extensions/kafka/go.mod ../tap/extensions/kafka/
 COPY tap/extensions/redis/go.mod ../tap/extensions/redis/
 RUN go mod download
-# cheap trick to make the build faster (as long as go.mod did not change)
-RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
 
 # Copy and build agent code
 COPY shared ../shared
+COPY logger ../logger
 COPY tap ../tap
 COPY agent .
 
@@ -69,6 +103,11 @@ ARG GIT_BRANCH
 ARG BUILD_TIMESTAMP
 ARG VER=0.0
 
+WORKDIR /app/tap/tlstapper
+
+RUN rm tlstapper_bpf*
+RUN GOARCH=${BUILDARCH} go generate tls_tapper.go
+
 WORKDIR /app/agent-build
 
 RUN go build -ldflags="-extldflags=-static -s -w \
@@ -78,17 +117,16 @@ RUN go build -ldflags="-extldflags=-static -s -w \
     -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum
-ADD https://github.com/up9inc/basenine/releases/download/v0.4.13/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
-ADD https://github.com/up9inc/basenine/releases/download/v0.4.13/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
-RUN shasum -a 256 -c basenine_linux_${GOARCH}.sha256
-RUN chmod +x ./basenine_linux_${GOARCH}
-RUN mv ./basenine_linux_${GOARCH} ./basenine
+ADD https://github.com/up9inc/basenine/releases/download/v0.8.3/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
+ADD https://github.com/up9inc/basenine/releases/download/v0.8.3/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
 
+RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
+    chmod +x ./basenine_linux_"${GOARCH}" && \
+    mv ./basenine_linux_"${GOARCH}" ./basenine
 
 ### The shipped image
 ARG TARGETARCH=amd64
 FROM ${TARGETARCH}/busybox:latest
-
 # gin-gonic runs in debug mode without this
 ENV GIN_MODE=release
 
@@ -99,7 +137,6 @@ WORKDIR /app
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
 COPY --from=builder ["/app/agent-build/basenine", "/usr/local/bin/basenine"]
 COPY --from=front-end ["/app/ui-build/build", "site"]
-COPY --from=front-end ["/app/ui-build/build-ent", "site-standalone"]
 
 # this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
 ENTRYPOINT ["/app/mizuagent"]

Makefile

@@ -31,9 +31,6 @@ cli: ## Build CLI.
 cli-debug: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build-debug
 
-build-cli-ci: ## Build CLI for CI.
-	@echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci
-
 agent: ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
@@ -57,10 +54,6 @@ push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
 	devops/build-push-featurebranch.sh
 
-build-docker-ci: ## Build agent docker image for CI.
-	@echo "building docker image for ci"
-	devops/build-agent-ci.sh
-
 push-cli: ## Build and publish CLI.
 	@echo "publishing CLI .. "
 	@cd cli; $(MAKE) build-all
@@ -80,17 +73,33 @@ clean-agent: ## Clean agent.
 clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
-clean-docker:
+clean-docker:  ## Run clean docker
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
-test-cli:
+test-lint:  ## Run lint on all modules
+	cd agent && golangci-lint run
+	cd shared && golangci-lint run
+	cd tap && golangci-lint run
+	cd cli && golangci-lint run
+	cd acceptanceTests && golangci-lint run
+	cd tap/api && golangci-lint run
+	cd tap/dbgctl && golangci-lint run
+	cd tap/extensions/ && for D in */; do cd $$D && golangci-lint run && cd ..; done
+
+test-cli:  ## Run cli tests
 	@echo "running cli tests"; cd cli && $(MAKE) test
 
-test-agent:
+test-agent:  ## Run agent tests
 	@echo "running agent tests"; cd agent && $(MAKE) test
 
-test-shared:
+test-shared:  ## Run shared tests
 	@echo "running shared tests"; cd shared && $(MAKE) test
 
-acceptance-test:
+test-extensions:  ## Run extensions tests
+	@echo "running http tests"; cd tap/extensions/http && $(MAKE) test
+	@echo "running redis tests"; cd tap/extensions/redis && $(MAKE) test
+	@echo "running kafka tests"; cd tap/extensions/kafka && $(MAKE) test
+	@echo "running amqp tests"; cd tap/extensions/amqp && $(MAKE) test
+
+acceptance-test:  ## Run acceptance tests
 	@echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test

README.md

@@ -8,10 +8,10 @@
         <img alt="GitHub Latest Release" src="https://img.shields.io/github/v/release/up9inc/mizu?logo=GitHub&style=flat-square">
     </a>
     <a href="https://hub.docker.com/r/up9inc/mizu">
-      <img alt="Docker pulls" src="https://img.shields.io/docker/pulls/up9inc/mizu?color=%23099cec">
+      <img alt="Docker pulls" src="https://img.shields.io/docker/pulls/up9inc/mizu?color=%23099cec&logo=Docker&style=flat-square">
     </a>
     <a href="https://hub.docker.com/r/up9inc/mizu">
-      <img alt="Image size" src="https://img.shields.io/docker/image-size/up9inc/mizu/latest">
+      <img alt="Image size" src="https://img.shields.io/docker/image-size/up9inc/mizu/latest?logo=Docker&style=flat-square">
     </a>
     <a href="https://join.slack.com/t/up9/shared_invite/zt-tfjnduli-QzlR8VV4Z1w3YnPIAJfhlQ">
       <img alt="Slack" src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social">
@@ -26,178 +26,17 @@ Think TCPDump and Wireshark re-invented for Kubernetes.
 
 ![Simple UI](assets/mizu-ui.png)
 
-## Features
+## Quickstart and documentation
 
-- Simple and powerful CLI
-- Monitoring network traffic in real-time. Supported protocols:
-  - [HTTP/1.x](https://datatracker.ietf.org/doc/html/rfc2616) (REST, GraphQL, SOAP, etc.)
-  - [HTTP/2](https://datatracker.ietf.org/doc/html/rfc7540) (gRPC)
-  - [AMQP](https://www.rabbitmq.com/amqp-0-9-1-reference.html) (RabbitMQ, Apache Qpid, etc.)
-  - [Apache Kafka](https://kafka.apache.org/protocol)
-  - [Redis](https://redis.io/topics/protocol)
-- Works with Kubernetes APIs. No installation or code instrumentation
-- Rich filtering
+You can run Mizu on any Kubernetes cluster (version of 1.16.0 or higher) in a matter of seconds. See the [Mizu Getting Started Guide](https://getmizu.io/docs/) for how.
 
-## Requirements
+For more comprehensive documentation, start with the [docs](https://getmizu.io/docs/mizu/mizu-cli).
 
-A Kubernetes server version of 1.16.0 or higher is required.
+## Working in this repo
 
-## Download
+We ❤️ pull requests! See [CONTRIBUTING.md](docs/CONTRIBUTING.md) for info on contributing changes. <br />
+In the wiki you can find an intorduction to [mizu components](https://github.com/up9inc/mizu/wiki/Introduction-to-Mizu), and [development workflows](https://github.com/up9inc/mizu/wiki/Development-Workflows).
 
-Download Mizu for your platform and operating system
+## Code of Conduct
 
-### Latest Stable Release
-
-* for MacOS - Intel 
-```
-curl -Lo mizu \
-https://github.com/up9inc/mizu/releases/latest/download/mizu_darwin_amd64 \
-&& chmod 755 mizu
-```
- 
-* for Linux - Intel 64bit
-```
-curl -Lo mizu \
-https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 \
-&& chmod 755 mizu
-``` 
-
-SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page
-
-### Development (unstable) Build
-Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page
-
-## How to Run
-
-1. Find pods you'd like to tap to in your Kubernetes cluster
-2. Run `mizu tap` or `mizu tap PODNAME`
-3. Open browser on `http://localhost:8899` **or** as instructed in the CLI
-4. Watch the API traffic flowing
-5. Type ^C to stop
-
-## Examples
-
-Run `mizu help` for usage options
-
-To tap all pods in current namespace - 
-``` 
- $ kubectl get pods 
- NAME                            READY   STATUS    RESTARTS   AGE
- carts-66c77f5fbb-fq65r          2/2     Running   0          20m
- catalogue-5f4cb7cf5-7zrmn       2/2     Running   0          20m
- front-end-649fc5fd6-kqbtn       2/2     Running   0          20m
- ..
-
- $ mizu tap
- +carts-66c77f5fbb-fq65r
- +catalogue-5f4cb7cf5-7zrmn
- +front-end-649fc5fd6-kqbtn
- Web interface is now available at http://localhost:8899
- ^C
-```
-
-
-### To tap specific pod
-```bash
- $ kubectl get pods 
- NAME                            READY   STATUS    RESTARTS   AGE
- front-end-649fc5fd6-kqbtn       2/2     Running   0          7m
- ..
-
- $ mizu tap front-end-649fc5fd6-kqbtn
- +front-end-649fc5fd6-kqbtn
- Web interface is now available at http://localhost:8899
- ^C
-```
-
-### To tap multiple pods using regex
-```bash
- $ kubectl get pods 
- NAME                            READY   STATUS    RESTARTS   AGE
- carts-66c77f5fbb-fq65r          2/2     Running   0          20m
- catalogue-5f4cb7cf5-7zrmn       2/2     Running   0          20m
- front-end-649fc5fd6-kqbtn       2/2     Running   0          20m
- ..
-
- $ mizu tap "^ca.*"
- +carts-66c77f5fbb-fq65r
- +catalogue-5f4cb7cf5-7zrmn
- Web interface is now available at http://localhost:8899
- ^C
-```
-
-## Configuration
-
-Mizu can optionally work with a config file that can be provided as a CLI argument (using `--set config-path=<PATH>`) or if not provided, will be stored at ${HOME}/.mizu/config.yaml 
-In case of partial configuration defined, all other fields will be used with defaults <br />
-You can always override the defaults or config file with CLI flags
-
-To get the default config params run `mizu config` <br />
-To generate a new config file with default values use `mizu config -r`
-
-
-## Advanced Usage
-
-### Kubeconfig
-
-It is possible to change the kubeconfig path using `KUBECONFIG` environment variable or the command like flag
-with `--set kube-config-path=<PATH>`. </br >
-If both are not set - Mizu assumes that configuration is at `${HOME}/.kube/config`
-
-### Namespace-Restricted Mode
-
-Some users have permission to only manage resources in one particular namespace assigned to them
-By default `mizu tap` creates a new namespace `mizu` for all of its Kubernetes resources. In order to instead install
-Mizu in an existing namespace, set the `mizu-resources-namespace` config option
-
-If `mizu-resources-namespace` is set to a value other than the default `mizu`, Mizu will operate in a
-Namespace-Restricted mode. It will only tap pods in `mizu-resources-namespace`. This way Mizu only requires permissions
-to the namespace set by `mizu-resources-namespace`. The user must set the tapped namespace to the same namespace by
-using the `--namespace` flag or by setting `tap.namespaces` in the config file
-
-Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
-
-For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document
-
-### User agent filtering
-
-User-agent filtering (like health checks) - can be configured using command-line options:
-
-```shell
-$ mizu tap "^ca.*" --set tap.ignored-user-agents=kube-probe --set tap.ignored-user-agents=prometheus
-+carts-66c77f5fbb-fq65r
-+catalogue-5f4cb7cf5-7zrmn
-Web interface is now available at http://localhost:8899
-^C
-
-```
-Any request that contains `User-Agent` header with one of the specified values (`kube-probe` or `prometheus`) will not be captured
-
-### Traffic validation rules
-
-This feature allows you to define set of simple rules, and test the traffic against them.
-Such validation may test response for specific JSON fields, headers, etc.
-
-Please see [TRAFFIC RULES](docs/POLICY_RULES.md) page for more details and syntax.
-
-### OpenAPI Specification (OAS) Contract Monitoring
-
-An OAS/Swagger file can contain schemas under `parameters` and `responses` fields. With `--contract catalogue.yaml`
-CLI option, you can pass your API description to Mizu and the traffic will automatically be validated
-against the contracts.
-
-Please see [CONTRACT MONITORING](docs/CONTRACT_MONITORING.md) page for more details and syntax.
-
-### Configure proxy host 
-
-By default, mizu will be accessible via local host: 'http://localhost:8899', it is possible to change the host, for
-instance, to '0.0.0.0' which can grant access via machine IP address. This setting can be changed via command line
-flag `--set tap.proxy-host=<value>` or via config file:
-tap proxy-host: 0.0.0.0 and when changed it will support accessing by IP
-
-### Install Mizu standalone
-
-Mizu can be run detached from the cli using the install command: `mizu install`. This type of mizu instance will run
-indefinitely in the cluster.
-
-For more information please refer to [INSTALL STANDALONE](docs/INSTALL_STANDALONE.md)
+This project is for everyone. We ask that our users and contributors take a few minutes to review our [Code of Conduct](docs/CODE_OF_CONDUCT.md).

acceptanceTests/.snyk

@@ -0,0 +1,6 @@
+# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
+version: v1.14.0
+ignore:
+  SNYK-GOLANG-GITHUBCOMPKGSFTP-569475:
+    - '*':
+      reason: None Given

acceptanceTests/Makefile

@@ -1,2 +1,3 @@
 test: ## Run acceptance tests.
+	@npm install cypress@10.0.1 -y
 	@go test ./... -timeout 1h -v

acceptanceTests/config_test.go

@@ -23,19 +23,19 @@ func TestConfigRegenerate(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := getCliPath()
+	cliPath, cliPathErr := GetCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	configPath, configPathErr := getConfigPath()
+	configPath, configPathErr := GetConfigPath()
 	if configPathErr != nil {
 		t.Errorf("failed to get config path, err: %v", cliPathErr)
 		return
 	}
 
-	configCmdArgs := getDefaultConfigCommandArgs()
+	configCmdArgs := GetDefaultConfigCommandArgs()
 
 	configCmdArgs = append(configCmdArgs, "-r")
 
@@ -74,13 +74,13 @@ func TestConfigGuiPort(t *testing.T) {
 
 	for _, guiPort := range tests {
 		t.Run(fmt.Sprintf("%d", guiPort), func(t *testing.T) {
-			cliPath, cliPathErr := getCliPath()
+			cliPath, cliPathErr := GetCliPath()
 			if cliPathErr != nil {
 				t.Errorf("failed to get cli path, err: %v", cliPathErr)
 				return
 			}
 
-			configPath, configPathErr := getConfigPath()
+			configPath, configPathErr := GetConfigPath()
 			if configPathErr != nil {
 				t.Errorf("failed to get config path, err: %v", cliPathErr)
 				return
@@ -100,16 +100,16 @@ func TestConfigGuiPort(t *testing.T) {
 				return
 			}
 
-			tapCmdArgs := getDefaultTapCommandArgs()
+			tapCmdArgs := GetDefaultTapCommandArgs()
 
-			tapNamespace := getDefaultTapNamespace()
+			tapNamespace := GetDefaultTapNamespace()
 			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 			tapCmd := exec.Command(cliPath, tapCmdArgs...)
 			t.Logf("running command: %v", tapCmd.String())
 
 			t.Cleanup(func() {
-				if err := cleanupCommand(tapCmd); err != nil {
+				if err := CleanupCommand(tapCmd); err != nil {
 					t.Logf("failed to cleanup tap command, err: %v", err)
 				}
 
@@ -123,9 +123,9 @@ func TestConfigGuiPort(t *testing.T) {
 				return
 			}
 
-			apiServerUrl := getApiServerUrl(guiPort)
+			apiServerUrl := GetApiServerUrl(guiPort)
 
-			if err := waitTapPodsReady(apiServerUrl); err != nil {
+			if err := WaitTapPodsReady(apiServerUrl); err != nil {
 				t.Errorf("failed to start tap pods on time, err: %v", err)
 				return
 			}
@@ -147,13 +147,13 @@ func TestConfigSetGuiPort(t *testing.T) {
 
 	for _, guiPortStruct := range tests {
 		t.Run(fmt.Sprintf("%d", guiPortStruct.SetGuiPort), func(t *testing.T) {
-			cliPath, cliPathErr := getCliPath()
+			cliPath, cliPathErr := GetCliPath()
 			if cliPathErr != nil {
 				t.Errorf("failed to get cli path, err: %v", cliPathErr)
 				return
 			}
 
-			configPath, configPathErr := getConfigPath()
+			configPath, configPathErr := GetConfigPath()
 			if configPathErr != nil {
 				t.Errorf("failed to get config path, err: %v", cliPathErr)
 				return
@@ -173,9 +173,9 @@ func TestConfigSetGuiPort(t *testing.T) {
 				return
 			}
 
-			tapCmdArgs := getDefaultTapCommandArgs()
+			tapCmdArgs := GetDefaultTapCommandArgs()
 
-			tapNamespace := getDefaultTapNamespace()
+			tapNamespace := GetDefaultTapNamespace()
 			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 			tapCmdArgs = append(tapCmdArgs, "--set", fmt.Sprintf("tap.gui-port=%v", guiPortStruct.SetGuiPort))
@@ -184,7 +184,7 @@ func TestConfigSetGuiPort(t *testing.T) {
 			t.Logf("running command: %v", tapCmd.String())
 
 			t.Cleanup(func() {
-				if err := cleanupCommand(tapCmd); err != nil {
+				if err := CleanupCommand(tapCmd); err != nil {
 					t.Logf("failed to cleanup tap command, err: %v", err)
 				}
 
@@ -198,9 +198,9 @@ func TestConfigSetGuiPort(t *testing.T) {
 				return
 			}
 
-			apiServerUrl := getApiServerUrl(guiPortStruct.SetGuiPort)
+			apiServerUrl := GetApiServerUrl(guiPortStruct.SetGuiPort)
 
-			if err := waitTapPodsReady(apiServerUrl); err != nil {
+			if err := WaitTapPodsReady(apiServerUrl); err != nil {
 				t.Errorf("failed to start tap pods on time, err: %v", err)
 				return
 			}
@@ -222,13 +222,13 @@ func TestConfigFlagGuiPort(t *testing.T) {
 
 	for _, guiPortStruct := range tests {
 		t.Run(fmt.Sprintf("%d", guiPortStruct.FlagGuiPort), func(t *testing.T) {
-			cliPath, cliPathErr := getCliPath()
+			cliPath, cliPathErr := GetCliPath()
 			if cliPathErr != nil {
 				t.Errorf("failed to get cli path, err: %v", cliPathErr)
 				return
 			}
 
-			configPath, configPathErr := getConfigPath()
+			configPath, configPathErr := GetConfigPath()
 			if configPathErr != nil {
 				t.Errorf("failed to get config path, err: %v", cliPathErr)
 				return
@@ -248,9 +248,9 @@ func TestConfigFlagGuiPort(t *testing.T) {
 				return
 			}
 
-			tapCmdArgs := getDefaultTapCommandArgs()
+			tapCmdArgs := GetDefaultTapCommandArgs()
 
-			tapNamespace := getDefaultTapNamespace()
+			tapNamespace := GetDefaultTapNamespace()
 			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 			tapCmdArgs = append(tapCmdArgs, "-p", fmt.Sprintf("%v", guiPortStruct.FlagGuiPort))
@@ -259,7 +259,7 @@ func TestConfigFlagGuiPort(t *testing.T) {
 			t.Logf("running command: %v", tapCmd.String())
 
 			t.Cleanup(func() {
-				if err := cleanupCommand(tapCmd); err != nil {
+				if err := CleanupCommand(tapCmd); err != nil {
 					t.Logf("failed to cleanup tap command, err: %v", err)
 				}
 
@@ -273,9 +273,9 @@ func TestConfigFlagGuiPort(t *testing.T) {
 				return
 			}
 
-			apiServerUrl := getApiServerUrl(guiPortStruct.FlagGuiPort)
+			apiServerUrl := GetApiServerUrl(guiPortStruct.FlagGuiPort)
 
-			if err := waitTapPodsReady(apiServerUrl); err != nil {
+			if err := WaitTapPodsReady(apiServerUrl); err != nil {
 				t.Errorf("failed to start tap pods on time, err: %v", err)
 				return
 			}

acceptanceTests/create_user.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Create a user in Minikube cluster "minikube"
+# Create context for user
+# Usage:
+#  ./create_user.sh <username>
+
+set -e
+
+NEW_USERNAME=$1
+CERT_DIR="${HOME}/certs"
+KEY_FILE="${CERT_DIR}/${NEW_USERNAME}.key"
+CRT_FILE="${CERT_DIR}/${NEW_USERNAME}.crt"
+MINIKUBE_KEY_FILE="${HOME}/.minikube/ca.key"
+MINIKUBE_CRT_FILE="${HOME}/.minikube/ca.crt"
+DAYS=1
+
+echo "Creating user and context for username \"${NEW_USERNAME}\" in Minikube cluster"
+
+if ! command -v openssl &> /dev/null
+then
+	 echo "Installing openssl"
+	 sudo apt-get update
+	 sudo apt-get install openssl
+fi
+
+echo "Creating certificate for user \"${NEW_USERNAME}\""
+mkdir -p ${CERT_DIR}
+echo "Generating key \"${KEY_FILE}\""
+openssl genrsa -out "${KEY_FILE}" 2048
+echo "Generating crt \"${CRT_FILE}\""
+openssl req -new -key "${KEY_FILE}" -out "${CRT_FILE}" -subj "/CN=${NEW_USERNAME}/O=group1"
+openssl x509 -req -in "${CRT_FILE}" -CA "${MINIKUBE_CRT_FILE}" -CAkey "${MINIKUBE_KEY_FILE}" -CAcreateserial -out "${CRT_FILE}" -days $DAYS
+
+echo "Creating context for user \"${NEW_USERNAME}\""
+kubectl config set-credentials "${NEW_USERNAME}" --client-certificate="${CRT_FILE}" --client-key="${KEY_FILE}"
+kubectl config set-context "${NEW_USERNAME}" --cluster=minikube --user="${NEW_USERNAME}"

acceptanceTests/cypress.config.js

@@ -0,0 +1,31 @@
+const { defineConfig } = require('cypress')
+
+module.exports = defineConfig({
+  watchForFileChanges: false,
+  viewportWidth: 1920,
+  viewportHeight: 1080,
+  video: false,
+  screenshotOnRunFailure: false,
+  defaultCommandTimeout: 6000,
+  env: {
+    testUrl: 'http://localhost:8899/',
+    redactHeaderContent: 'User-Header[REDACTED]',
+    redactBodyContent: '{ "User": "[REDACTED]" }',
+    regexMaskingBodyContent: '[REDACTED]',
+    greenFilterColor: 'rgb(210, 250, 210)',
+    redFilterColor: 'rgb(250, 214, 220)',
+    bodyJsonClass: '.hljs',
+    mizuWidth: 1920,
+    normalMizuHeight: 1080,
+    hugeMizuHeight: 3500,
+  },
+  e2e: {
+    // We've imported your old cypress plugins here.
+    // You may want to clean this up later by importing these.
+    // setupNodeEvents(on, config) {
+    //   return require('./cypress/plugins/index.js')(on, config)
+    // },
+    specPattern: 'cypress/e2e/tests/*.js',
+    supportFile: false
+  },
+})

acceptanceTests/cypress.json

@@ -1,26 +0,0 @@
-{
-  "watchForFileChanges":false,
-  "viewportWidth": 1920,
-  "viewportHeight": 1080,
-  "video": false,
-  "screenshotOnRunFailure": false,
-  "defaultCommandTimeout": 6000,
-  "testFiles": [
-    "tests/GuiPort.js",
-    "tests/MultipleNamespaces.js",
-    "tests/Redact.js",
-    "tests/NoRedact.js",
-    "tests/Regex.js",
-    "tests/RegexMasking.js",
-    "tests/IgnoredUserAgents.js",
-    "tests/UiTest.js"
-  ],
-
-  "env": {
-    "testUrl": "http://localhost:8899/",
-    "redactHeaderContent": "User-Header[REDACTED]",
-    "redactBodyContent": "{ \"User\": \"[REDACTED]\" }",
-    "regexMaskingBodyContent": "[REDACTED]",
-    "minimumEntries": 25
-  }
-}

acceptanceTests/cypress/e2e/testHelpers/StatusBarHelper.js

@@ -1,8 +1,7 @@
 const columns = {podName : 1, namespace : 2, tapping : 3};
-const greenStatusImageSrc = '/static/media/success.662997eb.svg';
 
 function getDomPathInStatusBar(line, column) {
-    return `.expandedStatusBar > :nth-child(2) > > :nth-child(2) > :nth-child(${line}) > :nth-child(${column})`;
+    return `[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) > :nth-child(${line}) > :nth-child(${column})`;
 }
 
 export function checkLine(line, expectedValues) {
@@ -12,14 +11,14 @@ export function checkLine(line, expectedValues) {
 
         cy.get(getDomPathInStatusBar(line, columns.namespace)).invoke('text').then(namespaceValue => {
             expect(namespaceValue).to.equal(expectedValues.namespace);
-            cy.get(getDomPathInStatusBar(line, columns.tapping)).children().should('have.attr', 'src', greenStatusImageSrc);
+            cy.get(getDomPathInStatusBar(line, columns.tapping)).children().should('have.attr', 'src').and("match", /success.*\.svg/);
         });
     });
 }
 
 export function findLineAndCheck(expectedValues) {
-    cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(1)').then(pods => {
-        cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(2)').then(namespaces => {
+    cy.get('[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) > > :nth-child(1)').then(pods => {
+        cy.get('[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) > > :nth-child(2)').then(namespaces => {
             // organizing namespaces array
             const podObjectsArray = Object.values(pods ?? {});
             const namespacesObjectsArray = Object.values(namespaces ?? {});

acceptanceTests/cypress/e2e/testHelpers/TrafficHelper.js

@@ -0,0 +1,181 @@
+export const valueTabs = {
+    response: 'RESPONSE',
+    request: 'REQUEST',
+    none: null
+}
+
+export function isValueExistsInElement(shouldInclude, content, domPathToContainer){
+    it(`should ${shouldInclude ? '' : 'not'} include '${content}'`, function () {
+        cy.get(domPathToContainer).then(htmlText => {
+            const allTextString = htmlText.text();
+            if (allTextString.includes(content) !== shouldInclude)
+                throw new Error(`One of the containers part contains ${content}`)
+        });
+    });
+}
+
+export function resizeToHugeMizu() {
+    cy.viewport(Cypress.env('mizuWidth'), Cypress.env('hugeMizuHeight'));
+}
+
+export function resizeToNormalMizu() {
+    cy.viewport(Cypress.env('mizuWidth'), Cypress.env('normalMizuHeight'));
+}
+
+export function verifyMinimumEntries() {
+    const entriesSent = Cypress.env('entriesCount');
+    const minimumEntries = Math.round((0.75 * entriesSent));
+
+    it(`Making sure that mizu shows at least ${minimumEntries} entries`, function () {
+        cy.get('#total-entries').then(number => {
+            const getNum = () => {
+                return parseInt(number.text());
+            };
+
+            cy.wrap({num: getNum}).invoke('num').should('be.gt', minimumEntries);
+        });
+    });
+}
+
+export function leftTextCheck(entryId, path, expectedText) {
+    cy.get(`#list #entry-${entryId} ${path}`).invoke('text').should('eq', expectedText);
+}
+
+export function leftOnHoverCheck(entryId, path, filterName) {
+    cy.get(`#list #entry-${entryId} ${path}`).trigger('mouseover');
+    cy.get(`#list #entry-${entryId} [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(filterName));
+}
+
+export function rightTextCheck(path, expectedText) {
+    cy.get(`#rightSideContainer ${path}`).should('have.text', expectedText);
+}
+
+export function rightOnHoverCheck(path, expectedText) {
+    cy.get(`#rightSideContainer ${path}`).trigger('mouseover');
+    cy.get(`#rightSideContainer [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(expectedText));
+}
+
+export function checkFilterByMethod(funcDict) {
+    const {protocol, method, methodQuery, summary, summaryQuery, numberOfRecords} = funcDict;
+    const summaryDict = getSummaryDict(summary, summaryQuery);
+    const methodDict = getMethodDict(method, methodQuery);
+    const protocolDict = getProtocolDict(protocol.name, protocol.text);
+
+    it(`Testing the method: ${method}`, function () {
+        // applying filter
+        cy.get('.w-tc-editor-text').clear().type(methodQuery);
+        cy.get('[type="submit"]').click();
+        cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
+
+        waitForFetch(numberOfRecords);
+        pauseStream();
+
+        cy.get(`#list [id^=entry]`).then(elements => {
+            const listElmWithIdAttr = Object.values(elements);
+            let doneCheckOnFirst = false;
+
+            cy.get('#entries-length').invoke('text').then(len => {
+                listElmWithIdAttr.forEach(entry => {
+                    if (entry?.id && entry.id.match(RegExp(/entry-(\d{24})$/gm))) {
+                        const entryId = getEntryId(entry.id);
+
+                        leftTextCheck(entryId, methodDict.pathLeft, methodDict.expectedText);
+                        leftTextCheck(entryId, protocolDict.pathLeft, protocolDict.expectedTextLeft);
+                        if (summaryDict)
+                            leftTextCheck(entryId, summaryDict.pathLeft, summaryDict.expectedText);
+
+                        if (!doneCheckOnFirst) {
+                            deepCheck(funcDict, protocolDict, methodDict, entry);
+                            doneCheckOnFirst = true;
+                        }
+                    }
+                });
+            });
+        });
+    });
+}
+
+export const refreshWaitTimeout = 10000;
+
+export function waitForFetch(gt) {
+    cy.get('#entries-length', {timeout: refreshWaitTimeout}).should((el) => {
+        expect(parseInt(el.text().trim(), 10)).to.be.greaterThan(gt);
+    });
+}
+
+export function pauseStream() {
+    cy.get('#pause-icon').click();
+    cy.get('#pause-icon').should('not.be.visible');
+}
+
+
+export function getEntryId(id) {
+    // take the second part from the string (entry-<ID>)
+    return id.split('-')[1];
+}
+
+function deepCheck(generalDict, protocolDict, methodDict, entry) {
+    const entryId = getEntryId(entry.id);
+    const {summary, value} = generalDict;
+    const summaryDict = getSummaryDict(summary);
+
+    leftOnHoverCheck(entryId, methodDict.pathLeft, methodDict.expectedOnHover);
+    leftOnHoverCheck(entryId, protocolDict.pathLeft, protocolDict.expectedOnHover);
+    if (summaryDict)
+        leftOnHoverCheck(entryId, summaryDict.pathLeft, summaryDict.expectedOnHover);
+
+    cy.get(`#${entry.id}`).click();
+
+    rightTextCheck(methodDict.pathRight, methodDict.expectedText);
+    rightTextCheck(protocolDict.pathRight, protocolDict.expectedTextRight);
+    if (summaryDict)
+        rightTextCheck(summaryDict.pathRight, summaryDict.expectedText);
+
+    rightOnHoverCheck(methodDict.pathRight, methodDict.expectedOnHover);
+    rightOnHoverCheck(protocolDict.pathRight, protocolDict.expectedOnHover);
+    if (summaryDict)
+        rightOnHoverCheck(summaryDict.pathRight, summaryDict.expectedOnHover);
+
+    if (value) {
+        if (value.tab === valueTabs.response)
+            // temporary fix, change to some "data-cy" attribute,
+            // this will fix the issue that happen because we have "response:" in the header of the right side
+            cy.get('#rightSideContainer > :nth-child(3)').contains('Response').click();
+        cy.get(Cypress.env('bodyJsonClass')).then(text => {
+            expect(text.text()).to.match(value.regex)
+        });
+    }
+}
+
+function getSummaryDict(value, query) {
+    if (value) {
+        return {
+            pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+            pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+            expectedText: value,
+            expectedOnHover: query
+        };
+    }
+    else {
+        return null;
+    }
+}
+
+function getMethodDict(value, query) {
+    return {
+        pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+        pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+        expectedText: value,
+        expectedOnHover: query
+    };
+}
+
+function getProtocolDict(protocol, protocolText) {
+    return {
+        pathLeft: '> :nth-child(1) > :nth-child(1)',
+        pathRight: '> :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(1)',
+        expectedTextLeft: protocol.toUpperCase(),
+        expectedTextRight: protocolText,
+        expectedOnHover: protocol.toLowerCase()
+    };
+}

acceptanceTests/cypress/e2e/tests/GuiPort.js

@@ -1,5 +1,4 @@
 import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
-import {verifyMinimumEntries} from "../testHelpers/TrafficHelper";
 
 it('check', function () {
     const podName = Cypress.env('name'), namespace = Cypress.env('namespace');
@@ -9,8 +8,6 @@ it('check', function () {
     cy.visit(`http://localhost:${port}`);
     cy.wait('@statusTap').its('response.statusCode').should('match', /^2\d{2}/);
 
-    verifyMinimumEntries();
-
-    cy.get('.podsCount').trigger('mouseover');
+    cy.get(`[data-cy="expandedStatusBar"]`).trigger('mouseover',{force: true});
     findLineAndCheck(getExpectedDetailsDict(podName, namespace));
 });

acceptanceTests/cypress/e2e/tests/IgnoredUserAgents.js

@@ -0,0 +1,25 @@
+import {
+    isValueExistsInElement,
+    resizeToHugeMizu,
+} from "../testHelpers/TrafficHelper";
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+checkEntries();
+
+function checkEntries() {
+    it('checking all entries', function () {
+        cy.get('#entries-length').should('not.have.text', '0').then(() => {
+            resizeToHugeMizu();
+
+            cy.get('#list [id^=entry]').each(entryElement => {
+                entryElement.click();
+                cy.get('#tbody-Headers').should('be.visible');
+                isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
+            });
+        });
+    });
+}
+

acceptanceTests/cypress/e2e/tests/MultipleNamespaces.js

@@ -2,7 +2,7 @@ import {findLineAndCheck, getExpectedDetailsDict} from '../testHelpers/StatusBar
 
 it('opening', function () {
     cy.visit(Cypress.env('testUrl'));
-    cy.get('.podsCount').trigger('mouseover');
+    cy.get(`[data-cy="podsCountText"]`).trigger('mouseover');
 });
 
 [1, 2, 3].map(doItFunc);

acceptanceTests/cypress/e2e/tests/NoRedact.js

@@ -1,10 +1,8 @@
-import {isValueExistsInElement, verifyMinimumEntries} from '../testHelpers/TrafficHelper';
+import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
 });
 
-verifyMinimumEntries();
-
 isValueExistsInElement(false, Cypress.env('redactHeaderContent'), '#tbody-Headers');
-isValueExistsInElement(false, Cypress.env('redactBodyContent'), '.hljs');
+isValueExistsInElement(false, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/e2e/tests/Rabbit.js

@@ -0,0 +1,68 @@
+import {checkFilterByMethod, valueTabs,} from "../testHelpers/TrafficHelper";
+
+it('opening mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+const rabbitProtocolDetails = {name: 'AMQP', text: 'Advanced Message Queuing Protocol 0-9-1'};
+const numberOfRecords = 5;
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method: 'exchange declare',
+    methodQuery: 'request.method == "exchange declare"',
+    summary: 'exchange',
+    summaryQuery: 'request.exchange == "exchange"',
+    numberOfRecords: numberOfRecords,
+    value: null
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method: 'queue declare',
+    methodQuery: 'request.method == "queue declare"',
+    summary: 'queue',
+    summaryQuery: 'request.queue == "queue"',
+    numberOfRecords: numberOfRecords,
+    value: null
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method:  'queue bind',
+    methodQuery: 'request.method == "queue bind"',
+    summary: 'queue',
+    summaryQuery: 'request.queue == "queue"',
+    numberOfRecords: numberOfRecords,
+    value: null
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method: 'basic publish',
+    methodQuery: 'request.method == "basic publish"',
+    summary: 'exchange',
+    summaryQuery: 'request.exchange == "exchange"',
+    numberOfRecords: numberOfRecords,
+    value: {tab: valueTabs.request, regex: /^message$/mg}
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method:  'basic consume',
+    methodQuery: 'request.method == "basic consume"',
+    summary: 'queue',
+    summaryQuery: 'request.queue == "queue"',
+    numberOfRecords: numberOfRecords,
+    value: null
+});
+
+checkFilterByMethod({
+    protocol: rabbitProtocolDetails,
+    method:  'basic deliver',
+    methodQuery: 'request.method == "basic deliver"',
+    summary: 'exchange',
+    summaryQuery: 'request.queue == "exchange"',
+    numberOfRecords: numberOfRecords,
+    value: {tab: valueTabs.request, regex: /^message$/mg}
+});

acceptanceTests/cypress/e2e/tests/Redact.js

@@ -0,0 +1,8 @@
+import {isValueExistsInElement} from '../testHelpers/TrafficHelper';
+
+it('Loading Mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+isValueExistsInElement(true, Cypress.env('redactHeaderContent'), '#tbody-Headers');
+isValueExistsInElement(true, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/e2e/tests/Redis.js

@@ -0,0 +1,58 @@
+import {checkFilterByMethod, valueTabs,} from "../testHelpers/TrafficHelper";
+
+it('opening mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+const redisProtocolDetails = {name: 'redis', text: 'Redis Serialization Protocol'};
+const numberOfRecords = 5;
+
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
+    method: 'PING',
+    methodQuery: 'request.command == "PING"',
+    summary: null,
+    summaryQuery: '',
+    numberOfRecords: numberOfRecords,
+    value: null
+})
+
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
+    method: 'SET',
+    methodQuery: 'request.command == "SET"',
+    summary: 'key',
+    summaryQuery: 'request.key == "key"',
+    numberOfRecords: numberOfRecords,
+    value: {tab: valueTabs.request, regex: /^\[value, keepttl]$/mg}
+})
+
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
+    method: 'EXISTS',
+    methodQuery: 'request.command == "EXISTS"',
+    summary: 'key',
+    summaryQuery: 'request.key == "key"',
+    numberOfRecords: numberOfRecords,
+    value: {tab: valueTabs.response, regex: /^1$/mg}
+})
+
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
+    method: 'GET',
+    methodQuery: 'request.command == "GET"',
+    summary: 'key',
+    summaryQuery: 'request.key == "key"',
+    numberOfRecords: numberOfRecords,
+    value: {tab: valueTabs.response, regex: /^value$/mg}
+})
+
+checkFilterByMethod({
+    protocol: redisProtocolDetails,
+    method: 'DEL',
+    methodQuery: 'request.command == "DEL"',
+    summary: 'key',
+    summaryQuery: 'request.key == "key"',
+    numberOfRecords: numberOfRecords,
+    value: {tab: valueTabs.response, regex: /^1$|^0$/mg}
+})

acceptanceTests/cypress/e2e/tests/Regex.js

@@ -3,9 +3,9 @@ import {getExpectedDetailsDict, checkLine} from '../testHelpers/StatusBarHelper'
 
 it('opening', function () {
     cy.visit(Cypress.env('testUrl'));
-    cy.get('.podsCount').trigger('mouseover');
+    cy.get(`[data-cy="podsCountText"]`).trigger('mouseover');
 
-    cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) >').should('have.length', 1); // one line
+    cy.get('[data-cy="expandedStatusBar"] > :nth-child(2) > > :nth-child(2) >').should('have.length', 1); // one line
 
     checkLine(1, getExpectedDetailsDict(Cypress.env('name'), Cypress.env('namespace')));
 });

acceptanceTests/cypress/e2e/tests/RegexMasking.js

@@ -1,9 +1,7 @@
-import {isValueExistsInElement, verifyMinimumEntries} from "../testHelpers/TrafficHelper";
+import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
 
 it('Loading Mizu', function () {
     cy.visit(Cypress.env('testUrl'));
 });
 
-verifyMinimumEntries();
-
-isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), '.hljs');
+isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/e2e/tests/UiTest.js

@@ -0,0 +1,351 @@
+import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
+import {
+    getEntryId,
+    leftOnHoverCheck,
+    leftTextCheck,
+    resizeToHugeMizu,
+    resizeToNormalMizu,
+    rightOnHoverCheck,
+    rightTextCheck,
+    verifyMinimumEntries,
+    refreshWaitTimeout,
+    waitForFetch,
+    pauseStream
+} from "../testHelpers/TrafficHelper";
+
+const fullParam = Cypress.env('arrayDict'); // "Name:fooNamespace:barName:foo1Namespace:bar1"
+const podsArray = fullParam.split('Name:').slice(1); // ["fooNamespace:bar", "foo1Namespace:bar1"]
+podsArray.forEach((podStr, index) => {
+    const podAndNamespaceArr = podStr.split('Namespace:'); // [foo, bar] / [foo1, bar1]
+    podsArray[index] = getExpectedDetailsDict(podAndNamespaceArr[0], podAndNamespaceArr[1]);
+});
+
+it('opening mizu', function () {
+    cy.visit(Cypress.env('testUrl'));
+});
+
+verifyMinimumEntries();
+
+it('top bar check', function () {
+    cy.get(`[data-cy="podsCountText"]`).trigger('mouseover');
+    podsArray.map(findLineAndCheck);
+    cy.reload();
+});
+
+it('filtering guide check', function () {
+    cy.reload();
+    cy.get('[title="Open Filtering Guide (Cheatsheet)"]').click();
+    cy.get('#modal-modal-title').should('be.visible');
+    cy.get('[lang="en"]').click(0, 0);
+    cy.get('#modal-modal-title').should('not.exist');
+});
+
+it('right side sanity test', function () {
+    cy.get('#entryDetailedTitleElapsedTime').then(timeInMs => {
+        const time = timeInMs.text();
+        if (time < '0ms') {
+            throw new Error(`The time in the top line cannot be negative ${time}`);
+        }
+    });
+
+    // temporary fix, change to some "data-cy" attribute,
+    // this will fix the issue that happen because we have "response:" in the header of the right side
+    cy.get('#rightSideContainer > :nth-child(3)').contains('Response').click();
+
+    cy.get('#rightSideContainer [title="Status Code"]').then(status => {
+        const statusCode = status.text();
+        cy.contains('Status').parent().next().then(statusInDetails => {
+            const statusCodeInDetails = statusInDetails.text();
+
+            expect(statusCode).to.equal(statusCodeInDetails, 'The status code in the top line should match the status code in details');
+        });
+    });
+});
+
+checkIllegalFilter('invalid filter');
+
+checkFilter({
+    filter: 'http',
+    leftSidePath: '> :nth-child(1) > :nth-child(1)',
+    leftSideExpectedText: 'HTTP',
+    rightSidePath: '[title=HTTP]',
+    rightSideExpectedText: 'Hypertext Transfer Protocol -- HTTP/1.1',
+    applyByCtrlEnter: true,
+    numberOfRecords: 20,
+});
+
+checkFilter({
+    filter: 'response.status == 200',
+    leftSidePath: '[title="Status Code"]',
+    leftSideExpectedText: '200',
+    rightSidePath: '> :nth-child(2) [title="Status Code"]',
+    rightSideExpectedText: '200',
+    applyByCtrlEnter: false,
+    numberOfRecords: 20
+});
+
+if (Cypress.env('shouldCheckSrcAndDest')) {
+    serviceMapCheck();
+
+    checkFilter({
+        filter: 'src.name == ""',
+        leftSidePath: '[title="Source Name"]',
+        leftSideExpectedText: '[Unresolved]',
+        rightSidePath: '> :nth-child(2) [title="Source Name"]',
+        rightSideExpectedText: '[Unresolved]',
+        applyByCtrlEnter: false,
+        numberOfRecords: 20
+    });
+
+    checkFilter({
+        filter: `dst.name == "httpbin.mizu-tests"`,
+        leftSidePath: '> :nth-child(3) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
+        leftSideExpectedText: 'httpbin.mizu-tests',
+        rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
+        rightSideExpectedText: 'httpbin.mizu-tests',
+        applyByCtrlEnter: false,
+        numberOfRecords: 20
+    });
+}
+
+checkFilter({
+    filter: 'request.method == "GET"',
+    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+    leftSideExpectedText: 'GET',
+    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
+    rightSideExpectedText: 'GET',
+    applyByCtrlEnter: true,
+    numberOfRecords: 20
+});
+
+checkFilter({
+    filter: 'request.path == "/get"',
+    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+    leftSideExpectedText: '/get',
+    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
+    rightSideExpectedText: '/get',
+    applyByCtrlEnter: false,
+    numberOfRecords: 20
+});
+
+checkFilter({
+    filter: 'src.ip == "127.0.0.1"',
+    leftSidePath: '[title="Source IP"]',
+    leftSideExpectedText: '127.0.0.1',
+    rightSidePath: '> :nth-child(2) [title="Source IP"]',
+    rightSideExpectedText: '127.0.0.1',
+    applyByCtrlEnter: false,
+    numberOfRecords: 20
+});
+
+checkFilterNoResults('request.method == "POST"');
+
+function checkFilterNoResults(filterName) {
+    it(`checking the filter: ${filterName}. Expecting no results`, function () {
+        cy.get('#total-entries').then(number => {
+            const totalEntries = number.text();
+
+            // applying the filter
+            cy.get('.w-tc-editor-text').type(filterName);
+            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
+            cy.get('[type="submit"]').click();
+
+            // waiting for the entries number to load
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+
+            // the DOM should show 0 entries
+            cy.get('#entries-length').should('have.text', '0');
+
+            cy.get('[title="Fetch old records"]').click();
+            cy.get('#noMoreDataTop', {timeout: refreshWaitTimeout}).should('be.visible');
+            cy.get('#entries-length').should('have.text', '0'); // after loading all entries there should still be 0 entries
+
+            // reloading then waiting for the entries number to load
+            cy.reload();
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+        });
+    });
+}
+
+function checkIllegalFilter(illegalFilterName) {
+    it(`should show red search bar with the input: ${illegalFilterName}`, function () {
+        cy.reload();
+        cy.get('#total-entries').then(number => {
+            const totalEntries = number.text();
+
+            cy.get('.w-tc-editor-text').type(illegalFilterName);
+            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('redFilterColor'));
+            cy.get('[type="submit"]').click();
+
+            cy.get('[role="alert"]').should('be.visible');
+            cy.get('.w-tc-editor-text').clear();
+
+            // reloading then waiting for the entries number to load
+            cy.reload();
+            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
+        });
+    });
+}
+
+function checkFilter(filterDetails) {
+    const {
+        filter,
+        leftSidePath,
+        rightSidePath,
+        rightSideExpectedText,
+        leftSideExpectedText,
+        applyByCtrlEnter,
+        numberOfRecords
+    } = filterDetails;
+
+    const entriesForDeeperCheck = 5;
+
+    it(`checking the filter: ${filter}`, function () {
+        cy.get('.w-tc-editor-text').clear();
+        // applying the filter with alt+enter or with the button
+        cy.get('.w-tc-editor-text').type(`${filter}${applyByCtrlEnter ? '{ctrl+enter}' : ''}`);
+        cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
+        if (!applyByCtrlEnter)
+            cy.get('[type="submit"]').click();
+
+        waitForFetch(numberOfRecords);
+        pauseStream();
+
+        cy.get(`#list [id^=entry]`).last().then(elem => {
+            const element = elem[0];
+            const entryId = getEntryId(element.id);
+
+            // only one entry in DOM after filtering, checking all checks on it
+            leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
+            leftOnHoverCheck(entryId, leftSidePath, filter);
+
+            rightTextCheck(rightSidePath, rightSideExpectedText);
+            rightOnHoverCheck(rightSidePath, filter);
+            checkRightSideResponseBody();
+        });
+
+        resizeToHugeMizu();
+
+        // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
+        cy.get(`#list [id^=entry]`).each(elem => {
+            const element = elem[0];
+            let entryId = getEntryId(element.id);
+            leftTextCheck(entryId, leftSidePath, leftSideExpectedText);
+        });
+
+        // making the other 3 checks on the first X entries (longer time for each check)
+        deeperCheck(leftSidePath, rightSidePath, filter, rightSideExpectedText, entriesForDeeperCheck);
+
+        // reloading then waiting for the entries number to load
+        resizeToNormalMizu();
+        cy.reload();
+        waitForFetch(numberOfRecords);
+        pauseStream();
+    });
+}
+
+function deeperCheck(leftSidePath, rightSidePath, filterName, rightSideExpectedText, entriesNumToCheck) {
+    cy.get(`#list [id^=entry]`).each((element, index) => {
+        if (index < entriesNumToCheck) {
+            const entryId = getEntryId(element[0].id);
+            leftOnHoverCheck(entryId, leftSidePath, filterName);
+
+            cy.get(`#list #entry-${entryId}`).click();
+            rightTextCheck(rightSidePath, rightSideExpectedText);
+            rightOnHoverCheck(rightSidePath, filterName);
+        }
+    });
+}
+
+function checkRightSideResponseBody() {
+    // temporary fix, change to some "data-cy" attribute,
+    // this will fix the issue that happen because we have "response:" in the header of the right side
+    cy.get('#rightSideContainer > :nth-child(3)').contains('Response').click();
+    clickCheckbox('Decode Base64');
+
+    cy.get(`${Cypress.env('bodyJsonClass')}`).then(value => {
+        const encodedBody = value.text();
+        const decodedBody = atob(encodedBody);
+        const responseBody = JSON.parse(decodedBody);
+
+
+        const expectedJsonBody = {
+            args: RegExp({}),
+            url: RegExp('http://.*/get'),
+            headers: {
+                "User-Agent": RegExp('client'),
+                "Accept-Encoding": RegExp('gzip'),
+                "X-Forwarded-Uri": RegExp('/api/v1/namespaces/.*/services/.*/proxy/get')
+            }
+        };
+
+        const expectedStringInJsonBody = RegExp('/api/v1/namespaces/.*/services/.*/proxy/get');
+        
+
+        expect(responseBody.args).to.match(expectedJsonBody.args);
+        expect(responseBody.url).to.match(expectedJsonBody.url);
+        expect(responseBody.headers['User-Agent']).to.match(expectedJsonBody.headers['User-Agent']);
+        expect(responseBody.headers['Accept-Encoding']).to.match(expectedJsonBody.headers['Accept-Encoding']);
+        expect(responseBody.headers['X-Forwarded-Uri']).to.match(expectedJsonBody.headers['X-Forwarded-Uri']);
+
+        cy.get(`${Cypress.env('bodyJsonClass')}`).should('have.text', encodedBody);
+        cy.get(`[data-cy="lineNumbersCheckBoxInput"]`).should('be.disabled');
+
+        clickCheckbox('Decode Base64');
+        cy.get(`[data-cy="lineNumbersCheckBoxInput"]`).should('not.be.disabled');
+
+        cy.get(`${Cypress.env('bodyJsonClass')} > `).its('length').should('be.gt', 1).then(linesNum => {
+            cy.get(`${Cypress.env('bodyJsonClass')} > >`).its('length').should('be.gt', linesNum).then(jsonItemsNum => {
+                checkOnlyLineNumberes(jsonItemsNum, expectedStringInJsonBody);
+            });
+        });
+    });
+}
+
+function clickCheckbox(type) {
+    cy.contains(`${type}`).prev().children().click();
+}
+
+function checkOnlyLineNumberes(jsonItems, decodedText) {
+    cy.get(`${Cypress.env('bodyJsonClass')} > >`).should('have.length', jsonItems);
+    cy.get(`${Cypress.env('bodyJsonClass')} >`).contains(decodedText);
+}
+
+function serviceMapCheck() {
+    it('service map test', function () {
+        cy.intercept(`${Cypress.env('testUrl')}/servicemap/get`).as('serviceMapRequest');
+        cy.get('#total-entries').should('not.have.text', '0').then(() => {
+            cy.get('#total-entries').invoke('text').then(entriesNum => {
+                cy.get('[alt="service-map"]').click();
+                cy.wait('@serviceMapRequest').then(({response}) => {
+                    const body = response.body;
+                    const nodeParams = {
+                        destination: 'httpbin.mizu-tests',
+                        source: '127.0.0.1'
+                    };
+                    serviceMapAPICheck(body, parseInt(entriesNum), nodeParams);
+                    cy.reload();
+                });
+            });
+        });
+    });
+}
+
+function serviceMapAPICheck(body, entriesNum, nodeParams) {
+    const {nodes, edges} = body;
+
+    expect(nodes.length).to.equal(Object.keys(nodeParams).length, `Expected nodes count`);
+
+    expect(edges.some(edge => edge.source.name === nodeParams.source)).to.be.true;
+    expect(edges.some(edge => edge.destination.name === nodeParams.destination)).to.be.true;
+
+    let count = 0;
+    edges.forEach(edge => {
+        count += edge.count;
+        if (edge.destination.name === nodeParams.destination) {
+            expect(edge.source.name).to.equal(nodeParams.source);
+        }
+    });
+
+    expect(count).to.equal(entriesNum);
+}

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -1,30 +0,0 @@
-export function isValueExistsInElement(shouldInclude, content, domPathToContainer){
-    it(`should ${shouldInclude ? '' : 'not'} include '${content}'`, function () {
-        cy.get(domPathToContainer).then(htmlText => {
-            const allTextString = htmlText.text();
-            if (allTextString.includes(content) !== shouldInclude)
-                throw new Error(`One of the containers part contains ${content}`)
-        });
-    });
-}
-
-export function resizeToHugeMizu() {
-    cy.viewport(1920, 3500);
-}
-
-export function resizeToNormalMizu() {
-    cy.viewport(1920, 1080);
-}
-
-export function verifyMinimumEntries() {
-    const minimumEntries = Cypress.env('minimumEntries');
-    it(`Making sure that mizu shows at least ${minimumEntries} entries`, async function () {
-        cy.get('#total-entries').then(number => {
-            const getNum = () => {
-                const numOfEntries = number.text();
-                return parseInt(numOfEntries);
-            };
-            cy.wrap({ there: getNum }).invoke('there').should('be.gte', minimumEntries);
-        });
-    });
-}

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -1,34 +0,0 @@
-import {isValueExistsInElement, resizeToHugeMizu, verifyMinimumEntries} from "../testHelpers/TrafficHelper";
-
-it('Loading Mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-verifyMinimumEntries();
-
-checkEntries();
-
-function checkEntries() {
-    it('checking all entries', function () {
-        checkThatAllEntriesShown();
-        resizeToHugeMizu();
-
-        cy.get('#total-entries').then(number => {
-            const numOfEntries = parseInt(number.text());
-            [...Array(numOfEntries).keys()].map(checkEntry);
-        });
-    });
-}
-
-function checkThatAllEntriesShown() {
-    cy.get('#entries-length').then(number => {
-        if (number.text() === '1')
-            cy.get('[title="Fetch old records"]').click();
-    });
-}
-
-function checkEntry(entryIndex) {
-    cy.get(`#entry-${entryIndex}`).click();
-    cy.get('#tbody-Headers').should('be.visible');
-    isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
-}

acceptanceTests/cypress/integration/tests/Redact.js

@@ -1,10 +0,0 @@
-import {isValueExistsInElement, verifyMinimumEntries} from '../testHelpers/TrafficHelper';
-
-it('Loading Mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-verifyMinimumEntries();
-
-isValueExistsInElement(true, Cypress.env('redactHeaderContent'), '#tbody-Headers');
-isValueExistsInElement(true, Cypress.env('redactBodyContent'), '.hljs');

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -1,336 +0,0 @@
-import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
-import {resizeToHugeMizu, resizeToNormalMizu, verifyMinimumEntries} from "../testHelpers/TrafficHelper";
-const greenFilterColor = 'rgb(210, 250, 210)';
-const redFilterColor = 'rgb(250, 214, 220)';
-const refreshWaitTimeout = 10000;
-const bodyJsonClass = '.hljs';
-
-it('opening mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-verifyMinimumEntries();
-
-it('top bar check', function () {
-    const podName1 = 'httpbin', namespace1 = 'mizu-tests';
-    const podName2 = 'httpbin2', namespace2 = 'mizu-tests';
-
-    cy.get('.podsCount').trigger('mouseover');
-    findLineAndCheck(getExpectedDetailsDict(podName1, namespace1));
-    findLineAndCheck(getExpectedDetailsDict(podName2, namespace2));
-    cy.reload();
-});
-
-it('filtering guide check', function () {
-    cy.get('[title="Open Filtering Guide (Cheatsheet)"]').click();
-    cy.get('#modal-modal-title').should('be.visible');
-    cy.get('[lang="en"]').click(0, 0);
-    cy.get('#modal-modal-title').should('not.exist');
-});
-
-it('right side sanity test', function () {
-    cy.get('#entryDetailedTitleBodySize').then(sizeTopLine => {
-        const sizeOnTopLine = sizeTopLine.text().replace(' B', '');
-        cy.contains('Response').click();
-        cy.contains('Body Size (bytes)').parent().next().then(size => {
-            const bodySizeByDetails = size.text();
-            expect(sizeOnTopLine).to.equal(bodySizeByDetails, 'The body size in the top line should match the details in the response');
-
-            if (parseInt(bodySizeByDetails) < 0) {
-                throw new Error(`The body size cannot be negative. got the size: ${bodySizeByDetails}`)
-            }
-
-            cy.get('#entryDetailedTitleElapsedTime').then(timeInMs => {
-                const time = timeInMs.text();
-                if (time < '0ms') {
-                    throw new Error(`The time in the top line cannot be negative ${time}`);
-                }
-
-                cy.get('#rightSideContainer [title="Status Code"]').then(status => {
-                    const statusCode = status.text();
-                    cy.contains('Status').parent().next().then(statusInDetails => {
-                        const statusCodeInDetails = statusInDetails.text();
-
-                        expect(statusCode).to.equal(statusCodeInDetails, 'The status code in the top line should match the status code in details');
-                    });
-                });
-            });
-        });
-    });
-});
-
-checkIllegalFilter('invalid filter');
-
-checkFilter({
-    name: 'http',
-    leftSidePath: '> :nth-child(1) > :nth-child(1)',
-    leftSideExpectedText: 'HTTP',
-    rightSidePath: '[title=HTTP]',
-    rightSideExpectedText: 'Hypertext Transfer Protocol -- HTTP/1.1',
-    applyByEnter: true
-});
-
-checkFilter({
-    name: 'response.status == 200',
-    leftSidePath: '[title="Status Code"]',
-    leftSideExpectedText: '200',
-    rightSidePath: '> :nth-child(2) [title="Status Code"]',
-    rightSideExpectedText: '200',
-    applyByEnter: false
-});
-
-checkFilter({
-    name: 'src.name == ""',
-    leftSidePath: '[title="Source Name"]',
-    leftSideExpectedText: '[Unresolved]',
-    rightSidePath: '> :nth-child(2) [title="Source Name"]',
-    rightSideExpectedText: '[Unresolved]',
-    applyByEnter: false
-});
-
-checkFilter({
-    name: 'method == "GET"',
-    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
-    leftSideExpectedText: 'GET',
-    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
-    rightSideExpectedText: 'GET',
-    applyByEnter: true
-});
-
-checkFilter({
-    name: 'summary == "/get"',
-    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
-    leftSideExpectedText: '/get',
-    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
-    rightSideExpectedText: '/get',
-    applyByEnter: false
-});
-
-checkFilter({
-    name: 'dst.name == "httpbin.mizu-tests"',
-    leftSidePath: '> :nth-child(3) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
-    leftSideExpectedText: 'httpbin.mizu-tests',
-    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
-    rightSideExpectedText: 'httpbin.mizu-tests',
-    applyByEnter: false
-});
-
-checkFilter({
-    name: 'src.ip == "127.0.0.1"',
-    leftSidePath: '[title="Source IP"]',
-    leftSideExpectedText: '127.0.0.1',
-    rightSidePath: '> :nth-child(2) [title="Source IP"]',
-    rightSideExpectedText: '127.0.0.1',
-    applyByEnter: false
-});
-
-checkFilterNoResults('method == "POST"');
-
-function checkFilterNoResults(filterName) {
-    it(`checking the filter: ${filterName}. Expecting no results`, function () {
-        cy.get('#total-entries').then(number => {
-            const totalEntries = number.text();
-
-            // applying the filter
-            cy.get('.w-tc-editor-text').type(filterName);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', greenFilterColor);
-            cy.get('[type="submit"]').click();
-
-            // waiting for the entries number to load
-            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-
-            // the DOM should show 0 entries
-            cy.get('#entries-length').should('have.text', '0');
-
-            // going through every potential entry and verifies that it doesn't exist
-            [...Array(parseInt(totalEntries)).keys()].map(shouldNotExist);
-
-            cy.get('[title="Fetch old records"]').click();
-            cy.get('#noMoreDataTop', {timeout: refreshWaitTimeout}).should('be.visible');
-            cy.get('#entries-length').should('have.text', '0'); // after loading all entries there should still be 0 entries
-
-            // reloading then waiting for the entries number to load
-            cy.reload();
-            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-        });
-    });
-}
-
-function shouldNotExist(entryNum) {
-    cy.get(`entry-${entryNum}`).should('not.exist');
-}
-
-function checkIllegalFilter(illegalFilterName) {
-    it(`should show red search bar with the input: ${illegalFilterName}`, function () {
-        cy.get('#total-entries').then(number => {
-            const totalEntries = number.text();
-
-            cy.get('.w-tc-editor-text').type(illegalFilterName);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', redFilterColor);
-            cy.get('[type="submit"]').click();
-
-            cy.get('[role="alert"]').should('be.visible');
-            cy.get('.w-tc-editor-text').clear();
-
-            // reloading then waiting for the entries number to load
-            cy.reload();
-            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-        });
-    });
-}
-function checkFilter(filterDetails){
-    const {name, leftSidePath, rightSidePath, rightSideExpectedText, leftSideExpectedText, applyByEnter} = filterDetails;
-    const entriesForDeeperCheck = 5;
-
-    it(`checking the filter: ${name}`, function () {
-        cy.get('#total-entries').then(number => {
-            const totalEntries = number.text();
-
-            // checks the hover on the last entry (the only one in DOM at the beginning)
-            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
-
-            // applying the filter with alt+enter or with the button
-            cy.get('.w-tc-editor-text').type(`${name}${applyByEnter ? '{alt+enter}' : ''}`);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', greenFilterColor);
-            if (!applyByEnter)
-                cy.get('[type="submit"]').click();
-
-            // only one entry in DOM after filtering, checking all checks on it
-            leftTextCheck(totalEntries - 1, leftSidePath, leftSideExpectedText);
-            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
-            rightTextCheck(rightSidePath, rightSideExpectedText);
-            rightOnHoverCheck(rightSidePath, name);
-            checkRightSideResponseBody();
-
-            cy.get('[title="Fetch old records"]').click();
-            resizeToHugeMizu();
-
-            // waiting for the entries number to load
-            cy.get('#entries-length', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-
-            // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
-            [...Array(parseInt(totalEntries)).keys()].forEach(entryNum => {
-                leftTextCheck(entryNum, leftSidePath, leftSideExpectedText);
-            });
-
-            // making the other 3 checks on the first X entries (longer time for each check)
-            deeperChcek(leftSidePath, rightSidePath, name, leftSideExpectedText, rightSideExpectedText, entriesForDeeperCheck);
-
-            // reloading then waiting for the entries number to load
-            resizeToNormalMizu();
-            cy.reload();
-            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-        });
-    });
-}
-
-function deeperChcek(leftSidePath, rightSidePath, filterName, leftSideExpectedText, rightSideExpectedText, entriesNumToCheck) {
-    [...Array(entriesNumToCheck).keys()].forEach(entryNum => {
-        leftOnHoverCheck(entryNum, leftSidePath, filterName);
-
-        cy.get(`#list #entry-${entryNum}`).click();
-        rightTextCheck(rightSidePath, rightSideExpectedText);
-        rightOnHoverCheck(rightSidePath, filterName);
-    });
-}
-
-function leftTextCheck(entryNum, path, expectedText) {
-    cy.get(`#list #entry-${entryNum} ${path}`).invoke('text').should('eq', expectedText);
-}
-
-function leftOnHoverCheck(entryNum, path, filterName) {
-    cy.get(`#list #entry-${entryNum} ${path}`).trigger('mouseover');
-    cy.get(`#list #entry-${entryNum} .Queryable-Tooltip`).should('have.text', filterName);
-}
-
-function rightTextCheck(path, expectedText) {
-    cy.get(`.TrafficPage-Container > :nth-child(2) ${path}`).should('have.text', expectedText);
-}
-
-function rightOnHoverCheck(path, expectedText) {
-    cy.get(`.TrafficPage-Container > :nth-child(2) ${path}`).trigger('mouseover');
-    cy.get(`.TrafficPage-Container > :nth-child(2) .Queryable-Tooltip`).should('have.text', expectedText);
-}
-
-
-function checkRightSideResponseBody() {
-    cy.contains('Response').click();
-    clickCheckbox('Decode Base64');
-
-    cy.get(`${bodyJsonClass}`).then(value => {
-        const encodedBody = value.text();
-        const decodedBody = atob(encodedBody);
-        const responseBody = JSON.parse(decodedBody);
-
-        const expectdJsonBody = {
-            args: RegExp({}),
-            url: RegExp('http://.*/get'),
-            headers: {
-                "User-Agent": RegExp('[REDACTED]'),
-                "Accept-Encoding": RegExp('gzip'),
-                "X-Forwarded-Uri": RegExp('/api/v1/namespaces/.*/services/.*/proxy/get')
-            }
-        };
-
-        expect(responseBody.args).to.match(expectdJsonBody.args);
-        expect(responseBody.url).to.match(expectdJsonBody.url);
-        expect(responseBody.headers['User-Agent']).to.match(expectdJsonBody.headers['User-Agent']);
-        expect(responseBody.headers['Accept-Encoding']).to.match(expectdJsonBody.headers['Accept-Encoding']);
-        expect(responseBody.headers['X-Forwarded-Uri']).to.match(expectdJsonBody.headers['X-Forwarded-Uri']);
-
-        cy.get(`${bodyJsonClass}`).should('have.text', encodedBody);
-        clickCheckbox('Decode Base64');
-
-        cy.get(`${bodyJsonClass} > `).its('length').should('be.gt', 1).then(linesNum => {
-            cy.get(`${bodyJsonClass} > >`).its('length').should('be.gt', linesNum).then(jsonItemsNum => {
-                checkPrettyAndLineNums(jsonItemsNum, decodedBody);
-
-                clickCheckbox('Line numbers');
-                checkPrettyOrNothing(jsonItemsNum, decodedBody);
-
-                clickCheckbox('Pretty');
-                checkPrettyOrNothing(jsonItemsNum, decodedBody);
-
-                clickCheckbox('Line numbers');
-                checkOnlyLineNumberes(jsonItemsNum, decodedBody);
-            });
-        });
-    });
-}
-
-function clickCheckbox(type) {
-    cy.contains(`${type}`).prev().children().click();
-}
-
-function checkPrettyAndLineNums(jsonItemsLen, decodedBody) {
-    decodedBody = decodedBody.replaceAll(' ', '');
-    cy.get(`${bodyJsonClass} >`).then(elements => {
-        const lines = Object.values(elements);
-        lines.forEach((line, index) => {
-            if (line.getAttribute) {
-                const cleanLine = getCleanLine(line);
-                const currentLineFromDecodedText = decodedBody.substring(0, cleanLine.length);
-
-                expect(cleanLine).to.equal(currentLineFromDecodedText, `expected the text in line number ${index + 1} to match the text that generated by the base64 decoding`)
-
-                decodedBody = decodedBody.substring(cleanLine.length);
-            }
-        });
-    });
-}
-
-function getCleanLine(lineElement) {
-    return (lineElement.innerText.substring(0, lineElement.innerText.length - 1)).replaceAll(' ', '');
-}
-
-function checkPrettyOrNothing(jsonItems, decodedBody) {
-    cy.get(`${bodyJsonClass} > `).should('have.length', jsonItems).then(text => {
-        const json = text.text();
-        expect(json).to.equal(decodedBody);
-    });
-}
-
-function checkOnlyLineNumberes(jsonItems, decodedText) {
-    cy.get(`${bodyJsonClass} >`).should('have.length', 1).and('have.text', decodedText);
-    cy.get(`${bodyJsonClass} > >`).should('have.length', jsonItems)
-}

acceptanceTests/extensions_test.go

@@ -0,0 +1,243 @@
+package acceptanceTests
+
+import (
+	"context"
+	"fmt"
+	"github.com/go-redis/redis/v8"
+	amqp "github.com/rabbitmq/amqp091-go"
+	"os/exec"
+	"testing"
+	"time"
+)
+
+func TestRedis(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	ctx := context.Background()
+
+	kubernetesProvider, err := NewKubernetesProvider()
+	if err != nil {
+		t.Errorf("failed to create k8s provider, err %v", err)
+		return
+	}
+
+	redisExternalIp, err := kubernetesProvider.GetServiceExternalIp(ctx, DefaultNamespaceName, "redis")
+	if err != nil {
+		t.Errorf("failed to get redis external ip, err: %v", err)
+		return
+	}
+
+	rdb := redis.NewClient(&redis.Options{
+		Addr: fmt.Sprintf("%v:6379", redisExternalIp),
+	})
+
+	for i := 0; i < DefaultEntriesCount/5; i++ {
+		requestErr := rdb.Ping(ctx).Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < DefaultEntriesCount/5; i++ {
+		requestErr := rdb.Set(ctx, "key", "value", -1).Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < DefaultEntriesCount/5; i++ {
+		requestErr := rdb.Exists(ctx, "key").Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < DefaultEntriesCount/5; i++ {
+		requestErr := rdb.Get(ctx, "key").Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	for i := 0; i < DefaultEntriesCount/5; i++ {
+		requestErr := rdb.Del(ctx, "key").Err()
+		if requestErr != nil {
+			t.Errorf("failed to send redis request, err: %v", requestErr)
+			return
+		}
+	}
+
+	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/Redis.js\"")
+}
+
+func TestAmqp(t *testing.T) {
+
+	t.Skip("ignoredd for now because those tests are not stable")
+
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := GetCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := GetDefaultTapCommandArgs()
+
+	tapNamespace := GetDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := CleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
+
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	ctx := context.Background()
+
+	kubernetesProvider, err := NewKubernetesProvider()
+	if err != nil {
+		t.Errorf("failed to create k8s provider, err %v", err)
+		return
+	}
+
+	rabbitmqExternalIp, err := kubernetesProvider.GetServiceExternalIp(ctx, DefaultNamespaceName, "rabbitmq")
+	if err != nil {
+		t.Errorf("failed to get RabbitMQ external ip, err: %v", err)
+		return
+	}
+
+	conn, err := amqp.Dial(fmt.Sprintf("amqp://guest:guest@%v:5672/", rabbitmqExternalIp))
+	if err != nil {
+		t.Errorf("failed to connect to RabbitMQ, err: %v", err)
+		return
+	}
+	defer conn.Close()
+
+	// Temporary fix for missing amqp entries
+	time.Sleep(10 * time.Second)
+
+	for i := 0; i < DefaultEntriesCount/5; i++ {
+		ch, err := conn.Channel()
+		if err != nil {
+			t.Errorf("failed to open a channel, err: %v", err)
+			return
+		}
+
+		exchangeName := "exchange"
+		err = ch.ExchangeDeclare(exchangeName, "direct", true, false, false, false, nil)
+		if err != nil {
+			t.Errorf("failed to declare an exchange, err: %v", err)
+			return
+		}
+
+		q, err := ch.QueueDeclare("queue", true, false, false, false, nil)
+		if err != nil {
+			t.Errorf("failed to declare a queue, err: %v", err)
+			return
+		}
+
+		routingKey := "routing_key"
+		err = ch.QueueBind(q.Name, routingKey, exchangeName, false, nil)
+		if err != nil {
+			t.Errorf("failed to bind the queue, err: %v", err)
+			return
+		}
+
+		err = ch.Publish(exchangeName, routingKey, false, false,
+			amqp.Publishing{
+				DeliveryMode: amqp.Persistent,
+				ContentType:  "text/plain",
+				Body:         []byte("message"),
+			})
+		if err != nil {
+			t.Errorf("failed to publish a message, err: %v", err)
+			return
+		}
+
+		msgChan, err := ch.Consume(q.Name, "Consumer", true, false, false, false, nil)
+		if err != nil {
+			t.Errorf("failed to create a consumer, err: %v", err)
+			return
+		}
+
+		select {
+		case <-msgChan:
+			break
+		case <-time.After(3 * time.Second):
+			t.Errorf("failed to consume a message on time")
+			return
+		}
+
+		err = ch.ExchangeDelete(exchangeName, false, false)
+		if err != nil {
+			t.Errorf("failed to delete the exchange, err: %v", err)
+			return
+		}
+
+		_, err = ch.QueueDelete(q.Name, false, false, false)
+		if err != nil {
+			t.Errorf("failed to delete the queue, err: %v", err)
+			return
+		}
+
+		ch.Close()
+	}
+
+	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/Rabbit.js\"")
+}

acceptanceTests/go.mod

@@ -1,13 +1,58 @@
-module github.com/up9inc/mizu/tests
+module github.com/up9inc/mizu/acceptanceTests
 
-go 1.16
+go 1.17
 
 require (
-	github.com/gorilla/websocket v1.4.2
+	github.com/go-redis/redis/v8 v8.11.4
+	github.com/rabbitmq/amqp091-go v1.3.0
 	github.com/up9inc/mizu/shared v0.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	k8s.io/apimachinery v0.23.3
+	k8s.io/client-go v0.23.3
 )
 
+require (
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-logr/logr v1.2.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.7 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/up9inc/mizu/logger v0.0.0 // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/api v0.23.3 // indirect
+	k8s.io/klog/v2 v2.40.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf // indirect
+	k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
+	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
+replace github.com/up9inc/mizu/logger v0.0.0 => ../logger
+
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
+
+replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../tap/dbgctl

acceptanceTests/logs_test.go

@@ -11,22 +11,22 @@ func TestLogs(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := getCliPath()
+	cliPath, cliPathErr := GetCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := getDefaultTapCommandArgs()
+	tapCmdArgs := GetDefaultTapCommandArgs()
 
-	tapNamespace := getDefaultTapNamespace()
+	tapNamespace := GetDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := cleanupCommand(tapCmd); err != nil {
+		if err := CleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -36,14 +36,14 @@ func TestLogs(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
 
-	if err := waitTapPodsReady(apiServerUrl); err != nil {
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	logsCmdArgs := getDefaultLogsCommandArgs()
+	logsCmdArgs := GetDefaultLogsCommandArgs()
 
 	logsCmd := exec.Command(cliPath, logsCmdArgs...)
 	t.Logf("running command: %v", logsCmd.String())
@@ -58,7 +58,7 @@ func TestLogs(t *testing.T) {
 		return
 	}
 
-	logsPath, logsPathErr := getLogsPath()
+	logsPath, logsPathErr := GetLogsPath()
 	if logsPathErr != nil {
 		t.Errorf("failed to get logs path, err: %v", logsPathErr)
 		return
@@ -112,22 +112,22 @@ func TestLogsPath(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := getCliPath()
+	cliPath, cliPathErr := GetCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := getDefaultTapCommandArgs()
+	tapCmdArgs := GetDefaultTapCommandArgs()
 
-	tapNamespace := getDefaultTapNamespace()
+	tapNamespace := GetDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := cleanupCommand(tapCmd); err != nil {
+		if err := CleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -137,14 +137,14 @@ func TestLogsPath(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
 
-	if err := waitTapPodsReady(apiServerUrl); err != nil {
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	logsCmdArgs := getDefaultLogsCommandArgs()
+	logsCmdArgs := GetDefaultLogsCommandArgs()
 
 	logsPath := "../logs.zip"
 	logsCmdArgs = append(logsCmdArgs, "-f", logsPath)

acceptanceTests/setup.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
+set -e
 
 PREFIX=$HOME/local/bin
 VERSION=v1.22.0
+TUNNEL_LOG="tunnel.log"
+PROXY_LOG="proxy.log"
 
 echo "Attempting to install minikube and assorted tools to $PREFIX"
 
@@ -11,7 +14,7 @@ if ! [ -x "$(command -v kubectl)" ]; then
   chmod +x kubectl
   mv kubectl "$PREFIX"
 else
-  echo "kubetcl is already installed"
+  echo "kubectl is already installed"
 fi
 
 if ! [ -x "$(command -v minikube)" ]; then
@@ -24,32 +27,58 @@ else
 fi
 
 echo "Starting minikube..."
-minikube start
+minikube start --cpus 2 --memory 6946
 
 echo "Creating mizu tests namespaces"
-kubectl create namespace mizu-tests
-kubectl create namespace mizu-tests2
+kubectl create namespace mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl create namespace mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating httpbin deployments"
-kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests
-kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
-kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2
+kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
+
+echo "Creating redis deployment"
+kubectl create deployment redis --image=redis -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+
+echo "Creating rabbitmq deployment"
+kubectl create deployment rabbitmq --image=rabbitmq -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Creating httpbin services"
-kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
-kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
-kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2
+kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
+
+echo "Creating redis service"
+kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
+
+echo "Creating rabbitmq service"
+kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
 
 echo "Starting proxy"
-kubectl proxy --port=8080 &
+rm -f ${PROXY_LOG}
+kubectl proxy --port=8080 > ${PROXY_LOG} &
+PID1=$!
+echo "kubectl proxy process id is ${PID1} and log of proxy in ${PROXY_LOG}"
 
-echo "Setting minikube docker env"
-eval $(minikube docker-env)
+if [[ -z "${CI}" ]]; then
+  echo "Setting env var of mizu ci image"
+  export MIZU_CI_IMAGE="mizu/ci:0.0"
+  echo "Build agent image"
+  docker build -t "${MIZU_CI_IMAGE}" .
+else
+  echo "not building docker image in CI because it is created as separate step"
+fi
 
-echo "Build agent image"
-make build-docker-ci
+minikube image load "${MIZU_CI_IMAGE}"
 
 echo "Build cli"
-make build-cli-ci
+cd cli && make build GIT_BRANCH=ci SUFFIX=ci
+
+echo "Starting tunnel"
+rm -f ${TUNNEL_LOG}
+minikube tunnel > ${TUNNEL_LOG} &
+PID2=$!
+echo "Minikube tunnel process id is ${PID2} and log of tunnel in ${TUNNEL_LOG}"

acceptanceTests/tap_test.go

@@ -14,6 +14,10 @@ import (
 )
 
 func TestTap(t *testing.T) {
+	basicTapTest(t, false)
+}
+
+func basicTapTest(t *testing.T, shouldCheckSrcAndDest bool, extraArgs... string) {
 	if testing.Short() {
 		t.Skip("ignored acceptance test")
 	}
@@ -22,22 +26,24 @@ func TestTap(t *testing.T) {
 
 	for _, entriesCount := range tests {
 		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
-			cliPath, cliPathErr := getCliPath()
+			cliPath, cliPathErr := GetCliPath()
 			if cliPathErr != nil {
 				t.Errorf("failed to get cli path, err: %v", cliPathErr)
 				return
 			}
 
-			tapCmdArgs := getDefaultTapCommandArgs()
+			tapCmdArgs := GetDefaultTapCommandArgs()
 
-			tapNamespace := getDefaultTapNamespace()
+			tapNamespace := GetDefaultTapNamespace()
 			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
+			tapCmdArgs = append(tapCmdArgs, extraArgs...)
+
 			tapCmd := exec.Command(cliPath, tapCmdArgs...)
 			t.Logf("running command: %v", tapCmd.String())
 
 			t.Cleanup(func() {
-				if err := cleanupCommand(tapCmd); err != nil {
+				if err := CleanupCommand(tapCmd); err != nil {
 					t.Logf("failed to cleanup tap command, err: %v", err)
 				}
 			})
@@ -47,22 +53,33 @@ func TestTap(t *testing.T) {
 				return
 			}
 
-			apiServerUrl := getApiServerUrl(defaultApiServerPort)
+			apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
 
-			if err := waitTapPodsReady(apiServerUrl); err != nil {
+			if err := WaitTapPodsReady(apiServerUrl); err != nil {
 				t.Errorf("failed to start tap pods on time, err: %v", err)
 				return
 			}
 
-			proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+			proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
 			for i := 0; i < entriesCount; i++ {
-				if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+				if _, requestErr := ExecuteHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
 					t.Errorf("failed to send proxy request, err: %v", requestErr)
 					return
 				}
 			}
 
-			runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/UiTest.js\"")
+			expectedPods := []PodDescriptor{
+				{Name: "httpbin", Namespace: "mizu-tests"},
+				{Name: "httpbin2", Namespace: "mizu-tests"},
+			}
+
+			var expectedPodsStr string
+			for i := 0; i < len(expectedPods); i++ {
+				expectedPodsStr += fmt.Sprintf("Name:%vNamespace:%v", expectedPods[i].Name, expectedPods[i].Namespace)
+			}
+
+			RunCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/e2e/tests/UiTest.js\" --env entriesCount=%d,arrayDict=%v,shouldCheckSrcAndDest=%v",
+				entriesCount, expectedPodsStr, shouldCheckSrcAndDest))
 		})
 	}
 }
@@ -76,15 +93,15 @@ func TestTapGuiPort(t *testing.T) {
 
 	for _, guiPort := range tests {
 		t.Run(fmt.Sprintf("%d", guiPort), func(t *testing.T) {
-			cliPath, cliPathErr := getCliPath()
+			cliPath, cliPathErr := GetCliPath()
 			if cliPathErr != nil {
 				t.Errorf("failed to get cli path, err: %v", cliPathErr)
 				return
 			}
 
-			tapCmdArgs := getDefaultTapCommandArgs()
+			tapCmdArgs := GetDefaultTapCommandArgs()
 
-			tapNamespace := getDefaultTapNamespace()
+			tapNamespace := GetDefaultTapNamespace()
 			tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 			tapCmdArgs = append(tapCmdArgs, "-p", fmt.Sprintf("%d", guiPort))
@@ -93,7 +110,7 @@ func TestTapGuiPort(t *testing.T) {
 			t.Logf("running command: %v", tapCmd.String())
 
 			t.Cleanup(func() {
-				if err := cleanupCommand(tapCmd); err != nil {
+				if err := CleanupCommand(tapCmd); err != nil {
 					t.Logf("failed to cleanup tap command, err: %v", err)
 				}
 			})
@@ -103,22 +120,22 @@ func TestTapGuiPort(t *testing.T) {
 				return
 			}
 
-			apiServerUrl := getApiServerUrl(guiPort)
+			apiServerUrl := GetApiServerUrl(guiPort)
 
-			if err := waitTapPodsReady(apiServerUrl); err != nil {
+			if err := WaitTapPodsReady(apiServerUrl); err != nil {
 				t.Errorf("failed to start tap pods on time, err: %v", err)
 				return
 			}
 
-			proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
-			for i := 0; i < defaultEntriesCount; i++ {
-				if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+			proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
+			for i := 0; i < DefaultEntriesCount; i++ {
+				if _, requestErr := ExecuteHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
 					t.Errorf("failed to send proxy request, err: %v", requestErr)
 					return
 				}
 			}
 
-			runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env name=%v,namespace=%v,port=%d",
+			RunCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/e2e/tests/GuiPort.js\" --env name=%v,namespace=%v,port=%d",
 				"httpbin", "mizu-tests", guiPort))
 		})
 	}
@@ -135,20 +152,20 @@ func TestTapAllNamespaces(t *testing.T) {
 		{Name: "httpbin", Namespace: "mizu-tests2"},
 	}
 
-	cliPath, cliPathErr := getCliPath()
+	cliPath, cliPathErr := GetCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := getDefaultTapCommandArgs()
+	tapCmdArgs := GetDefaultTapCommandArgs()
 	tapCmdArgs = append(tapCmdArgs, "-A")
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := cleanupCommand(tapCmd); err != nil {
+		if err := CleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -158,14 +175,14 @@ func TestTapAllNamespaces(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
 
-	if err := waitTapPodsReady(apiServerUrl); err != nil {
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
+	RunCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/e2e/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
 		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
 }
 
@@ -180,13 +197,13 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		{Name: "httpbin", Namespace: "mizu-tests2"},
 	}
 
-	cliPath, cliPathErr := getCliPath()
+	cliPath, cliPathErr := GetCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := getDefaultTapCommandArgs()
+	tapCmdArgs := GetDefaultTapCommandArgs()
 	var namespacesCmd []string
 	for _, expectedPod := range expectedPods {
 		namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace)
@@ -197,7 +214,7 @@ func TestTapMultipleNamespaces(t *testing.T) {
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := cleanupCommand(tapCmd); err != nil {
+		if err := CleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -207,14 +224,14 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
 
-	if err := waitTapPodsReady(apiServerUrl); err != nil {
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
+	RunCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/e2e/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
 		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
 }
 
@@ -228,22 +245,22 @@ func TestTapRegex(t *testing.T) {
 		{Name: regexPodName, Namespace: "mizu-tests"},
 	}
 
-	cliPath, cliPathErr := getCliPath()
+	cliPath, cliPathErr := GetCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := getDefaultTapCommandArgsWithRegex(regexPodName)
+	tapCmdArgs := GetDefaultTapCommandArgsWithRegex(regexPodName)
 
-	tapNamespace := getDefaultTapNamespace()
+	tapNamespace := GetDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := cleanupCommand(tapCmd); err != nil {
+		if err := CleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -253,14 +270,14 @@ func TestTapRegex(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
 
-	if err := waitTapPodsReady(apiServerUrl); err != nil {
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Regex.js\" --env name=%v,namespace=%v",
+	RunCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/e2e/tests/Regex.js\" --env name=%v,namespace=%v",
 		expectedPods[0].Name, expectedPods[0].Namespace))
 }
 
@@ -269,15 +286,15 @@ func TestTapDryRun(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := getCliPath()
+	cliPath, cliPathErr := GetCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := getDefaultTapCommandArgs()
+	tapCmdArgs := GetDefaultTapCommandArgs()
 
-	tapNamespace := getDefaultTapNamespace()
+	tapNamespace := GetDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	tapCmdArgs = append(tapCmdArgs, "--dry-run")
@@ -301,7 +318,7 @@ func TestTapDryRun(t *testing.T) {
 	}()
 
 	go func() {
-		time.Sleep(shortRetriesCount * time.Second)
+		time.Sleep(ShortRetriesCount * time.Second)
 		resultChannel <- "fail"
 	}()
 
@@ -316,22 +333,23 @@ func TestTapRedact(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := getCliPath()
+	cliPath, cliPathErr := GetCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := getDefaultTapCommandArgs()
+	tapCmdArgs := GetDefaultTapCommandArgs()
 
-	tapNamespace := getDefaultTapNamespace()
+	tapNamespace := GetDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+	tapCmdArgs = append(tapCmdArgs, "--redact")
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := cleanupCommand(tapCmd); err != nil {
+		if err := CleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -341,24 +359,24 @@ func TestTapRedact(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
 
-	if err := waitTapPodsReady(apiServerUrl); err != nil {
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
 	requestHeaders := map[string]string{"User-Header": "Mizu"}
 	requestBody := map[string]string{"User": "Mizu"}
-	for i := 0; i < defaultEntriesCount; i++ {
-		if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
+	for i := 0; i < DefaultEntriesCount; i++ {
+		if _, requestErr := ExecuteHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/Redact.js\"")
+	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/Redact.js\"")
 }
 
 func TestTapNoRedact(t *testing.T) {
@@ -366,24 +384,22 @@ func TestTapNoRedact(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := getCliPath()
+	cliPath, cliPathErr := GetCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := getDefaultTapCommandArgs()
+	tapCmdArgs := GetDefaultTapCommandArgs()
 
-	tapNamespace := getDefaultTapNamespace()
+	tapNamespace := GetDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
-	tapCmdArgs = append(tapCmdArgs, "--no-redact")
-
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := cleanupCommand(tapCmd); err != nil {
+		if err := CleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -393,24 +409,24 @@ func TestTapNoRedact(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
 
-	if err := waitTapPodsReady(apiServerUrl); err != nil {
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
 	requestHeaders := map[string]string{"User-Header": "Mizu"}
 	requestBody := map[string]string{"User": "Mizu"}
-	for i := 0; i < defaultEntriesCount; i++ {
-		if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
+	for i := 0; i < DefaultEntriesCount; i++ {
+		if _, requestErr := ExecuteHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/NoRedact.js\"")
+	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/NoRedact.js\"")
 }
 
 func TestTapRegexMasking(t *testing.T) {
@@ -418,24 +434,26 @@ func TestTapRegexMasking(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := getCliPath()
+	cliPath, cliPathErr := GetCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := getDefaultTapCommandArgs()
+	tapCmdArgs := GetDefaultTapCommandArgs()
 
-	tapNamespace := getDefaultTapNamespace()
+	tapNamespace := GetDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
+	tapCmdArgs = append(tapCmdArgs, "--redact")
+
 	tapCmdArgs = append(tapCmdArgs, "-r", "Mizu")
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := cleanupCommand(tapCmd); err != nil {
+		if err := CleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -445,23 +463,23 @@ func TestTapRegexMasking(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
 
-	if err := waitTapPodsReady(apiServerUrl); err != nil {
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
-	for i := 0; i < defaultEntriesCount; i++ {
+	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
+	for i := 0; i < DefaultEntriesCount; i++ {
 		response, requestErr := http.Post(fmt.Sprintf("%v/post", proxyUrl), "text/plain", bytes.NewBufferString("Mizu"))
-		if _, requestErr = executeHttpRequest(response, requestErr); requestErr != nil {
+		if _, requestErr = ExecuteHttpRequest(response, requestErr); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	runCypressTests(t, "npx cypress run --spec \"cypress/integration/tests/RegexMasking.js\"")
+	RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/RegexMasking.js\"")
 
 }
 
@@ -470,15 +488,15 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := getCliPath()
+	cliPath, cliPathErr := GetCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := getDefaultTapCommandArgs()
+	tapCmdArgs := GetDefaultTapCommandArgs()
 
-	tapNamespace := getDefaultTapNamespace()
+	tapNamespace := GetDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	ignoredUserAgentValue := "ignore"
@@ -488,7 +506,7 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 	t.Logf("running command: %v", tapCmd.String())
 
 	t.Cleanup(func() {
-		if err := cleanupCommand(tapCmd); err != nil {
+		if err := CleanupCommand(tapCmd); err != nil {
 			t.Logf("failed to cleanup tap command, err: %v", err)
 		}
 	})
@@ -498,32 +516,32 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
 
-	if err := waitTapPodsReady(apiServerUrl); err != nil {
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
+	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
 
 	ignoredUserAgentCustomHeader := "Ignored-User-Agent"
 	headers := map[string]string{"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
-	for i := 0; i < defaultEntriesCount; i++ {
-		if _, requestErr := executeHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil {
+	for i := 0; i < DefaultEntriesCount; i++ {
+		if _, requestErr := ExecuteHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	for i := 0; i < defaultEntriesCount; i++ {
-		if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
+	for i := 0; i < DefaultEntriesCount; i++ {
+		if _, requestErr := ExecuteHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/IgnoredUserAgents.js\"")
+	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/IgnoredUserAgents.js\"")
 }
 
 func TestTapDumpLogs(t *testing.T) {
@@ -531,15 +549,15 @@ func TestTapDumpLogs(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	cliPath, cliPathErr := getCliPath()
+	cliPath, cliPathErr := GetCliPath()
 	if cliPathErr != nil {
 		t.Errorf("failed to get cli path, err: %v", cliPathErr)
 		return
 	}
 
-	tapCmdArgs := getDefaultTapCommandArgs()
+	tapCmdArgs := GetDefaultTapCommandArgs()
 
-	tapNamespace := getDefaultTapNamespace()
+	tapNamespace := GetDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
 
 	tapCmdArgs = append(tapCmdArgs, "--set", "dump-logs=true")
@@ -552,19 +570,19 @@ func TestTapDumpLogs(t *testing.T) {
 		return
 	}
 
-	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
 
-	if err := waitTapPodsReady(apiServerUrl); err != nil {
+	if err := WaitTapPodsReady(apiServerUrl); err != nil {
 		t.Errorf("failed to start tap pods on time, err: %v", err)
 		return
 	}
 
-	if err := cleanupCommand(tapCmd); err != nil {
+	if err := CleanupCommand(tapCmd); err != nil {
 		t.Errorf("failed to cleanup tap command, err: %v", err)
 		return
 	}
 
-	mizuFolderPath, mizuPathErr := getMizuFolderPath()
+	mizuFolderPath, mizuPathErr := GetMizuFolderPath()
 	if mizuPathErr != nil {
 		t.Errorf("failed to get mizu folder path, err: %v", mizuPathErr)
 		return
@@ -632,3 +650,44 @@ func TestTapDumpLogs(t *testing.T) {
 		return
 	}
 }
+
+func TestIpResolving(t *testing.T) {
+	namespace := AllNamespaces
+
+	t.Log("add permissions for ip-resolution for current user")
+	if err := ApplyKubeFilesForTest(
+		t,
+		"minikube",
+		namespace,
+		"../cli/cmd/permissionFiles/permissions-all-namespaces-ip-resolution-optional.yaml",
+	); err != nil {
+		t.Errorf("failed to create k8s permissions, %v", err)
+		return
+	}
+
+	basicTapTest(t, true)
+}
+
+func TestRestrictedMode(t *testing.T) {
+	namespace := "mizu-tests"
+
+	t.Log("creating permissions for restricted user")
+	if err := ApplyKubeFilesForTest(
+		t,
+		"minikube",
+		namespace,
+		"../cli/cmd/permissionFiles/permissions-ns-tap.yaml",
+	); err != nil {
+		t.Errorf("failed to create k8s permissions, %v", err)
+		return
+	}
+
+	t.Log("switching k8s context to user")
+	if err := SwitchKubeContextForTest(t, "user-with-restricted-access"); err != nil {
+		t.Errorf("failed to switch k8s context, %v", err)
+		return
+	}
+
+	extraArgs := []string{"--set", fmt.Sprintf("mizu-resources-namespace=%s", namespace)}
+	t.Run("basic tap", func (testingT *testing.T) {basicTapTest(testingT, false, extraArgs...)})
+}

acceptanceTests/testsUtils.go

@@ -2,6 +2,7 @@ package acceptanceTests
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -9,22 +10,29 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strings"
 	"syscall"
 	"testing"
 	"time"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/homedir"
+
 	"github.com/up9inc/mizu/shared"
 )
 
 const (
-	longRetriesCount      = 100
-	shortRetriesCount     = 10
-	defaultApiServerPort  = shared.DefaultApiServerPort
-	defaultNamespaceName  = "mizu-tests"
-	defaultServiceName    = "httpbin"
-	defaultEntriesCount   = 50
-	waitAfterTapPodsReady = 3 * time.Second
+	LongRetriesCount      = 100
+	ShortRetriesCount     = 10
+	DefaultApiServerPort  = shared.DefaultApiServerPort
+	DefaultNamespaceName  = "mizu-tests"
+	DefaultServiceName    = "httpbin"
+	DefaultEntriesCount   = 50
+	WaitAfterTapPodsReady = 3 * time.Second
+	AllNamespaces         = ""
 )
 
 type PodDescriptor struct {
@@ -32,7 +40,7 @@ type PodDescriptor struct {
 	Namespace string
 }
 
-func getCliPath() (string, error) {
+func GetCliPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {
 		return "", filePathErr
@@ -42,7 +50,7 @@ func getCliPath() (string, error) {
 	return cliPath, nil
 }
 
-func getMizuFolderPath() (string, error) {
+func GetMizuFolderPath() (string, error) {
 	home, homeDirErr := os.UserHomeDir()
 	if homeDirErr != nil {
 		return "", homeDirErr
@@ -51,8 +59,8 @@ func getMizuFolderPath() (string, error) {
 	return path.Join(home, ".mizu"), nil
 }
 
-func getConfigPath() (string, error) {
-	mizuFolderPath, mizuPathError := getMizuFolderPath()
+func GetConfigPath() (string, error) {
+	mizuFolderPath, mizuPathError := GetMizuFolderPath()
 	if mizuPathError != nil {
 		return "", mizuPathError
 	}
@@ -60,72 +68,210 @@ func getConfigPath() (string, error) {
 	return path.Join(mizuFolderPath, "config.yaml"), nil
 }
 
-func getProxyUrl(namespace string, service string) string {
+func GetProxyUrl(namespace string, service string) string {
 	return fmt.Sprintf("http://localhost:8080/api/v1/namespaces/%v/services/%v/proxy", namespace, service)
 }
 
-func getApiServerUrl(port uint16) string {
+func GetApiServerUrl(port uint16) string {
 	return fmt.Sprintf("http://localhost:%v", port)
 }
 
+func NewKubernetesProvider() (*KubernetesProvider, error) {
+	home := homedir.HomeDir()
+	configLoadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(home, ".kube", "config")}
+	clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		configLoadingRules,
+		&clientcmd.ConfigOverrides{
+			CurrentContext: "",
+		},
+	)
+
+	restClientConfig, err := clientConfig.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	clientSet, err := kubernetes.NewForConfig(restClientConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return &KubernetesProvider{clientSet}, nil
+}
+
+type KubernetesProvider struct {
+	clientSet *kubernetes.Clientset
+}
+
+func (kp *KubernetesProvider) GetServiceExternalIp(ctx context.Context, namespace string, service string) (string, error) {
+	serviceObj, err := kp.clientSet.CoreV1().Services(namespace).Get(ctx, service, metav1.GetOptions{})
+	if err != nil {
+		return "", err
+	}
+
+	externalIp := serviceObj.Status.LoadBalancer.Ingress[0].IP
+	return externalIp, nil
+}
+
+func SwitchKubeContextForTest(t *testing.T, newContextName string) error {
+	prevKubeContextName, err := GetKubeCurrentContextName()
+	if err != nil {
+		return err
+	}
+
+	if err := SetKubeCurrentContext(newContextName); err != nil {
+		return err
+	}
+
+	t.Cleanup(func() {
+		if err := SetKubeCurrentContext(prevKubeContextName); err != nil {
+			t.Errorf("failed to set Kubernetes context to %s, err: %v", prevKubeContextName, err)
+			t.Errorf("cleanup failed, subsequent tests may be affected")
+		}
+	})
+
+	return nil
+}
+
+func GetKubeCurrentContextName() (string, error) {
+	cmd := exec.Command("kubectl", "config", "current-context")
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("%v, %s", err, string(output))
+	}
+
+	return string(bytes.TrimSpace(output)), nil
+}
+
+func SetKubeCurrentContext(contextName string) error {
+	cmd := exec.Command("kubectl", "config", "use-context", contextName)
+
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("%v, %s", err, string(output))
+	}
+
+	return nil
+}
+
+func ApplyKubeFilesForTest(t *testing.T, kubeContext string, namespace string, filename ...string) error {
+	for i := range filename {
+		fname := filename[i]
+		if err := ApplyKubeFile(kubeContext, namespace, fname); err != nil {
+			return err
+		}
+
+		t.Cleanup(func() {
+			if err := DeleteKubeFile(kubeContext, namespace, fname); err != nil {
+				t.Errorf(
+					"failed to delete Kubernetes resources in namespace %s from filename %s, err: %v",
+					namespace,
+					fname,
+					err,
+				)
+			}
+		})
+	}
+
+	return nil
+}
+
+func ApplyKubeFile(kubeContext string, namespace string, filename string) error {
+	cmdArgs := []string{
+		"apply",
+		"--context", kubeContext,
+		"-f", filename,
+	}
+	if namespace != AllNamespaces {
+		cmdArgs = append(cmdArgs, "-n", namespace)
+	}
+	cmd := exec.Command("kubectl", cmdArgs...)
+
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("%v, %s", err, string(output))
+	}
+
+	return nil
+}
+
+func DeleteKubeFile(kubeContext string, namespace string, filename string) error {
+	cmdArgs := []string{
+		"delete",
+		"--context", kubeContext,
+		"-f", filename,
+	}
+	if namespace != AllNamespaces {
+		cmdArgs = append(cmdArgs, "-n", namespace)
+	}
+	cmd := exec.Command("kubectl", cmdArgs...)
+
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("%v, %s", err, string(output))
+	}
+
+	return nil
+}
+
 func getDefaultCommandArgs() []string {
+	agentImageValue := os.Getenv("MIZU_CI_IMAGE")
 	setFlag := "--set"
 	telemetry := "telemetry=false"
-	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
+	agentImage := fmt.Sprintf("agent-image=%s", agentImageValue)
 	imagePullPolicy := "image-pull-policy=IfNotPresent"
 	headless := "headless=true"
 
 	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
 }
 
-func getDefaultTapCommandArgs() []string {
+func GetDefaultTapCommandArgs() []string {
 	tapCommand := "tap"
 	defaultCmdArgs := getDefaultCommandArgs()
 
 	return append([]string{tapCommand}, defaultCmdArgs...)
 }
 
-func getDefaultTapCommandArgsWithRegex(regex string) []string {
+func GetDefaultTapCommandArgsWithRegex(regex string) []string {
 	tapCommand := "tap"
 	defaultCmdArgs := getDefaultCommandArgs()
 
 	return append([]string{tapCommand, regex}, defaultCmdArgs...)
 }
 
-func getDefaultLogsCommandArgs() []string {
+func GetDefaultLogsCommandArgs() []string {
 	logsCommand := "logs"
 	defaultCmdArgs := getDefaultCommandArgs()
 
 	return append([]string{logsCommand}, defaultCmdArgs...)
 }
 
-func getDefaultTapNamespace() []string {
+func GetDefaultTapNamespace() []string {
 	return []string{"-n", "mizu-tests"}
 }
 
-func getDefaultConfigCommandArgs() []string {
+func GetDefaultConfigCommandArgs() []string {
 	configCommand := "config"
 	defaultCmdArgs := getDefaultCommandArgs()
 
 	return append([]string{configCommand}, defaultCmdArgs...)
 }
 
-func runCypressTests(t *testing.T, cypressRunCmd string) {
+func RunCypressTests(t *testing.T, cypressRunCmd string) {
 	cypressCmd := exec.Command("bash", "-c", cypressRunCmd)
 	t.Logf("running command: %v", cypressCmd.String())
-	out, err := cypressCmd.Output()
+	out, err := cypressCmd.CombinedOutput()
 	if err != nil {
-		t.Errorf("%s", out)
+		t.Errorf("error running cypress, error: %v, output: %v", err, string(out))
 		return
 	}
+
 	t.Logf("%s", out)
 }
 
-func retriesExecute(retriesCount int, executeFunc func() error) error {
+func RetriesExecute(retriesCount int, executeFunc func() error) error {
 	var lastError interface{}
 
 	for i := 0; i < retriesCount; i++ {
-		if err := tryExecuteFunc(executeFunc); err != nil {
+		if err := TryExecuteFunc(executeFunc); err != nil {
 			lastError = err
 
 			time.Sleep(1 * time.Second)
@@ -138,7 +284,7 @@ func retriesExecute(retriesCount int, executeFunc func() error) error {
 	return fmt.Errorf("reached max retries count, retries count: %v, last err: %v", retriesCount, lastError)
 }
 
-func tryExecuteFunc(executeFunc func() error) (err interface{}) {
+func TryExecuteFunc(executeFunc func() error) (err interface{}) {
 	defer func() {
 		if panicErr := recover(); panicErr != nil {
 			err = panicErr
@@ -148,10 +294,10 @@ func tryExecuteFunc(executeFunc func() error) (err interface{}) {
 	return executeFunc()
 }
 
-func waitTapPodsReady(apiServerUrl string) error {
+func WaitTapPodsReady(apiServerUrl string) error {
 	resolvingUrl := fmt.Sprintf("%v/status/connectedTappersCount", apiServerUrl)
 	tapPodsReadyFunc := func() error {
-		requestResult, requestErr := executeHttpGetRequest(resolvingUrl)
+		requestResult, requestErr := ExecuteHttpGetRequest(resolvingUrl)
 		if requestErr != nil {
 			return requestErr
 		}
@@ -160,14 +306,14 @@ func waitTapPodsReady(apiServerUrl string) error {
 		if connectedTappersCount == 0 {
 			return fmt.Errorf("no connected tappers running")
 		}
-		time.Sleep(waitAfterTapPodsReady)
+		time.Sleep(WaitAfterTapPodsReady)
 		return nil
 	}
 
-	return retriesExecute(longRetriesCount, tapPodsReadyFunc)
+	return RetriesExecute(LongRetriesCount, tapPodsReadyFunc)
 }
 
-func jsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
+func JsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
 	var result interface{}
 	if parseErr := json.Unmarshal(jsonBytes, &result); parseErr != nil {
 		return nil, parseErr
@@ -176,7 +322,7 @@ func jsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
 	return result, nil
 }
 
-func executeHttpRequest(response *http.Response, requestErr error) (interface{}, error) {
+func ExecuteHttpRequest(response *http.Response, requestErr error) (interface{}, error) {
 	if requestErr != nil {
 		return nil, requestErr
 	} else if response.StatusCode != 200 {
@@ -190,10 +336,10 @@ func executeHttpRequest(response *http.Response, requestErr error) (interface{},
 		return nil, readErr
 	}
 
-	return jsonBytesToInterface(data)
+	return JsonBytesToInterface(data)
 }
 
-func executeHttpGetRequestWithHeaders(url string, headers map[string]string) (interface{}, error) {
+func ExecuteHttpGetRequestWithHeaders(url string, headers map[string]string) (interface{}, error) {
 	request, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return nil, err
@@ -205,15 +351,15 @@ func executeHttpGetRequestWithHeaders(url string, headers map[string]string) (in
 
 	client := &http.Client{}
 	response, requestErr := client.Do(request)
-	return executeHttpRequest(response, requestErr)
+	return ExecuteHttpRequest(response, requestErr)
 }
 
-func executeHttpGetRequest(url string) (interface{}, error) {
+func ExecuteHttpGetRequest(url string) (interface{}, error) {
 	response, requestErr := http.Get(url)
-	return executeHttpRequest(response, requestErr)
+	return ExecuteHttpRequest(response, requestErr)
 }
 
-func executeHttpPostRequestWithHeaders(url string, headers map[string]string, body interface{}) (interface{}, error) {
+func ExecuteHttpPostRequestWithHeaders(url string, headers map[string]string, body interface{}) (interface{}, error) {
 	requestBody, jsonErr := json.Marshal(body)
 	if jsonErr != nil {
 		return nil, jsonErr
@@ -231,10 +377,10 @@ func executeHttpPostRequestWithHeaders(url string, headers map[string]string, bo
 
 	client := &http.Client{}
 	response, requestErr := client.Do(request)
-	return executeHttpRequest(response, requestErr)
+	return ExecuteHttpRequest(response, requestErr)
 }
 
-func cleanupCommand(cmd *exec.Cmd) error {
+func CleanupCommand(cmd *exec.Cmd) error {
 	if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
 		return err
 	}
@@ -246,7 +392,7 @@ func cleanupCommand(cmd *exec.Cmd) error {
 	return nil
 }
 
-func getLogsPath() (string, error) {
+func GetLogsPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {
 		return "", filePathErr

agent/go.mod

@@ -1,42 +1,150 @@
 module github.com/up9inc/mizu/agent
 
-go 1.16
+go 1.17
 
 require (
 	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
-	github.com/chanced/openapi v0.0.7
-	github.com/djherbis/atime v1.0.0
-	github.com/elastic/go-elasticsearch/v7 v7.16.0
-	github.com/getkin/kin-openapi v0.76.0
+	github.com/chanced/openapi v0.0.8
+	github.com/djherbis/atime v1.1.0
+	github.com/gin-contrib/pprof v1.3.0
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.7.7
-	github.com/go-playground/locales v0.13.0
-	github.com/go-playground/universal-translator v0.17.0
-	github.com/go-playground/validator/v10 v10.5.0
-	github.com/google/uuid v1.1.2
+	github.com/go-playground/locales v0.14.0
+	github.com/go-playground/universal-translator v0.18.0
+	github.com/go-playground/validator/v10 v10.10.0
+	github.com/google/uuid v1.3.0
 	github.com/gorilla/websocket v1.4.2
+	github.com/jinzhu/copier v0.3.5
 	github.com/nav-inc/datetime v0.1.3
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
-	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
-	github.com/ory/keto-client-go v0.7.0-alpha.1
-	github.com/ory/kratos-client-go v0.8.2-alpha.1
-	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/orcaman/concurrent-map v1.0.0
 	github.com/stretchr/testify v1.7.0
-	github.com/up9inc/basenine/client/go v0.0.0-20220125035724-573fff0d5075
+	github.com/up9inc/basenine/client/go v0.0.0-20220612112747-3b28eeac9c51
+	github.com/up9inc/mizu/logger v0.0.0
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
+	github.com/up9inc/mizu/tap/dbgctl v0.0.0
 	github.com/up9inc/mizu/tap/extensions/amqp v0.0.0
 	github.com/up9inc/mizu/tap/extensions/http v0.0.0
 	github.com/up9inc/mizu/tap/extensions/kafka v0.0.0
 	github.com/up9inc/mizu/tap/extensions/redis v0.0.0
 	github.com/wI2L/jsondiff v0.1.1
-	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
-	k8s.io/api v0.21.2
-	k8s.io/apimachinery v0.21.2
-	k8s.io/client-go v0.21.2
+	k8s.io/api v0.23.3
+	k8s.io/apimachinery v0.23.3
+	k8s.io/client-go v0.23.3
 )
 
+require (
+	cloud.google.com/go/compute v1.2.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.24 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.1 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/MakeNowJust/heredoc v1.0.0 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/beevik/etree v1.1.0 // indirect
+	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
+	github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect
+	github.com/cilium/ebpf v0.8.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
+	github.com/fatih/camelcase v1.0.0 // indirect
+	github.com/fvbommel/sortorder v1.0.2 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-logr/logr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.6 // indirect
+	github.com/go-openapi/swag v0.21.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/btree v1.0.1 // indirect
+	github.com/google/go-cmp v0.5.7 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/martian v2.1.0+incompatible // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.14.2 // indirect
+	github.com/knightsc/gapstone v0.0.0-20191231144527-6fa5afaf11a9 // indirect
+	github.com/leodido/go-urn v1.2.1 // indirect
+	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mertyildiran/gqlparser/v2 v2.4.6 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/ohler55/ojg v1.12.12 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/russross/blackfriday v1.6.0 // indirect
+	github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
+	github.com/segmentio/kafka-go v0.4.27 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/spf13/cobra v1.3.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/struCoder/pidusage v0.2.1 // indirect
+	github.com/tidwall/gjson v1.14.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/sjson v1.2.4 // indirect
+	github.com/tklauser/go-sysconf v0.3.10 // indirect
+	github.com/tklauser/numcpus v0.4.0 // indirect
+	github.com/ugorji/go/codec v1.2.6 // indirect
+	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
+	github.com/xlab/treeprint v1.1.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	go.starlark.net v0.0.0-20220203230714-bb14e151c28f // indirect
+	golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	k8s.io/cli-runtime v0.23.3 // indirect
+	k8s.io/component-base v0.23.3 // indirect
+	k8s.io/klog/v2 v2.40.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf // indirect
+	k8s.io/kubectl v0.23.3 // indirect
+	k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
+	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
+	sigs.k8s.io/kustomize/api v0.11.1 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.13.3 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
+replace github.com/up9inc/mizu/logger v0.0.0 => ../logger
+
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 
 replace github.com/up9inc/mizu/tap v0.0.0 => ../tap
@@ -50,3 +158,5 @@ replace github.com/up9inc/mizu/tap/extensions/http v0.0.0 => ../tap/extensions/h
 replace github.com/up9inc/mizu/tap/extensions/kafka v0.0.0 => ../tap/extensions/kafka
 
 replace github.com/up9inc/mizu/tap/extensions/redis v0.0.0 => ../tap/extensions/redis
+
+replace github.com/up9inc/mizu/tap/dbgctl v0.0.0 => ../tap/dbgctl

agent/keto/Dockerfile

@@ -1,13 +0,0 @@
-FROM oryd/keto:v0.7.0-alpha.1-sqlite
-
-USER root
-
-RUN apk add sqlite
-
-RUN mkdir -p /etc/config/keto
-
-COPY ./keto.yml /etc/config/keto/keto.yml
-COPY ./start.sh /opt/start.sh
-RUN chmod +x /opt/start.sh
-
-ENTRYPOINT ["/opt/start.sh"]

agent/keto/build-push-featurebranch.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-set -e
-
-GCP_PROJECT=up9-docker-hub
-REPOSITORY=gcr.io/$GCP_PROJECT
-SERVER_NAME=mizu-keto
-GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
-
-DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
-VER=${VER=0.0}
-
-DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$VER")
-
-if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
-then
-  echo "Pushing to $GIT_BRANCH is allowed only via CI"
-  exit 1
-fi
-
-echo "building ${DOCKER_TAGGED_BUILDS[@]}"
-DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
-docker build $DOCKER_TAGS_ARGS --build-arg VER=${VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
-
-for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
-do
-  echo pushing "$DOCKER_TAG"
-  docker push "$DOCKER_TAG"
-done

agent/keto/keto.yml

@@ -1,22 +0,0 @@
-version: v0.7.0-alpha.1
-
-dsn: sqlite:///app/data/kratos.sqlite?_fk=true
-
-log:
-  level: info
-  format: text
-  leak_sensitive_values: false
-
-serve:
-  read:
-    host: 0.0.0.0
-    port: 4466
-  write:
-    host: 0.0.0.0
-    port: 4467
-
-namespaces:
-  - id: 0
-    name: system
-  - id: 1
-    name: workspaces

agent/keto/start.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-keto migrate up -c /etc/config/keto/keto.yml --yes # this initializes the db
-keto serve -c /etc/config/keto/keto.yml # start keto

agent/kratos/Dockerfile

@@ -1,14 +0,0 @@
-FROM gcr.io/up9-docker-hub/mizu-kratos-base/simple-password-policy:latest
-
-USER root
-
-RUN apk add sqlite
-
-RUN mkdir -p /etc/config/kratos
-
-COPY ./kratos.yml /etc/config/kratos/kratos.yml
-COPY ./identity.schema.json /etc/config/kratos/identity.schema.json
-COPY ./start.sh /opt/start.sh
-RUN chmod +x /opt/start.sh
-
-ENTRYPOINT ["/opt/start.sh"]

agent/kratos/build-push-featurebranch.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-set -e
-
-GCP_PROJECT=up9-docker-hub
-REPOSITORY=gcr.io/$GCP_PROJECT
-SERVER_NAME=mizu-kratos
-GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
-
-DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
-VER=${VER=0.0.0}
-
-DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$VER")
-
-if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
-then
-  echo "Pushing to $GIT_BRANCH is allowed only via CI"
-  exit 1
-fi
-
-echo "building ${DOCKER_TAGGED_BUILDS[@]}"
-DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
-docker build $DOCKER_TAGS_ARGS --build-arg VER=${VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
-
-for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
-do
-  echo pushing "$DOCKER_TAG"
-  docker push "$DOCKER_TAG"
-done

agent/kratos/identity.schema.json

@@ -1,43 +0,0 @@
-{
-	"$id": "https://schemas.ory.sh/presets/kratos/quickstart/email-password/identity.schema.json",
-	"$schema": "http://json-schema.org/draft-07/schema#",
-	"title": "Person",
-	"type": "object",
-	"properties": {
-		"traits": {
-			"type": "object",
-			"properties": {
-				"username": {
-					"type": "string",
-					"format": "username",
-					"title": "Username",
-					"minLength": 3,
-					"ory.sh/kratos": {
-						"credentials": {
-							"password": {
-								"identifier": true
-							}
-						}
-					}
-				},
-				"name": {
-					"type": "object",
-					"properties": {
-						"first": {
-							"title": "First Name",
-							"type": "string"
-						},
-						"last": {
-							"title": "Last Name",
-							"type": "string"
-						}
-					}
-				}
-			},
-			"required": [
-				"username"
-			],
-			"additionalProperties": false
-		}
-	}
-}

agent/kratos/kratos.yml

@@ -1,84 +0,0 @@
-version: v0.8.2-alpha.1
-
-dsn: sqlite:///app/data/kratos.sqlite?_fk=true
-
-serve:
-  public:
-    base_url: http://127.0.0.1:4433/
-    cors:
-      enabled: true
-  admin:
-    base_url: http://kratos:4434/
-
-selfservice:
-  default_browser_return_url: http://127.0.0.1:4455/
-  whitelisted_return_urls:
-    - http://127.0.0.1:4455
-
-  methods:
-    password:
-      enabled: true
-
-  flows:
-    error:
-      ui_url: http://127.0.0.1:4455/error
-
-    settings:
-      ui_url: http://127.0.0.1:4455/settings
-      privileged_session_max_age: 15m
-
-    recovery:
-      enabled: true
-      ui_url: http://127.0.0.1:4455/recovery
-
-    verification:
-      enabled: false
-      ui_url: http://127.0.0.1:4455/verification
-      after:
-        default_browser_return_url: http://127.0.0.1:4455/
-
-    logout:
-      after:
-        default_browser_return_url: http://127.0.0.1:4455/login
-
-    login:
-      ui_url: http://127.0.0.1:4455/login
-      lifespan: 10m
-
-    registration:
-      lifespan: 10m
-      ui_url: http://127.0.0.1:4455/registration
-      after:
-        password:
-          hooks:
-            -
-              hook: session
-
-log:
-  level: info
-  format: text
-  leak_sensitive_values: false
-
-secrets:
-  cookie:
-    - PLEASE-CHANGE-ME-I-AM-VERY-INSECURE
-  cipher:
-    - 32-LONG-SECRET-NOT-SECURE-AT-ALL
-
-ciphers:
-  algorithm: xchacha20-poly1305
-
-hashers:
-  argon2:
-    parallelism: 1
-    memory: 128MB
-    iterations: 2
-    salt_length: 16
-    key_length: 16
-
-identity:
-  default_schema_url: file:///etc/config/kratos/identity.schema.json
-
-courier:
-  smtp:
-    connection_uri: smtps://test:test@mailslurper:1025/?skip_ssl_verify=true

agent/kratos/start.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-kratos migrate sql sqlite:///app/data/kratos.sqlite?_fk=true --yes # this initializes the db
-kratos serve -c /etc/config/kratos/kratos.yml --watch-courier # start kratos

agent/main.go

@@ -9,44 +9,34 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
-	"sort"
 	"strconv"
 	"strings"
 	"syscall"
 	"time"
 
+	"github.com/gin-contrib/pprof"
+	"github.com/gin-contrib/static"
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/entries"
 	"github.com/up9inc/mizu/agent/pkg/middlewares"
 	"github.com/up9inc/mizu/agent/pkg/models"
 	"github.com/up9inc/mizu/agent/pkg/oas"
 	"github.com/up9inc/mizu/agent/pkg/routes"
 	"github.com/up9inc/mizu/agent/pkg/servicemap"
-	"github.com/up9inc/mizu/agent/pkg/up9"
 	"github.com/up9inc/mizu/agent/pkg/utils"
 
-	"github.com/up9inc/mizu/agent/pkg/elastic"
-
-	"github.com/up9inc/mizu/agent/pkg/controllers"
-
 	"github.com/up9inc/mizu/agent/pkg/api"
+	"github.com/up9inc/mizu/agent/pkg/app"
 	"github.com/up9inc/mizu/agent/pkg/config"
 
-	v1 "k8s.io/api/core/v1"
-
-	"github.com/antelman107/net-wait-go/wait"
-	"github.com/gin-contrib/static"
-	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	"github.com/op/go-logging"
-	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap"
 	tapApi "github.com/up9inc/mizu/tap/api"
-
-	amqpExt "github.com/up9inc/mizu/tap/extensions/amqp"
-	httpExt "github.com/up9inc/mizu/tap/extensions/http"
-	kafkaExt "github.com/up9inc/mizu/tap/extensions/kafka"
-	redisExt "github.com/up9inc/mizu/tap/extensions/redis"
+	"github.com/up9inc/mizu/tap/dbgctl"
 )
 
 var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API")
@@ -56,11 +46,7 @@ var apiServerAddress = flag.String("api-server-address", "", "Address of mizu AP
 var namespace = flag.String("namespace", "", "Resolve IPs if they belong to resources in this namespace (default is all)")
 var harsReaderMode = flag.Bool("hars-read", false, "Run in hars-read mode")
 var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
-
-var extensions []*tapApi.Extension             // global
-var extensionsMap map[string]*tapApi.Extension // global
-
-var startTime int64
+var profiler = flag.Bool("profiler", false, "Run pprof server")
 
 const (
 	socketConnectionRetries    = 30
@@ -69,84 +55,32 @@ const (
 )
 
 func main() {
+	initializeDependencies()
 	logLevel := determineLogLevel()
-	logger.InitLoggerStderrOnly(logLevel)
+	logger.InitLoggerStd(logLevel)
 	flag.Parse()
-	if err := config.LoadConfig(); err != nil {
-		logger.Log.Fatalf("Error loading config file %v", err)
-	}
-	loadExtensions()
+
+	app.LoadExtensions()
 
 	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode {
-		panic("One of the flags --tap, --api or --standalone or --hars-read must be provided")
+		panic("One of the flags --tap, --api-server, --standalone or --hars-read must be provided")
 	}
 
 	if *standaloneMode {
-		api.StartResolving(*namespace)
-
-		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
-		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-
-		filteringOptions := getTrafficFilteringOptions()
-		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-		tapOpts := &tap.TapOpts{HostMode: hostMode}
-		tap.StartPassiveTapper(tapOpts, outputItemsChannel, extensions, filteringOptions)
-
-		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
-		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
-
-		hostApi(nil)
+		runInStandaloneMode()
 	} else if *tapperMode {
-		logger.Log.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
-		if *apiServerAddress == "" {
-			panic("API server address must be provided with --api-server-address when using --tap")
-		}
-
-		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-		tapOpts := &tap.TapOpts{HostMode: hostMode}
-		tapTargets := getTapTargets()
-		if tapTargets != nil {
-			tapOpts.FilterAuthorities = tapTargets
-			logger.Log.Infof("Filtering for the following authorities: %v", tapOpts.FilterAuthorities)
-		}
-
-		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-
-		filteringOptions := getTrafficFilteringOptions()
-		tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions)
-		socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
-		if err != nil {
-			panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
-		}
-		logger.Log.Infof("Connected successfully to websocket %s", *apiServerAddress)
-
-		go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
+		runInTapperMode()
 	} else if *apiServerMode {
-		configureBasenineServer(shared.BasenineHost, shared.BaseninePort)
-		startTime = time.Now().UnixNano() / int64(time.Millisecond)
-		api.StartResolving(*namespace)
+		ginApp := runInApiServerMode(*namespace)
 
-		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
-		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-		enableExpFeatureIfNeeded()
-		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
-		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
-
-		syncEntriesConfig := getSyncEntriesConfig()
-		if syncEntriesConfig != nil {
-			if err := up9.SyncEntries(syncEntriesConfig); err != nil {
-				logger.Log.Error("Error syncing entries, err: %v", err)
-			}
+		if *profiler {
+			pprof.Register(ginApp)
 		}
 
-		hostApi(outputItemsChannel)
-	} else if *harsReaderMode {
-		outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
-		filteredHarChannel := make(chan *tapApi.OutputChannelItem)
+		utils.StartServer(ginApp)
 
-		go filterItems(outputItemsChannel, filteredHarChannel)
-		go api.StartReadingEntries(filteredHarChannel, harsDir, extensionsMap)
-		hostApi(nil)
+	} else if *harsReaderMode {
+		runInHarReaderMode()
 	}
 
 	signalChan := make(chan os.Signal, 1)
@@ -156,142 +90,128 @@ func main() {
 	logger.Log.Info("Exiting")
 }
 
-func enableExpFeatureIfNeeded() {
-	if config.Config.OAS {
-		oas.GetOasGeneratorInstance().Start()
-	}
-	if config.Config.ServiceMap {
-		servicemap.GetInstance().SetConfig(config.Config)
-	}
-	elastic.GetInstance().Configure(config.Config.Elastic)
-}
+func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engine {
+	ginApp := gin.Default()
 
-func configureBasenineServer(host string, port string) {
-	if !wait.New(
-		wait.WithProto("tcp"),
-		wait.WithWait(200*time.Millisecond),
-		wait.WithBreak(50*time.Millisecond),
-		wait.WithDeadline(5*time.Second),
-		wait.WithDebug(config.Config.LogLevel == logging.DEBUG),
-	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
-		logger.Log.Panicf("Basenine is not available!")
-	}
-
-	// Limit the database size to default 200MB
-	err := basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
-	if err != nil {
-		logger.Log.Panicf("Error while limiting database size: %v", err)
-	}
-
-	// Define the macros
-	for _, extension := range extensions {
-		macros := extension.Dissector.Macros()
-		for macro, expanded := range macros {
-			err = basenine.Macro(host, port, macro, expanded)
-			if err != nil {
-				logger.Log.Panicf("Error while adding a macro: %v", err)
-			}
-		}
-	}
-}
-
-func loadExtensions() {
-	extensions = make([]*tapApi.Extension, 4)
-	extensionsMap = make(map[string]*tapApi.Extension)
-
-	extensionAmqp := &tapApi.Extension{}
-	dissectorAmqp := amqpExt.NewDissector()
-	dissectorAmqp.Register(extensionAmqp)
-	extensionAmqp.Dissector = dissectorAmqp
-	extensions[0] = extensionAmqp
-	extensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
-
-	extensionHttp := &tapApi.Extension{}
-	dissectorHttp := httpExt.NewDissector()
-	dissectorHttp.Register(extensionHttp)
-	extensionHttp.Dissector = dissectorHttp
-	extensions[1] = extensionHttp
-	extensionsMap[extensionHttp.Protocol.Name] = extensionHttp
-
-	extensionKafka := &tapApi.Extension{}
-	dissectorKafka := kafkaExt.NewDissector()
-	dissectorKafka.Register(extensionKafka)
-	extensionKafka.Dissector = dissectorKafka
-	extensions[2] = extensionKafka
-	extensionsMap[extensionKafka.Protocol.Name] = extensionKafka
-
-	extensionRedis := &tapApi.Extension{}
-	dissectorRedis := redisExt.NewDissector()
-	dissectorRedis.Register(extensionRedis)
-	extensionRedis.Dissector = dissectorRedis
-	extensions[3] = extensionRedis
-	extensionsMap[extensionRedis.Protocol.Name] = extensionRedis
-
-	sort.Slice(extensions, func(i, j int) bool {
-		return extensions[i].Protocol.Priority < extensions[j].Protocol.Priority
-	})
-
-	for _, extension := range extensions {
-		logger.Log.Infof("Extension Properties: %+v", extension)
-	}
-
-	controllers.InitExtensionsMap(extensionsMap)
-}
-
-func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
-	app := gin.Default()
-
-	app.GET("/echo", func(c *gin.Context) {
-		c.String(http.StatusOK, "Here is Mizu agent")
+	ginApp.GET("/echo", func(c *gin.Context) {
+		c.JSON(http.StatusOK, "Here is Mizu agent")
 	})
 
 	eventHandlers := api.RoutesEventHandlers{
 		SocketOutChannel: socketHarOutputChannel,
 	}
 
-	app.Use(DisableRootStaticCache())
-
-	var staticFolder string
-	if config.Config.StandaloneMode {
-		staticFolder = "./site-standalone"
-	} else {
-		staticFolder = "./site"
-	}
+	ginApp.Use(disableRootStaticCache())
 
+	staticFolder := "./site"
 	indexStaticFile := staticFolder + "/index.html"
 	if err := setUIFlags(indexStaticFile); err != nil {
 		logger.Log.Errorf("Error setting ui flags, err: %v", err)
 	}
 
-	app.Use(static.ServeRoot("/", staticFolder))
-	app.NoRoute(func(c *gin.Context) {
+	ginApp.Use(static.ServeRoot("/", staticFolder))
+	ginApp.NoRoute(func(c *gin.Context) {
 		c.File(indexStaticFile)
 	})
 
-	app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
+	ginApp.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if it's called before
 
-	api.WebSocketRoutes(app, &eventHandlers, startTime)
+	api.WebSocketRoutes(ginApp, &eventHandlers)
 
-	if config.Config.StandaloneMode {
-		routes.ConfigRoutes(app)
-		routes.UserRoutes(app)
-		routes.InstallRoutes(app)
-	}
-	if config.Config.OAS {
-		routes.OASRoutes(app)
+	if config.Config.OAS.Enable {
+		routes.OASRoutes(ginApp)
 	}
+
 	if config.Config.ServiceMap {
-		routes.ServiceMapRoutes(app)
+		routes.ServiceMapRoutes(ginApp)
 	}
 
-	routes.QueryRoutes(app)
-	routes.EntriesRoutes(app)
-	routes.MetadataRoutes(app)
-	routes.StatusRoutes(app)
-	utils.StartServer(app)
+	routes.QueryRoutes(ginApp)
+	routes.EntriesRoutes(ginApp)
+	routes.MetadataRoutes(ginApp)
+	routes.StatusRoutes(ginApp)
+	routes.DbRoutes(ginApp)
+	routes.ReplayRoutes(ginApp)
+
+	return ginApp
 }
 
-func DisableRootStaticCache() gin.HandlerFunc {
+func runInApiServerMode(namespace string) *gin.Engine {
+	if err := config.LoadConfig(); err != nil {
+		logger.Log.Fatalf("Error loading config file %v", err)
+	}
+	app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel, config.Config.InsertionFilter)
+	api.StartResolving(namespace)
+
+	enableExpFeatureIfNeeded()
+
+	return hostApi(app.GetEntryInputChannel())
+}
+
+func runInTapperMode() {
+	logger.Log.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
+	if *apiServerAddress == "" {
+		panic("API server address must be provided with --api-server-address when using --tap")
+	}
+
+	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+	tapOpts := &tap.TapOpts{
+		HostMode: hostMode,
+	}
+
+	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+
+	filteringOptions := getTrafficFilteringOptions()
+	tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, app.Extensions, filteringOptions)
+	socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
+	if err != nil {
+		panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
+	}
+	logger.Log.Infof("Connected successfully to websocket %s", *apiServerAddress)
+
+	go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
+}
+
+func runInStandaloneMode() {
+	api.StartResolving(*namespace)
+
+	outputItemsChannel := make(chan *tapApi.OutputChannelItem)
+	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+
+	filteringOptions := getTrafficFilteringOptions()
+	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+	tapOpts := &tap.TapOpts{HostMode: hostMode}
+	tap.StartPassiveTapper(tapOpts, outputItemsChannel, app.Extensions, filteringOptions)
+
+	go app.FilterItems(outputItemsChannel, filteredOutputItemsChannel)
+	go api.StartReadingEntries(filteredOutputItemsChannel, nil, app.ExtensionsMap)
+
+	ginApp := hostApi(nil)
+	utils.StartServer(ginApp)
+}
+
+func runInHarReaderMode() {
+	outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
+	filteredHarChannel := make(chan *tapApi.OutputChannelItem)
+
+	go app.FilterItems(outputItemsChannel, filteredHarChannel)
+	go api.StartReadingEntries(filteredHarChannel, harsDir, app.ExtensionsMap)
+	ginApp := hostApi(nil)
+	utils.StartServer(ginApp)
+}
+
+func enableExpFeatureIfNeeded() {
+	if config.Config.OAS.Enable {
+		oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
+		oasGenerator.Start()
+	}
+	if config.Config.ServiceMap {
+		serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap)
+		serviceMapGenerator.Enable()
+	}
+}
+
+func disableRootStaticCache() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		if c.Request.RequestURI == "/" {
 			// Disable cache only for the main static route
@@ -308,7 +228,7 @@ func setUIFlags(uiIndexPath string) error {
 		return err
 	}
 
-	replacedContent := strings.Replace(string(read), "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS), 1)
+	replacedContent := strings.Replace(string(read), "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS.Enable), 1)
 	replacedContent = strings.Replace(replacedContent, "__IS_SERVICE_MAP_ENABLED__", strconv.FormatBool(config.Config.ServiceMap), 1)
 
 	err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
@@ -319,28 +239,6 @@ func setUIFlags(uiIndexPath string) error {
 	return nil
 }
 
-func parseEnvVar(env string) map[string][]v1.Pod {
-	var mapOfList map[string][]v1.Pod
-
-	val, present := os.LookupEnv(env)
-
-	if !present {
-		return mapOfList
-	}
-
-	err := json.Unmarshal([]byte(val), &mapOfList)
-	if err != nil {
-		panic(fmt.Sprintf("env var %s's value of %v is invalid! must be map[string][]v1.Pod %v", env, mapOfList, err))
-	}
-	return mapOfList
-}
-
-func getTapTargets() []v1.Pod {
-	nodeName := os.Getenv(shared.NodeNameEnvVar)
-	tappedAddressesPerNodeDict := parseEnvVar(shared.TappedAddressesPerNodeDictEnvVar)
-	return tappedAddressesPerNodeDict[nodeName]
-}
-
 func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
 	filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar)
 	if filteringOptionsJson == "" {
@@ -357,16 +255,6 @@ func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
 	return &filteringOptions
 }
 
-func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
-	for message := range inChannel {
-		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
-			continue
-		}
-
-		outChannel <- message
-	}
-}
-
 func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tapApi.OutputChannelItem) {
 	if connection == nil {
 		panic("Websocket connection is nil")
@@ -383,6 +271,10 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
 			continue
 		}
 
+		if dbgctl.MizuTapperDisableSending {
+			continue
+		}
+
 		// NOTE: This is where the `*tapApi.OutputChannelItem` leaves the code
 		// and goes into the intermediate WebSocket.
 		err = connection.WriteMessage(websocket.TextMessage, marshaledData)
@@ -402,21 +294,6 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
 	}
 }
 
-func getSyncEntriesConfig() *shared.SyncEntriesConfig {
-	syncEntriesConfigJson := os.Getenv(shared.SyncEntriesConfigEnvVar)
-	if syncEntriesConfigJson == "" {
-		return nil
-	}
-
-	var syncEntriesConfig = &shared.SyncEntriesConfig{}
-	err := json.Unmarshal([]byte(syncEntriesConfigJson), syncEntriesConfig)
-	if err != nil {
-		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.SyncEntriesConfig struct, err: %v", shared.SyncEntriesConfigEnvVar, syncEntriesConfigJson, err))
-	}
-
-	return syncEntriesConfig
-}
-
 func determineLogLevel() (logLevel logging.Level) {
 	logLevel, err := logging.LogLevel(os.Getenv(shared.LogLevelEnvVar))
 	if err != nil {
@@ -435,6 +312,7 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time.
 	for i := 1; i < retryAmount; i++ {
 		socketConnection, _, err := dialer.Dial(socketAddress, nil)
 		if err != nil {
+			lastErr = err
 			if i < retryAmount {
 				logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay/time.Second)
 				time.Sleep(retryDelay)
@@ -468,6 +346,14 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
 					} else {
 						tap.UpdateTapTargets(tapConfigMessage.TapTargets)
 					}
+				case shared.WebSocketMessageTypeUpdateTappedPods:
+					var tappedPodsMessage shared.WebSocketTappedPodsMessage
+					if err := json.Unmarshal(message, &tappedPodsMessage); err != nil {
+						logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
+						return
+					}
+					nodeName := os.Getenv(shared.NodeNameEnvVar)
+					tap.UpdateTapTargets(tappedPodsMessage.NodeToTappedPodMap[nodeName])
 				default:
 					logger.Log.Warningf("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
 				}
@@ -475,3 +361,12 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
 		}
 	}
 }
+
+func initializeDependencies() {
+	dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
+	dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} { return oas.GetDefaultOasGeneratorInstance(config.Config.OAS.MaxExampleLen) })
+	dependency.RegisterGenerator(dependency.EntriesInserter, func() interface{} { return api.GetBasenineEntryInserterInstance() })
+	dependency.RegisterGenerator(dependency.EntriesProvider, func() interface{} { return &entries.BasenineEntriesProvider{} })
+	dependency.RegisterGenerator(dependency.EntriesSocketStreamer, func() interface{} { return &api.BasenineEntryStreamer{} })
+	dependency.RegisterGenerator(dependency.EntryStreamerSocketConnector, func() interface{} { return &api.DefaultEntryStreamerSocketConnector{} })
+}

agent/pkg/api/contract_validation.go

@@ -1,107 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-
-	"github.com/getkin/kin-openapi/openapi3"
-	"github.com/getkin/kin-openapi/openapi3filter"
-	"github.com/getkin/kin-openapi/routers"
-	legacyrouter "github.com/getkin/kin-openapi/routers/legacy"
-
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap/api"
-)
-
-const (
-	ContractNotApplicable api.ContractStatus = 0
-	ContractPassed        api.ContractStatus = 1
-	ContractFailed        api.ContractStatus = 2
-)
-
-func loadOAS(ctx context.Context) (doc *openapi3.T, contractContent string, router routers.Router, err error) {
-	path := fmt.Sprintf("%s%s", shared.ConfigDirPath, shared.ContractFileName)
-	bytes, err := ioutil.ReadFile(path)
-	if err != nil {
-		return
-	}
-	contractContent = string(bytes)
-	loader := &openapi3.Loader{Context: ctx}
-	doc, _ = loader.LoadFromData(bytes)
-	err = doc.Validate(ctx)
-	if err != nil {
-		return
-	}
-	router, _ = legacyrouter.NewRouter(doc)
-	return
-}
-
-func validateOAS(ctx context.Context, doc *openapi3.T, router routers.Router, req *http.Request, res *http.Response) (isValid bool, reqErr error, resErr error) {
-	isValid = true
-	reqErr = nil
-	resErr = nil
-
-	// Find route
-	route, pathParams, err := router.FindRoute(req)
-	if err != nil {
-		return
-	}
-
-	// Validate request
-	requestValidationInput := &openapi3filter.RequestValidationInput{
-		Request:    req,
-		PathParams: pathParams,
-		Route:      route,
-	}
-	if reqErr = openapi3filter.ValidateRequest(ctx, requestValidationInput); reqErr != nil {
-		isValid = false
-	}
-
-	responseValidationInput := &openapi3filter.ResponseValidationInput{
-		RequestValidationInput: requestValidationInput,
-		Status:                 res.StatusCode,
-		Header:                 res.Header,
-	}
-
-	if res.Body != nil {
-		body, _ := ioutil.ReadAll(res.Body)
-		res.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-		responseValidationInput.SetBodyBytes(body)
-	}
-
-	// Validate response.
-	if resErr = openapi3filter.ValidateResponse(ctx, responseValidationInput); resErr != nil {
-		isValid = false
-	}
-
-	return
-}
-
-func handleOAS(ctx context.Context, doc *openapi3.T, router routers.Router, req *http.Request, res *http.Response, contractContent string) (contract api.Contract) {
-	contract = api.Contract{
-		Content: contractContent,
-		Status:  ContractNotApplicable,
-	}
-
-	isValid, reqErr, resErr := validateOAS(ctx, doc, router, req, res)
-	if isValid {
-		contract.Status = ContractPassed
-	} else {
-		contract.Status = ContractFailed
-		if reqErr != nil {
-			contract.RequestReason = reqErr.Error()
-		} else {
-			contract.RequestReason = ""
-		}
-		if resErr != nil {
-			contract.ResponseReason = resErr.Error()
-		} else {
-			contract.ResponseReason = ""
-		}
-	}
-
-	return
-}

agent/pkg/api/entry_streamer_socket_connector.go

@@ -0,0 +1,71 @@
+package api
+
+import (
+	"fmt"
+
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+type EntryStreamerSocketConnector interface {
+	SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error
+	SendMetadata(socketId int, metadata *basenine.Metadata) error
+	SendToastError(socketId int, err error) error
+	CleanupSocket(socketId int)
+}
+
+type DefaultEntryStreamerSocketConnector struct{}
+
+func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error {
+	var message []byte
+	if params.EnableFullEntries {
+		message, _ = models.CreateFullEntryWebSocketMessage(entry)
+	} else {
+		protocol, ok := protocolsMap[entry.ProtocolId]
+		if !ok {
+			return fmt.Errorf("protocol not found, protocol: %v", protocol)
+		}
+
+		extension, ok := extensionsMap[protocol.Name]
+		if !ok {
+			return fmt.Errorf("extension not found, extension: %v", protocol.Name)
+		}
+
+		base := extension.Dissector.Summarize(entry)
+		message, _ = models.CreateBaseEntryWebSocketMessage(base)
+	}
+
+	if err := SendToSocket(socketId, message); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (e *DefaultEntryStreamerSocketConnector) SendMetadata(socketId int, metadata *basenine.Metadata) error {
+	metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
+	if err := SendToSocket(socketId, metadataBytes); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (e *DefaultEntryStreamerSocketConnector) SendToastError(socketId int, err error) error {
+	toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
+		Type:      "error",
+		AutoClose: 5000,
+		Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
+	})
+	if err := SendToSocket(socketId, toastBytes); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (e *DefaultEntryStreamerSocketConnector) CleanupSocket(socketId int) {
+	socketObj := connectedWebsockets[socketId]
+	socketCleanup(socketId, socketObj)
+}

agent/pkg/api/main.go

@@ -11,23 +11,19 @@ import (
 	"strings"
 	"time"
 
-	"github.com/up9inc/mizu/agent/pkg/elastic"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/oas"
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
+
 	"github.com/up9inc/mizu/agent/pkg/har"
 	"github.com/up9inc/mizu/agent/pkg/holder"
 	"github.com/up9inc/mizu/agent/pkg/providers"
 
-	"github.com/up9inc/mizu/agent/pkg/servicemap"
-
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/oas"
 	"github.com/up9inc/mizu/agent/pkg/resolver"
 	"github.com/up9inc/mizu/agent/pkg/utils"
 
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
-
-	basenine "github.com/up9inc/basenine/client/go"
 )
 
 var k8sResolver *resolver.Resolver
@@ -102,82 +98,69 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		panic("Channel of captured messages is nil")
 	}
 
-	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
-	if err != nil {
-		panic(err)
-	}
-	connection.InsertMode()
-
-	disableOASValidation := false
-	ctx := context.Background()
-	doc, contractContent, router, err := loadOAS(ctx)
-	if err != nil {
-		logger.Log.Infof("Disabled OAS validation: %s", err.Error())
-		disableOASValidation = true
-	}
-
 	for item := range outputItems {
 		extension := extensionsMap[item.Protocol.Name]
-		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
-		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
-		if extension.Protocol.Name == "http" {
-			if !disableOASValidation {
-				var httpPair tapApi.HTTPRequestResponsePair
-				if err := json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair); err != nil {
-					logger.Log.Error(err)
-				}
+		resolvedSource, resolvedDestination, namespace := resolveIP(item.ConnectionInfo)
 
-				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
-				mizuEntry.ContractStatus = contract.Status
-				mizuEntry.ContractRequestReason = contract.RequestReason
-				mizuEntry.ContractResponseReason = contract.ResponseReason
-				mizuEntry.ContractContent = contract.Content
-			}
-
-			harEntry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
-			if err == nil {
-				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
-				mizuEntry.Rules = rules
-			}
-
-			entryWSource := oas.EntryWithSource{Entry: *harEntry, Source: mizuEntry.Source.Name}
-			oas.GetOasGeneratorInstance().PushEntry(&entryWSource)
+		if namespace == "" && item.Namespace != tapApi.UnknownNamespace {
+			namespace = item.Namespace
 		}
 
+		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestination, namespace)
+
 		data, err := json.Marshal(mizuEntry)
 		if err != nil {
-			panic(err)
+			logger.Log.Errorf("Error while marshaling entry: %v", err)
+			continue
 		}
 
-		providers.EntryAdded(len(data))
+		entryInserter := dependency.GetInstance(dependency.EntriesInserter).(EntryInserter)
+		if err := entryInserter.Insert(mizuEntry); err != nil {
+			logger.Log.Errorf("Error inserting entry, err: %v", err)
+		}
 
-		connection.SendText(string(data))
+		summary := extension.Dissector.Summarize(mizuEntry)
+		providers.EntryAdded(len(data), summary)
 
-		servicemap.GetInstance().NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
-		elastic.GetInstance().PushEntry(mizuEntry)
+		serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMapSink)
+		serviceMapGenerator.NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
+
+		oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGeneratorSink)
+		oasGenerator.HandleEntry(mizuEntry, &item.Protocol)
 	}
 }
 
-func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, resolvedDestination string) {
+func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, resolvedDestination string, namespace string) {
 	if k8sResolver != nil {
 		unresolvedSource := connectionInfo.ClientIP
-		resolvedSource = k8sResolver.Resolve(unresolvedSource)
-		if resolvedSource == "" {
+		resolvedSourceObject := k8sResolver.Resolve(unresolvedSource)
+		if resolvedSourceObject == nil {
 			logger.Log.Debugf("Cannot find resolved name to source: %s", unresolvedSource)
 			if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
 				return
 			}
+		} else {
+			resolvedSource = resolvedSourceObject.FullAddress
+			namespace = resolvedSourceObject.Namespace
 		}
+
 		unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
-		resolvedDestination = k8sResolver.Resolve(unresolvedDestination)
-		if resolvedDestination == "" {
+		resolvedDestinationObject := k8sResolver.Resolve(unresolvedDestination)
+		if resolvedDestinationObject == nil {
 			logger.Log.Debugf("Cannot find resolved name to dest: %s", unresolvedDestination)
 			if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
 				return
 			}
+		} else {
+			resolvedDestination = resolvedDestinationObject.FullAddress
+			// Overwrite namespace (if it was set according to the source)
+			// Only overwrite if non-empty
+			if resolvedDestinationObject.Namespace != "" {
+				namespace = resolvedDestinationObject.Namespace
+			}
 		}
 	}
-	return resolvedSource, resolvedDestination
+	return resolvedSource, resolvedDestination, namespace
 }
 
 func CheckIsServiceIP(address string) bool {

agent/pkg/api/socket_data_inserter.go

@@ -0,0 +1,71 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap/api"
+	"sync"
+	"time"
+)
+
+type EntryInserter interface {
+	Insert(entry *api.Entry) error
+}
+
+type BasenineEntryInserter struct {
+	connection *basenine.Connection
+}
+
+var instance *BasenineEntryInserter
+var once sync.Once
+
+func GetBasenineEntryInserterInstance() *BasenineEntryInserter {
+	once.Do(func() {
+		instance = &BasenineEntryInserter{}
+	})
+
+	return instance
+}
+
+func (e *BasenineEntryInserter) Insert(entry *api.Entry) error {
+	if e.connection == nil {
+		e.connection = initializeConnection()
+	}
+
+	data, err := json.Marshal(entry)
+	if err != nil {
+		return fmt.Errorf("error marshling entry, err: %v", err)
+	}
+
+	if err := e.connection.SendText(string(data)); err != nil {
+		e.connection.Close()
+		e.connection = nil
+
+		return fmt.Errorf("error sending text to database, err: %v", err)
+	}
+
+	return nil
+}
+
+func initializeConnection() *basenine.Connection{
+	for {
+		connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+		if err != nil {
+			logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
+			time.Sleep(shared.BasenineReconnectInterval * time.Second)
+			continue
+		}
+
+		if err = connection.InsertMode(); err != nil {
+			logger.Log.Errorf("Insert mode call failed: %v", err)
+			connection.Close()
+			time.Sleep(shared.BasenineReconnectInterval * time.Second)
+			continue
+		}
+
+		return connection
+	}
+}

agent/pkg/api/socket_data_streamer.go

@@ -0,0 +1,171 @@
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+type EntryStreamer interface {
+	Get(ctx context.Context, socketId int, params *WebSocketParams) error
+}
+
+type BasenineEntryStreamer struct{}
+
+func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *WebSocketParams) error {
+	var connection *basenine.Connection
+
+	entryStreamerSocketConnector := dependency.GetInstance(dependency.EntryStreamerSocketConnector).(EntryStreamerSocketConnector)
+
+	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
+	if err != nil {
+		logger.Log.Errorf("Failed to establish a connection to Basenine: %v", err)
+		entryStreamerSocketConnector.CleanupSocket(socketId)
+		return err
+	}
+
+	data := make(chan []byte)
+	meta := make(chan []byte)
+
+	query := params.Query
+	if err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query); err != nil {
+		if err := entryStreamerSocketConnector.SendToastError(socketId, err); err != nil {
+			return err
+		}
+
+		entryStreamerSocketConnector.CleanupSocket(socketId)
+		return err
+	}
+
+	leftOff, err := e.fetch(socketId, params, entryStreamerSocketConnector)
+	if err != nil {
+		logger.Log.Errorf("Fetch error: %v", err)
+	}
+
+	handleDataChannel := func(c *basenine.Connection, data chan []byte) {
+		for {
+			bytes := <-data
+
+			if string(bytes) == basenine.CloseChannel {
+				return
+			}
+
+			var entry *tapApi.Entry
+			if err = json.Unmarshal(bytes, &entry); err != nil {
+				logger.Log.Debugf("Error unmarshalling entry: %v", err)
+				continue
+			}
+
+			if err := entryStreamerSocketConnector.SendEntry(socketId, entry, params); err != nil {
+				logger.Log.Errorf("Error sending entry to socket, err: %v", err)
+				return
+			}
+		}
+	}
+
+	handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
+		for {
+			bytes := <-meta
+
+			if string(bytes) == basenine.CloseChannel {
+				return
+			}
+
+			var metadata *basenine.Metadata
+			if err = json.Unmarshal(bytes, &metadata); err != nil {
+				logger.Log.Debugf("Error unmarshalling metadata: %v", err)
+				continue
+			}
+
+			if err := entryStreamerSocketConnector.SendMetadata(socketId, metadata); err != nil {
+				logger.Log.Errorf("Error sending metadata to socket, err: %v", err)
+				return
+			}
+		}
+	}
+
+	go handleDataChannel(connection, data)
+	go handleMetaChannel(connection, meta)
+
+	if err = connection.Query(leftOff, query, data, meta); err != nil {
+		logger.Log.Errorf("Query mode call failed: %v", err)
+		entryStreamerSocketConnector.CleanupSocket(socketId)
+		return err
+	}
+
+	go func() {
+		<-ctx.Done()
+		data <- []byte(basenine.CloseChannel)
+		meta <- []byte(basenine.CloseChannel)
+		connection.Close()
+	}()
+
+	return nil
+}
+
+// Reverses a []byte slice.
+func (e *BasenineEntryStreamer) fetch(socketId int, params *WebSocketParams, connector EntryStreamerSocketConnector) (leftOff string, err error) {
+	if params.Fetch <= 0 {
+		leftOff = params.LeftOff
+		return
+	}
+
+	var data [][]byte
+	var firstMeta []byte
+	var lastMeta []byte
+	data, firstMeta, lastMeta, err = basenine.Fetch(
+		shared.BasenineHost,
+		shared.BaseninePort,
+		params.LeftOff,
+		-1,
+		params.Query,
+		params.Fetch,
+		time.Duration(params.TimeoutMs)*time.Millisecond,
+	)
+	if err != nil {
+		return
+	}
+
+	var firstMetadata *basenine.Metadata
+	if err = json.Unmarshal(firstMeta, &firstMetadata); err != nil {
+		return
+	}
+
+	leftOff = firstMetadata.LeftOff
+
+	var lastMetadata *basenine.Metadata
+	if err = json.Unmarshal(lastMeta, &lastMetadata); err != nil {
+		return
+	}
+
+	if err = connector.SendMetadata(socketId, lastMetadata); err != nil {
+		return
+	}
+
+	data = e.reverseBytesSlice(data)
+	for _, row := range data {
+		var entry *tapApi.Entry
+		if err = json.Unmarshal(row, &entry); err != nil {
+			break
+		}
+
+		if err = connector.SendEntry(socketId, entry, params); err != nil {
+			return
+		}
+	}
+	return
+}
+
+// Reverses a []byte slice.
+func (e *BasenineEntryStreamer) reverseBytesSlice(arr [][]byte) (newArr [][]byte) {
+	for i := len(arr) - 1; i >= 0; i-- {
+		newArr = append(newArr, arr[i])
+	}
+	return newArr
+}

agent/pkg/api/socket_routes.go

@@ -1,26 +1,33 @@
 package api
 
 import (
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"sync"
 	"time"
 
-	"github.com/up9inc/mizu/agent/pkg/models"
-
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+	"github.com/up9inc/mizu/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
+var (
+	extensionsMap map[string]*tapApi.Extension // global
+	protocolsMap  map[string]*tapApi.Protocol  //global
+)
+
+func InitMaps(extensions map[string]*tapApi.Extension, protocols map[string]*tapApi.Protocol) {
+	extensionsMap = extensions
+	protocolsMap = protocols
+}
+
 type EventHandlers interface {
-	WebSocketConnect(socketId int, isTapper bool)
+	WebSocketConnect(c *gin.Context, socketId int, isTapper bool)
 	WebSocketDisconnect(socketId int, isTapper bool)
-	WebSocketMessage(socketId int, message []byte)
+	WebSocketMessage(socketId int, isTapper bool, message []byte)
 }
 
 type SocketConnection struct {
@@ -30,34 +37,54 @@ type SocketConnection struct {
 	isTapper      bool
 }
 
-var websocketUpgrader = websocket.Upgrader{
-	ReadBufferSize:  1024,
-	WriteBufferSize: 1024,
+type WebSocketParams struct {
+	LeftOff           string `json:"leftOff"`
+	Query             string `json:"query"`
+	EnableFullEntries bool   `json:"enableFullEntries"`
+	Fetch             int    `json:"fetch"`
+	TimeoutMs         int    `json:"timeoutMs"`
 }
 
-var websocketIdsLock = sync.Mutex{}
-var connectedWebsockets map[int]*SocketConnection
-var connectedWebsocketIdCounter = 0
+var (
+	websocketUpgrader = websocket.Upgrader{
+		ReadBufferSize:  1024,
+		WriteBufferSize: 1024,
+	}
+
+	websocketIdsLock            = sync.Mutex{}
+	connectedWebsockets         map[int]*SocketConnection
+	connectedWebsocketIdCounter = 0
+	SocketGetBrowserHandler     gin.HandlerFunc
+	SocketGetTapperHandler      gin.HandlerFunc
+)
 
 func init() {
 	websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
 	connectedWebsockets = make(map[int]*SocketConnection)
 }
 
-func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
+func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
+	SocketGetBrowserHandler = func(c *gin.Context) {
+		websocketHandler(c, eventHandlers, false)
+	}
+
+	SocketGetTapperHandler = func(c *gin.Context) {
+		websocketHandler(c, eventHandlers, true)
+	}
+
 	app.GET("/ws", func(c *gin.Context) {
-		websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime)
+		SocketGetBrowserHandler(c)
 	})
 
 	app.GET("/wsTapper", func(c *gin.Context) { // TODO: add m2m authentication to this route
-		websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime)
+		SocketGetTapperHandler(c)
 	})
 }
 
-func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool, startTime int64) {
-	ws, err := websocketUpgrader.Upgrade(w, r, nil)
+func websocketHandler(c *gin.Context, eventHandlers EventHandlers, isTapper bool) {
+	ws, err := websocketUpgrader.Upgrade(c.Writer, c.Request, nil)
 	if err != nil {
-		logger.Log.Errorf("Failed to set websocket upgrade: %v", err)
+		logger.Log.Errorf("failed to set websocket upgrade: %v", err)
 		return
 	}
 
@@ -69,30 +96,13 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 
 	websocketIdsLock.Unlock()
 
-	var connection *basenine.Connection
-	var isQuerySet bool
-
-	// `!isTapper` means it's a connection from the web UI
-	if !isTapper {
-		connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	data := make(chan []byte)
-	meta := make(chan []byte)
-
 	defer func() {
 		socketCleanup(socketId, connectedWebsockets[socketId])
-		data <- []byte(basenine.CloseChannel)
-		meta <- []byte(basenine.CloseChannel)
-		connection.Close()
 	}()
 
-	eventHandlers.WebSocketConnect(socketId, isTapper)
+	eventHandlers.WebSocketConnect(c, socketId, isTapper)
 
-	startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime)
+	startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(utils.StartTime)
 
 	if err = SendToSocket(socketId, startTimeBytes); err != nil {
 		logger.Log.Error(err)
@@ -102,86 +112,48 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 		_, msg, err := ws.ReadMessage()
 		if err != nil {
 			if _, ok := err.(*websocket.CloseError); ok {
-				logger.Log.Debugf("Received websocket close message, socket id: %d", socketId)
+				logger.Log.Debugf("received websocket close message, socket id: %d", socketId)
 			} else {
-				logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
+				logger.Log.Errorf("error reading message, socket id: %d, error: %v", socketId, err)
 			}
 
 			break
 		}
 
-		if !isTapper && !isQuerySet {
-			query := string(msg)
-			err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
-			if err != nil {
-				toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
-					Type:      "error",
-					AutoClose: 5000,
-					Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
-				})
-				if err := SendToSocket(socketId, toastBytes); err != nil {
-					logger.Log.Error(err)
-				}
-				break
-			}
-
-			isQuerySet = true
-
-			handleDataChannel := func(c *basenine.Connection, data chan []byte) {
-				for {
-					bytes := <-data
-
-					if string(bytes) == basenine.CloseChannel {
-						return
-					}
-
-					var entry *tapApi.Entry
-					err = json.Unmarshal(bytes, &entry)
-
-					base := tapApi.Summarize(entry)
-
-					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
-					if err := SendToSocket(socketId, baseEntryBytes); err != nil {
-						logger.Log.Error(err)
-					}
-				}
-			}
-
-			handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
-				for {
-					bytes := <-meta
-
-					if string(bytes) == basenine.CloseChannel {
-						return
-					}
-
-					var metadata *basenine.Metadata
-					err = json.Unmarshal(bytes, &metadata)
-					if err != nil {
-						logger.Log.Debugf("Error recieving metadata: %v", err.Error())
-					}
-
-					metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
-					if err := SendToSocket(socketId, metadataBytes); err != nil {
-						logger.Log.Error(err)
-					}
-				}
-			}
-
-			go handleDataChannel(connection, data)
-			go handleMetaChannel(connection, meta)
-
-			connection.Query(query, data, meta)
-		} else {
-			eventHandlers.WebSocketMessage(socketId, msg)
-		}
+		eventHandlers.WebSocketMessage(socketId, isTapper, msg)
 	}
 }
 
+func SendToSocket(socketId int, message []byte) error {
+	socketObj := connectedWebsockets[socketId]
+	if socketObj == nil {
+		return fmt.Errorf("socket %v is disconnected", socketId)
+	}
+
+	socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
+	defer socketObj.lock.Unlock()
+
+	if connectedWebsockets[socketId] == nil {
+		return fmt.Errorf("socket %v is disconnected", socketId)
+	}
+
+	if err := socketObj.connection.SetWriteDeadline(time.Now().Add(time.Second * 10)); err != nil {
+		socketCleanup(socketId, socketObj)
+		return fmt.Errorf("error setting timeout to socket %v, err: %v", socketId, err)
+	}
+
+	if err := socketObj.connection.WriteMessage(websocket.TextMessage, message); err != nil {
+		socketCleanup(socketId, socketObj)
+		return fmt.Errorf("failed to write message to socket %v, err: %v", socketId, err)
+	}
+
+	return nil
+}
+
 func socketCleanup(socketId int, socketConnection *SocketConnection) {
 	err := socketConnection.connection.Close()
 	if err != nil {
-		logger.Log.Errorf("Error closing socket connection for socket id %d: %v", socketId, err)
+		logger.Log.Errorf("error closing socket connection for socket id %d: %v", socketId, err)
 	}
 
 	websocketIdsLock.Lock()
@@ -190,28 +162,3 @@ func socketCleanup(socketId int, socketConnection *SocketConnection) {
 
 	socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
 }
-
-func SendToSocket(socketId int, message []byte) error {
-	socketObj := connectedWebsockets[socketId]
-	if socketObj == nil {
-		return fmt.Errorf("Socket %v is disconnected", socketId)
-	}
-
-	var sent = false
-	time.AfterFunc(time.Second*5, func() {
-		if !sent {
-			logger.Log.Error("Socket timed out")
-			socketCleanup(socketId, socketObj)
-		}
-	})
-
-	socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
-	err := socketObj.connection.WriteMessage(1, message)
-	socketObj.lock.Unlock()
-	sent = true
-
-	if err != nil {
-		return fmt.Errorf("Failed to write message to socket %v, err: %w", socketId, err)
-	}
-	return nil
-}

agent/pkg/api/socket_server_handlers.go

@@ -1,22 +1,28 @@
 package api
 
 import (
+	"context"
 	"encoding/json"
-	"fmt"
 	"sync"
 
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
 	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
 	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
-	"github.com/up9inc/mizu/agent/pkg/up9"
 
 	tapApi "github.com/up9inc/mizu/tap/api"
 
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
-var browserClientSocketUUIDs = make([]int, 0)
+type BrowserClient struct {
+	dataStreamCancelFunc context.CancelFunc
+}
+
+var browserClients = make(map[int]*BrowserClient, 0)
+var tapperClientSocketUUIDs = make([]int, 0)
 var socketListLock = sync.Mutex{}
 
 type RoutesEventHandlers struct {
@@ -24,19 +30,25 @@ type RoutesEventHandlers struct {
 	SocketOutChannel chan<- *tapApi.OutputChannelItem
 }
 
-func init() {
-	go up9.UpdateAnalyzeStatus(BroadcastToBrowserClients)
-}
-
-func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
+func (h *RoutesEventHandlers) WebSocketConnect(_ *gin.Context, socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
 		tappers.Connected()
+
+		socketListLock.Lock()
+		tapperClientSocketUUIDs = append(tapperClientSocketUUIDs, socketId)
+		socketListLock.Unlock()
+
+		nodeToTappedPodMap := tappedPods.GetNodeToTappedPodMap()
+		SendTappedPods(socketId, nodeToTappedPodMap)
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
+
 		socketListLock.Lock()
-		browserClientSocketUUIDs = append(browserClientSocketUUIDs, socketId)
+		browserClients[socketId] = &BrowserClient{}
 		socketListLock.Unlock()
+
+		BroadcastTappedPodsStatus()
 	}
 }
 
@@ -44,16 +56,23 @@ func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
 		tappers.Disconnected()
+
+		socketListLock.Lock()
+		removeSocketUUIDFromTapperSlice(socketId)
+		socketListLock.Unlock()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
 		socketListLock.Lock()
-		removeSocketUUIDFromBrowserSlice(socketId)
+		if browserClients[socketId] != nil && browserClients[socketId].dataStreamCancelFunc != nil {
+			browserClients[socketId].dataStreamCancelFunc()
+		}
+		delete(browserClients, socketId)
 		socketListLock.Unlock()
 	}
 }
 
 func BroadcastToBrowserClients(message []byte) {
-	for _, socketId := range browserClientSocketUUIDs {
+	for socketId := range browserClients {
 		go func(socketId int) {
 			if err := SendToSocket(socketId, message); err != nil {
 				logger.Log.Error(err)
@@ -62,7 +81,43 @@ func BroadcastToBrowserClients(message []byte) {
 	}
 }
 
-func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
+func BroadcastToTapperClients(message []byte) {
+	for _, socketId := range tapperClientSocketUUIDs {
+		go func(socketId int) {
+			if err := SendToSocket(socketId, message); err != nil {
+				logger.Log.Error(err)
+			}
+		}(socketId)
+	}
+}
+
+func (h *RoutesEventHandlers) WebSocketMessage(socketId int, isTapper bool, message []byte) {
+	if isTapper {
+		HandleTapperIncomingMessage(message, h.SocketOutChannel, BroadcastToBrowserClients)
+	} else {
+		// we initiate the basenine stream after the first websocket message we receive (it contains the entry query), we then store a cancelfunc to later cancel this stream
+		if browserClients[socketId] != nil && browserClients[socketId].dataStreamCancelFunc == nil {
+			var params WebSocketParams
+			if err := json.Unmarshal(message, &params); err != nil {
+				logger.Log.Errorf("Error: %v", socketId, err)
+				return
+			}
+
+			entriesStreamer := dependency.GetInstance(dependency.EntriesSocketStreamer).(EntryStreamer)
+			ctx, cancelFunc := context.WithCancel(context.Background())
+			err := entriesStreamer.Get(ctx, socketId, &params)
+
+			if err != nil {
+				logger.Log.Errorf("error initializing basenine stream for browser socket %d %+v", socketId, err)
+				cancelFunc()
+			} else {
+				browserClients[socketId].dataStreamCancelFunc = cancelFunc
+			}
+		}
+	}
+}
+
+func HandleTapperIncomingMessage(message []byte, socketOutChannel chan<- *tapApi.OutputChannelItem, broadcastMessageFunc func([]byte)) {
 	var socketMessageBase shared.WebSocketMessageMetadata
 	err := json.Unmarshal(message, &socketMessageBase)
 	if err != nil {
@@ -76,7 +131,7 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
 			} else {
 				// NOTE: This is where the message comes back from the intermediate WebSocket to code.
-				h.SocketOutChannel <- tappedEntryMessage.Data
+				socketOutChannel <- tappedEntryMessage.Data
 			}
 		case shared.WebSocketMessageTypeUpdateStatus:
 			var statusMessage shared.WebSocketStatusMessage
@@ -84,15 +139,7 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 			if err != nil {
 				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
 			} else {
-				BroadcastToBrowserClients(message)
-			}
-		case shared.WebsocketMessageTypeOutboundLink:
-			var outboundLinkMessage models.WebsocketOutboundLinkMessage
-			err := json.Unmarshal(message, &outboundLinkMessage)
-			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
-			} else {
-				handleTLSLink(outboundLinkMessage)
+				broadcastMessageFunc(message)
 			}
 		default:
 			logger.Log.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
@@ -100,35 +147,12 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 	}
 }
 
-func handleTLSLink(outboundLinkMessage models.WebsocketOutboundLinkMessage) {
-	resolvedName := k8sResolver.Resolve(outboundLinkMessage.Data.DstIP)
-	if resolvedName != "" {
-		outboundLinkMessage.Data.DstIP = resolvedName
-	} else if outboundLinkMessage.Data.SuggestedResolvedName != "" {
-		outboundLinkMessage.Data.DstIP = outboundLinkMessage.Data.SuggestedResolvedName
-	}
-	cacheKey := fmt.Sprintf("%s -> %s:%d", outboundLinkMessage.Data.Src, outboundLinkMessage.Data.DstIP, outboundLinkMessage.Data.DstPort)
-	_, isInCache := providers.RecentTLSLinks.Get(cacheKey)
-	if isInCache {
-		return
-	} else {
-		providers.RecentTLSLinks.SetDefault(cacheKey, outboundLinkMessage.Data)
-	}
-	marshaledMessage, err := json.Marshal(outboundLinkMessage)
-	if err != nil {
-		logger.Log.Errorf("Error marshaling outbound link message for broadcasting: %v", err)
-	} else {
-		logger.Log.Errorf("Broadcasting outboundlink message %s", string(marshaledMessage))
-		BroadcastToBrowserClients(marshaledMessage)
-	}
-}
-
-func removeSocketUUIDFromBrowserSlice(uuidToRemove int) {
-	newUUIDSlice := make([]int, 0, len(browserClientSocketUUIDs))
-	for _, uuid := range browserClientSocketUUIDs {
+func removeSocketUUIDFromTapperSlice(uuidToRemove int) {
+	newUUIDSlice := make([]int, 0, len(tapperClientSocketUUIDs))
+	for _, uuid := range tapperClientSocketUUIDs {
 		if uuid != uuidToRemove {
 			newUUIDSlice = append(newUUIDSlice, uuid)
 		}
 	}
-	browserClientSocketUUIDs = newUUIDSlice
+	tapperClientSocketUUIDs = newUUIDSlice
 }

agent/pkg/api/utils.go

@@ -0,0 +1,40 @@
+package api
+
+import (
+	"encoding/json"
+
+	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+)
+
+func BroadcastTappedPodsStatus() {
+	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
+
+	message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
+	if jsonBytes, err := json.Marshal(message); err != nil {
+		logger.Log.Errorf("Could not Marshal message %v", err)
+	} else {
+		BroadcastToBrowserClients(jsonBytes)
+	}
+}
+
+func SendTappedPods(socketId int, nodeToTappedPodMap shared.NodeToPodsMap) {
+	message := shared.CreateWebSocketTappedPodsMessage(nodeToTappedPodMap)
+	if jsonBytes, err := json.Marshal(message); err != nil {
+		logger.Log.Errorf("Could not Marshal message %v", err)
+	} else {
+		if err := SendToSocket(socketId, jsonBytes); err != nil {
+			logger.Log.Error(err)
+		}
+	}
+}
+
+func BroadcastTappedPodsToTappers(nodeToTappedPodMap shared.NodeToPodsMap) {
+	message := shared.CreateWebSocketTappedPodsMessage(nodeToTappedPodMap)
+	if jsonBytes, err := json.Marshal(message); err != nil {
+		logger.Log.Errorf("Could not Marshal message %v", err)
+	} else {
+		BroadcastToTapperClients(jsonBytes)
+	}
+}

agent/pkg/app/main.go

@@ -0,0 +1,138 @@
+package app
+
+import (
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/antelman107/net-wait-go/wait"
+	"github.com/op/go-logging"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/agent/pkg/api"
+	"github.com/up9inc/mizu/agent/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+	"github.com/up9inc/mizu/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
+	"github.com/up9inc/mizu/tap/dbgctl"
+	amqpExt "github.com/up9inc/mizu/tap/extensions/amqp"
+	httpExt "github.com/up9inc/mizu/tap/extensions/http"
+	kafkaExt "github.com/up9inc/mizu/tap/extensions/kafka"
+	redisExt "github.com/up9inc/mizu/tap/extensions/redis"
+)
+
+var (
+	Extensions    []*tapApi.Extension          // global
+	ExtensionsMap map[string]*tapApi.Extension // global
+	ProtocolsMap  map[string]*tapApi.Protocol  //global
+)
+
+func LoadExtensions() {
+	Extensions = make([]*tapApi.Extension, 0)
+	ExtensionsMap = make(map[string]*tapApi.Extension)
+	ProtocolsMap = make(map[string]*tapApi.Protocol)
+
+	extensionHttp := &tapApi.Extension{}
+	dissectorHttp := httpExt.NewDissector()
+	dissectorHttp.Register(extensionHttp)
+	extensionHttp.Dissector = dissectorHttp
+	Extensions = append(Extensions, extensionHttp)
+	ExtensionsMap[extensionHttp.Protocol.Name] = extensionHttp
+	protocolsHttp := dissectorHttp.GetProtocols()
+	for k, v := range protocolsHttp {
+		ProtocolsMap[k] = v
+	}
+
+	if !dbgctl.MizuTapperDisableNonHttpExtensions {
+		extensionAmqp := &tapApi.Extension{}
+		dissectorAmqp := amqpExt.NewDissector()
+		dissectorAmqp.Register(extensionAmqp)
+		extensionAmqp.Dissector = dissectorAmqp
+		Extensions = append(Extensions, extensionAmqp)
+		ExtensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
+		protocolsAmqp := dissectorAmqp.GetProtocols()
+		for k, v := range protocolsAmqp {
+			ProtocolsMap[k] = v
+		}
+
+		extensionKafka := &tapApi.Extension{}
+		dissectorKafka := kafkaExt.NewDissector()
+		dissectorKafka.Register(extensionKafka)
+		extensionKafka.Dissector = dissectorKafka
+		Extensions = append(Extensions, extensionKafka)
+		ExtensionsMap[extensionKafka.Protocol.Name] = extensionKafka
+		protocolsKafka := dissectorKafka.GetProtocols()
+		for k, v := range protocolsKafka {
+			ProtocolsMap[k] = v
+		}
+
+		extensionRedis := &tapApi.Extension{}
+		dissectorRedis := redisExt.NewDissector()
+		dissectorRedis.Register(extensionRedis)
+		extensionRedis.Dissector = dissectorRedis
+		Extensions = append(Extensions, extensionRedis)
+		ExtensionsMap[extensionRedis.Protocol.Name] = extensionRedis
+		protocolsRedis := dissectorRedis.GetProtocols()
+		for k, v := range protocolsRedis {
+			ProtocolsMap[k] = v
+		}
+	}
+
+	sort.Slice(Extensions, func(i, j int) bool {
+		return Extensions[i].Protocol.Priority < Extensions[j].Protocol.Priority
+	})
+
+	api.InitMaps(ExtensionsMap, ProtocolsMap)
+	providers.InitProtocolToColor(ProtocolsMap)
+}
+
+func ConfigureBasenineServer(host string, port string, dbSize int64, logLevel logging.Level, insertionFilter string) {
+	if !wait.New(
+		wait.WithProto("tcp"),
+		wait.WithWait(200*time.Millisecond),
+		wait.WithBreak(50*time.Millisecond),
+		wait.WithDeadline(20*time.Second),
+		wait.WithDebug(logLevel == logging.DEBUG),
+	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
+		logger.Log.Panicf("Basenine is not available!")
+	}
+
+	if err := basenine.Limit(host, port, dbSize); err != nil {
+		logger.Log.Panicf("Error while limiting database size: %v", err)
+	}
+
+	// Define the macros
+	for _, extension := range Extensions {
+		macros := extension.Dissector.Macros()
+		for macro, expanded := range macros {
+			if err := basenine.Macro(host, port, macro, expanded); err != nil {
+				logger.Log.Panicf("Error while adding a macro: %v", err)
+			}
+		}
+	}
+
+	// Set the insertion filter that comes from the config
+	if err := basenine.InsertionFilter(host, port, insertionFilter); err != nil {
+		logger.Log.Errorf("Error while setting the insertion filter: %v", err)
+	}
+
+	utils.StartTime = time.Now().UnixNano() / int64(time.Millisecond)
+}
+
+func GetEntryInputChannel() chan *tapApi.OutputChannelItem {
+	outputItemsChannel := make(chan *tapApi.OutputChannelItem)
+	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+	go FilterItems(outputItemsChannel, filteredOutputItemsChannel)
+	go api.StartReadingEntries(filteredOutputItemsChannel, nil, ExtensionsMap)
+
+	return outputItemsChannel
+}
+
+func FilterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
+	for message := range inChannel {
+		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
+			continue
+		}
+
+		outChannel <- message
+	}
+}

agent/pkg/controllers/config_controller.go

@@ -1,155 +0,0 @@
-package controllers
-
-import (
-	"context"
-	"net/http"
-	"regexp"
-	"time"
-
-	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/agent/pkg/config"
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/providers"
-	"github.com/up9inc/mizu/agent/pkg/providers/tapConfig"
-	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
-	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
-	v1 "k8s.io/api/core/v1"
-)
-
-var cancelTapperSyncer context.CancelFunc
-
-func PostTapConfig(c *gin.Context) {
-	requestTapConfig := &models.TapConfig{}
-
-	if err := c.Bind(requestTapConfig); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-		return
-	}
-
-	if cancelTapperSyncer != nil {
-		cancelTapperSyncer()
-
-		tappedPods.Set([]*shared.PodInfo{})
-		tappers.ResetStatus()
-
-		broadcastTappedPodsStatus()
-	}
-
-	var tappedNamespaces []string
-	for namespace, tapped := range requestTapConfig.TappedNamespaces {
-		if tapped {
-			tappedNamespaces = append(tappedNamespaces, namespace)
-		}
-	}
-
-	podRegex, _ := regexp.Compile(".*")
-
-	kubernetesProvider, err := providers.GetKubernetesProvider()
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, err)
-		return
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-
-	if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{}, tapApi.TrafficFilteringOptions{}, false); err != nil {
-		c.JSON(http.StatusInternalServerError, err)
-		cancel()
-		return
-	}
-
-	cancelTapperSyncer = cancel
-	tapConfig.Save(requestTapConfig)
-
-	c.JSON(http.StatusOK, "OK")
-}
-
-func GetTapConfig(c *gin.Context) {
-	kubernetesProvider, err := providers.GetKubernetesProvider()
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, err)
-		return
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	namespaces, err := kubernetesProvider.ListAllNamespaces(ctx)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, err)
-		return
-	}
-
-	savedTapConfig := tapConfig.Get()
-
-	tappedNamespaces := make(map[string]bool)
-	for _, namespace := range namespaces {
-		if namespace.Name == config.Config.MizuResourcesNamespace {
-			continue
-		}
-
-		tappedNamespaces[namespace.Name] = savedTapConfig.TappedNamespaces[namespace.Name]
-	}
-
-	tapConfigToReturn := models.TapConfig{TappedNamespaces: tappedNamespaces}
-	c.JSON(http.StatusOK, tapConfigToReturn)
-}
-
-func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, serviceMesh bool) (*kubernetes.MizuTapperSyncer, error) {
-	tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
-		TargetNamespaces:         targetNamespaces,
-		PodFilterRegex:           podFilterRegex,
-		MizuResourcesNamespace:   config.Config.MizuResourcesNamespace,
-		AgentImage:               config.Config.AgentImage,
-		TapperResources:          config.Config.TapperResources,
-		ImagePullPolicy:          v1.PullPolicy(config.Config.PullPolicy),
-		LogLevel:                 config.Config.LogLevel,
-		IgnoredUserAgents:        ignoredUserAgents,
-		MizuApiFilteringOptions:  mizuApiFilteringOptions,
-		MizuServiceAccountExists: true, //assume service account exists since install mode will not function without it anyway
-		ServiceMesh:              serviceMesh,
-	}, time.Now())
-
-	if err != nil {
-		return nil, err
-	}
-
-	// handle tapperSyncer events (pod changes and errors)
-	go func() {
-		for {
-			select {
-			case syncerErr, ok := <-tapperSyncer.ErrorOut:
-				if !ok {
-					logger.Log.Debug("mizuTapperSyncer err channel closed, ending listener loop")
-					return
-				}
-				logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr)
-			case _, ok := <-tapperSyncer.TapPodChangesOut:
-				if !ok {
-					logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop")
-					return
-				}
-
-				tappedPods.Set(kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods))
-				broadcastTappedPodsStatus()
-			case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut:
-				if !ok {
-					logger.Log.Debug("mizuTapperSyncer tapper status changed channel closed, ending listener loop")
-					return
-				}
-
-				tappers.SetStatus(&tapperStatus)
-				broadcastTappedPodsStatus()
-			case <-ctx.Done():
-				logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")
-				return
-			}
-		}
-	}()
-
-	return tapperSyncer, nil
-}

agent/pkg/controllers/db_controller.go

@@ -0,0 +1,28 @@
+package controllers
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/agent/pkg/app"
+	"github.com/up9inc/mizu/agent/pkg/config"
+	"github.com/up9inc/mizu/shared"
+)
+
+func Flush(c *gin.Context) {
+	if err := basenine.Flush(shared.BasenineHost, shared.BaseninePort); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	} else {
+		c.JSON(http.StatusOK, "Flushed.")
+	}
+}
+
+func Reset(c *gin.Context) {
+	if err := basenine.Reset(shared.BasenineHost, shared.BaseninePort); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+	} else {
+		app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel, config.Config.InsertionFilter)
+		c.JSON(http.StatusOK, "Resetted.")
+	}
+}

agent/pkg/controllers/entries_controller.go

@@ -1,30 +1,19 @@
 package controllers
 
 import (
-	"encoding/json"
 	"net/http"
-	"strconv"
-	"time"
 
-	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/entries"
 	"github.com/up9inc/mizu/agent/pkg/models"
 	"github.com/up9inc/mizu/agent/pkg/validation"
 
 	"github.com/gin-gonic/gin"
 
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
+	"github.com/up9inc/mizu/logger"
 )
 
-var extensionsMap map[string]*tapApi.Extension // global
-
-func InitExtensionsMap(ref map[string]*tapApi.Extension) {
-	extensionsMap = ref
-}
-
-func Error(c *gin.Context, err error) bool {
+func HandleEntriesError(c *gin.Context, err error) bool {
 	if err != nil {
 		logger.Log.Errorf("Error getting entry: %v", err)
 		_ = c.Error(err)
@@ -54,44 +43,18 @@ func GetEntries(c *gin.Context) {
 		entriesRequest.TimeoutMs = 3000
 	}
 
-	data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
-		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
-		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, validationError)
-	}
-
-	response := &models.EntriesResponse{}
-	var dataSlice []interface{}
-
-	for _, row := range data {
-		var entry *tapApi.Entry
-		err = json.Unmarshal(row, &entry)
-		if err != nil {
-			c.JSON(http.StatusBadRequest, gin.H{
-				"error":     true,
-				"type":      "error",
-				"autoClose": "5000",
-				"msg":       string(row),
-			})
-			return // exit
+	entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
+	entryWrappers, metadata, err := entriesProvider.GetEntries(entriesRequest)
+	if !HandleEntriesError(c, err) {
+		baseEntries := make([]interface{}, 0)
+		for _, entry := range entryWrappers {
+			baseEntries = append(baseEntries, entry.Base)
 		}
-
-		base := tapApi.Summarize(entry)
-
-		dataSlice = append(dataSlice, base)
+		c.JSON(http.StatusOK, models.EntriesResponse{
+			Data: baseEntries,
+			Meta: metadata,
+		})
 	}
-
-	var metadata *basenine.Metadata
-	err = json.Unmarshal(meta, &metadata)
-	if err != nil {
-		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
-	}
-
-	response.Data = dataSlice
-	response.Meta = metadata
-
-	c.JSON(http.StatusOK, response)
 }
 
 func GetEntry(c *gin.Context) {
@@ -105,44 +68,12 @@ func GetEntry(c *gin.Context) {
 		c.JSON(http.StatusBadRequest, validationError)
 	}
 
-	id, _ := strconv.Atoi(c.Param("id"))
-	var entry *tapApi.Entry
-	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id, singleEntryRequest.Query)
-	if Error(c, err) {
-		return // exit
-	}
-	err = json.Unmarshal(bytes, &entry)
-	if err != nil {
-		c.JSON(http.StatusNotFound, gin.H{
-			"error":     true,
-			"type":      "error",
-			"autoClose": "5000",
-			"msg":       string(bytes),
-		})
-		return // exit
-	}
+	id := c.Param("id")
 
-	extension := extensionsMap[entry.Protocol.Name]
-	representation, bodySize, _ := extension.Dissector.Represent(entry.Request, entry.Response)
+	entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
+	entry, err := entriesProvider.GetEntry(singleEntryRequest, id)
 
-	var rules []map[string]interface{}
-	var isRulesEnabled bool
-	if entry.Protocol.Name == "http" {
-		harEntry, _ := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
-		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
-		isRulesEnabled = _isRulesEnabled
-		inrec, _ := json.Marshal(rulesMatched)
-		if err := json.Unmarshal(inrec, &rules); err != nil {
-			logger.Log.Error(err)
-		}
+	if !HandleEntriesError(c, err) {
+		c.JSON(http.StatusOK, entry)
 	}
-
-	c.JSON(http.StatusOK, tapApi.EntryWrapper{
-		Protocol:       entry.Protocol,
-		Representation: string(representation),
-		BodySize:       bodySize,
-		Data:           entry,
-		Rules:          rules,
-		IsRulesEnabled: isRulesEnabled,
-	})
 }

agent/pkg/controllers/install_controller.go

@@ -1,24 +0,0 @@
-package controllers
-
-import (
-	"net/http"
-
-	"github.com/up9inc/mizu/agent/pkg/providers"
-
-	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-func IsSetupNecessary(c *gin.Context) {
-	if IsInstallNeeded, err := providers.IsInstallNeeded(); err != nil {
-		logger.Log.Errorf("unknown internal while checking if install is needed %s", err)
-		c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while checking if install is needed"})
-	} else {
-		c.JSON(http.StatusOK, IsInstallNeeded)
-	}
-}
-
-func SetupAdminUser(c *gin.Context) {
-	token, err, formErrorMessages := providers.CreateAdminUser(c.PostForm("password"), c.Request.Context())
-	handleRegistration(token, err, formErrorMessages, c)
-}

agent/pkg/controllers/oas_controller.go

@@ -5,13 +5,15 @@ import (
 
 	"github.com/chanced/openapi"
 	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
 	"github.com/up9inc/mizu/agent/pkg/oas"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 func GetOASServers(c *gin.Context) {
 	m := make([]string, 0)
-	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
+	oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
+	oasGenerator.GetServiceSpecs().Range(func(key, value interface{}) bool {
 		m = append(m, key.(string))
 		return true
 	})
@@ -20,7 +22,8 @@ func GetOASServers(c *gin.Context) {
 }
 
 func GetOASSpec(c *gin.Context) {
-	res, ok := oas.GetOasGeneratorInstance().ServiceSpecs.Load(c.Param("id"))
+	oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
+	res, ok := oasGenerator.GetServiceSpecs().Load(c.Param("id"))
 	if !ok {
 		c.JSON(http.StatusNotFound, gin.H{
 			"error":     true,
@@ -48,7 +51,9 @@ func GetOASSpec(c *gin.Context) {
 
 func GetOASAllSpecs(c *gin.Context) {
 	res := map[string]*openapi.OpenAPI{}
-	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
+
+	oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
+	oasGenerator.GetServiceSpecs().Range(func(key, value interface{}) bool {
 		svc := key.(string)
 		gen := value.(*oas.SpecGen)
 		spec, err := gen.GetSpec()

agent/pkg/controllers/oas_controller_test.go

@@ -4,39 +4,42 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/up9inc/mizu/agent/pkg/oas"
-
 	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/dependency"
+	"github.com/up9inc/mizu/agent/pkg/oas"
 )
 
 func TestGetOASServers(t *testing.T) {
-	recorder := httptest.NewRecorder()
-	c, _ := gin.CreateTestContext(recorder)
-	oas.GetOasGeneratorInstance().Start()
-	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+	recorder, c := getRecorderAndContext()
 
 	GetOASServers(c)
 	t.Logf("Written body: %s", recorder.Body.String())
 }
 
 func TestGetOASAllSpecs(t *testing.T) {
-	recorder := httptest.NewRecorder()
-	c, _ := gin.CreateTestContext(recorder)
-	oas.GetOasGeneratorInstance().Start()
-	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+	recorder, c := getRecorderAndContext()
 
 	GetOASAllSpecs(c)
 	t.Logf("Written body: %s", recorder.Body.String())
 }
 
 func TestGetOASSpec(t *testing.T) {
-	recorder := httptest.NewRecorder()
-	c, _ := gin.CreateTestContext(recorder)
-	oas.GetOasGeneratorInstance().Start()
-	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
+	recorder, c := getRecorderAndContext()
 
 	c.Params = []gin.Param{{Key: "id", Value: "some"}}
 
 	GetOASSpec(c)
 	t.Logf("Written body: %s", recorder.Body.String())
 }
+
+func getRecorderAndContext() (*httptest.ResponseRecorder, *gin.Context) {
+	dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} {
+		return oas.GetDefaultOasGeneratorInstance(-1)
+	})
+
+	recorder := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(recorder)
+	oas.GetDefaultOasGeneratorInstance(-1).Start()
+	oas.GetDefaultOasGeneratorInstance(-1).GetServiceSpecs().Store("some", oas.NewGen("some"))
+	return recorder, c
+}

agent/pkg/controllers/replay_controller.go

@@ -0,0 +1,34 @@
+package controllers
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/replay"
+	"github.com/up9inc/mizu/agent/pkg/validation"
+	"github.com/up9inc/mizu/logger"
+)
+
+const (
+	replayTimeout = 10 * time.Second
+)
+
+func ReplayRequest(c *gin.Context) {
+	logger.Log.Debug("Starting replay")
+	replayDetails := &replay.Details{}
+	if err := c.Bind(replayDetails); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+
+	logger.Log.Debugf("Validating replay, %v", replayDetails)
+	if err := validation.Validate(replayDetails); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+
+	logger.Log.Debug("Executing replay, %v", replayDetails)
+	result := replay.ExecuteRequest(replayDetails, replayTimeout)
+	c.JSON(http.StatusOK, result)
+}

agent/pkg/controllers/service_map_controller.go

@@ -3,6 +3,7 @@ package controllers
 import (
 	"net/http"
 
+	"github.com/up9inc/mizu/agent/pkg/dependency"
 	"github.com/up9inc/mizu/agent/pkg/servicemap"
 
 	"github.com/gin-gonic/gin"
@@ -13,8 +14,9 @@ type ServiceMapController struct {
 }
 
 func NewServiceMapController() *ServiceMapController {
+	serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap)
 	return &ServiceMapController{
-		service: servicemap.GetInstance(),
+		service: serviceMapGenerator,
 	}
 }
 

agent/pkg/controllers/service_map_controller_test.go

@@ -7,11 +7,11 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/up9inc/mizu/agent/pkg/dependency"
 	"github.com/up9inc/mizu/agent/pkg/servicemap"
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
-	"github.com/up9inc/mizu/shared"
 	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
@@ -58,11 +58,11 @@ type ServiceMapControllerSuite struct {
 }
 
 func (s *ServiceMapControllerSuite) SetupTest() {
+	dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
+
 	s.c = NewServiceMapController()
-	s.c.service.SetConfig(&shared.MizuAgentConfig{
-		ServiceMap: true,
-	})
-	s.c.service.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
+	s.c.service.Enable()
+	s.c.service.(servicemap.ServiceMapSink).NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
 
 	s.w = httptest.NewRecorder()
 	s.g, _ = gin.CreateTestContext(s.w)
@@ -101,16 +101,18 @@ func (s *ServiceMapControllerSuite) TestGet() {
 
 	// response nodes
 	aNode := servicemap.ServiceMapNode{
-		Id:    1,
-		Name:  TCPEntryA.IP,
-		Entry: TCPEntryA,
-		Count: 1,
+		Id:       1,
+		Name:     TCPEntryA.Name,
+		Entry:    TCPEntryA,
+		Resolved: true,
+		Count:    1,
 	}
 	bNode := servicemap.ServiceMapNode{
-		Id:    2,
-		Name:  TCPEntryB.IP,
-		Entry: TCPEntryB,
-		Count: 1,
+		Id:       2,
+		Name:     TCPEntryB.Name,
+		Entry:    TCPEntryB,
+		Resolved: true,
+		Count:    1,
 	}
 	assert.Contains(response.Nodes, aNode)
 	assert.Contains(response.Nodes, bNode)

agent/pkg/controllers/status_controller.go

@@ -1,19 +1,20 @@
 package controllers
 
 import (
-	"encoding/json"
 	"net/http"
 
+	core "k8s.io/api/core/v1"
+
 	"github.com/gin-gonic/gin"
 	"github.com/up9inc/mizu/agent/pkg/api"
 	"github.com/up9inc/mizu/agent/pkg/holder"
 	"github.com/up9inc/mizu/agent/pkg/providers"
 	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
 	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
-	"github.com/up9inc/mizu/agent/pkg/up9"
 	"github.com/up9inc/mizu/agent/pkg/validation"
+	"github.com/up9inc/mizu/logger"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/shared/kubernetes"
 )
 
 func HealthCheck(c *gin.Context) {
@@ -31,26 +32,21 @@ func HealthCheck(c *gin.Context) {
 }
 
 func PostTappedPods(c *gin.Context) {
-	var requestTappedPods []*shared.PodInfo
+	var requestTappedPods []core.Pod
 	if err := c.Bind(&requestTappedPods); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
 
+	podInfos := kubernetes.GetPodInfosForPods(requestTappedPods)
+
 	logger.Log.Infof("[Status] POST request: %d tapped pods", len(requestTappedPods))
-	tappedPods.Set(requestTappedPods)
-	broadcastTappedPodsStatus()
-}
+	tappedPods.Set(podInfos)
+	api.BroadcastTappedPodsStatus()
 
-func broadcastTappedPodsStatus() {
-	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
-
-	message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
-	if jsonBytes, err := json.Marshal(message); err != nil {
-		logger.Log.Errorf("Could not Marshal message %v", err)
-	} else {
-		api.BroadcastToBrowserClients(jsonBytes)
-	}
+	nodeToTappedPodMap := kubernetes.GetNodeHostToTappedPodsMap(requestTappedPods)
+	tappedPods.SetNodeToTappedPodMap(nodeToTappedPodMap)
+	api.BroadcastTappedPodsToTappers(nodeToTappedPodMap)
 }
 
 func PostTapperStatus(c *gin.Context) {
@@ -67,38 +63,24 @@ func PostTapperStatus(c *gin.Context) {
 
 	logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus)
 	tappers.SetStatus(tapperStatus)
-	broadcastTappedPodsStatus()
+	api.BroadcastTappedPodsStatus()
 }
 
 func GetConnectedTappersCount(c *gin.Context) {
 	c.JSON(http.StatusOK, tappers.GetConnectedCount())
 }
 
-func GetAuthStatus(c *gin.Context) {
-	authStatus, err := providers.GetAuthStatus()
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, err)
-		return
-	}
-
-	c.JSON(http.StatusOK, authStatus)
-}
-
 func GetTappingStatus(c *gin.Context) {
 	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
 	c.JSON(http.StatusOK, tappedPodsStatus)
 }
 
-func AnalyzeInformation(c *gin.Context) {
-	c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
-}
-
 func GetGeneralStats(c *gin.Context) {
 	c.JSON(http.StatusOK, providers.GetGeneralStats())
 }
 
-func GetRecentTLSLinks(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
+func GetTrafficStats(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.GetTrafficStats())
 }
 
 func GetCurrentResolvingInformation(c *gin.Context) {

agent/pkg/controllers/user_controller.go

@@ -1,46 +0,0 @@
-package controllers
-
-import (
-	"github.com/up9inc/mizu/agent/pkg/providers"
-
-	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/shared/logger"
-
-	ory "github.com/ory/kratos-client-go"
-)
-
-func Login(c *gin.Context) {
-	if token, err := providers.PerformLogin(c.PostForm("username"), c.PostForm("password"), c.Request.Context()); err != nil {
-		c.AbortWithStatusJSON(401, gin.H{"error": "bad login"})
-	} else {
-		c.JSON(200, gin.H{"token": token})
-	}
-}
-
-func Logout(c *gin.Context) {
-	token := c.GetHeader("x-session-token")
-	if err := providers.Logout(token, c.Request.Context()); err != nil {
-		c.AbortWithStatusJSON(500, gin.H{"error": "error occured while logging out, the session might still be valid"})
-	} else {
-		c.JSON(200, "")
-	}
-}
-
-func Register(c *gin.Context) {
-	token, _, err, formErrorMessages := providers.RegisterUser(c.PostForm("username"), c.PostForm("password"), c.Request.Context())
-	handleRegistration(token, err, formErrorMessages, c)
-}
-
-func handleRegistration(token *string, err error, formErrorMessages map[string][]ory.UiText, c *gin.Context) {
-	if err != nil {
-		if formErrorMessages != nil {
-			logger.Log.Infof("user attempted to register but had form errors %v %v", formErrorMessages, err)
-			c.AbortWithStatusJSON(400, formErrorMessages)
-		} else {
-			logger.Log.Errorf("unknown internal error registering user %s", err)
-			c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while registering"})
-		}
-	} else {
-		c.JSON(201, gin.H{"token": token})
-	}
-}

agent/pkg/dependency/container.go

@@ -0,0 +1,11 @@
+package dependency
+
+var typeInitializerMap = make(map[ContainerType]func() interface{}, 0)
+
+func RegisterGenerator(name ContainerType, fn func() interface{}) {
+	typeInitializerMap[name] = fn
+}
+
+func GetInstance(name ContainerType) interface{} {
+	return typeInitializerMap[name]()
+}

agent/pkg/dependency/type_names.go

@@ -0,0 +1,12 @@
+package dependency
+
+type ContainerType string
+
+const (
+	ServiceMapGeneratorDependency ContainerType = "ServiceMapGeneratorDependency"
+	OasGeneratorDependency        ContainerType = "OasGeneratorDependency"
+	EntriesInserter               ContainerType = "EntriesInserter"
+	EntriesProvider               ContainerType = "EntriesProvider"
+	EntriesSocketStreamer         ContainerType = "EntriesSocketStreamer"
+	EntryStreamerSocketConnector  ContainerType = "EntryStreamerSocketConnector"
+)

agent/pkg/elastic/esClient.go

@@ -1,120 +0,0 @@
-package elastic
-
-import (
-	"bytes"
-	"crypto/tls"
-	"encoding/json"
-	"github.com/elastic/go-elasticsearch/v7"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/tap/api"
-	"net/http"
-	"sync"
-	"time"
-)
-
-type client struct {
-	es            *elasticsearch.Client
-	index         string
-	insertedCount int
-}
-
-var instance *client
-var once sync.Once
-
-func GetInstance() *client {
-	once.Do(func() {
-		instance = newClient()
-	})
-	return instance
-}
-
-func (client *client) Configure(config shared.ElasticConfig) {
-	if config.Url == "" || config.User == "" || config.Password == "" {
-		logger.Log.Infof("No elastic configuration was supplied, elastic exporter disabled")
-		return
-	}
-	transport := http.DefaultTransport
-	tlsClientConfig := &tls.Config{InsecureSkipVerify: true}
-	transport.(*http.Transport).TLSClientConfig = tlsClientConfig
-	cfg := elasticsearch.Config{
-		Addresses: []string{config.Url},
-		Username:  config.User,
-		Password:  config.Password,
-		Transport: transport,
-	}
-
-	es, err := elasticsearch.NewClient(cfg)
-	if err != nil {
-		logger.Log.Fatalf("Failed to initialize elastic client %v", err)
-	}
-
-	// Have the client instance return a response
-	res, err := es.Info()
-	if err != nil {
-		logger.Log.Fatalf("Elastic client.Info() ERROR: %v", err)
-	} else {
-		client.es = es
-		client.index = "mizu_traffic_http_" + time.Now().Format("2006_01_02_15_04")
-		client.insertedCount = 0
-		logger.Log.Infof("Elastic client configured, index: %s, cluster info: %v", client.index, res)
-	}
-	defer res.Body.Close()
-}
-
-func newClient() *client {
-	return &client{
-		es:    nil,
-		index: "",
-	}
-}
-
-type httpEntry struct {
-	Source      *api.TCP               `json:"src"`
-	Destination *api.TCP               `json:"dst"`
-	Outgoing    bool                   `json:"outgoing"`
-	CreatedAt   time.Time              `json:"createdAt"`
-	Request     map[string]interface{} `json:"request"`
-	Response    map[string]interface{} `json:"response"`
-	Summary     string                 `json:"summary"`
-	Method      string                 `json:"method"`
-	Status      int                    `json:"status"`
-	ElapsedTime int64                  `json:"elapsedTime"`
-	Path        string                 `json:"path"`
-}
-
-func (client *client) PushEntry(entry *api.Entry) {
-	if client.es == nil {
-		return
-	}
-
-	if entry.Protocol.Name != "http" {
-		return
-	}
-
-	entryToPush := httpEntry{
-		Source:      entry.Source,
-		Destination: entry.Destination,
-		Outgoing:    entry.Outgoing,
-		CreatedAt:   entry.StartTime,
-		Request:     entry.Request,
-		Response:    entry.Response,
-		Summary:     entry.Summary,
-		Method:      entry.Method,
-		Status:      entry.Status,
-		ElapsedTime: entry.ElapsedTime,
-		Path:        entry.Path,
-	}
-
-	entryJson, err := json.Marshal(entryToPush)
-	if err != nil {
-		logger.Log.Errorf("json.Marshal ERROR: %v", err)
-		return
-	}
-	var buffer bytes.Buffer
-	buffer.WriteString(string(entryJson))
-	res, _ := client.es.Index(client.index, &buffer)
-	if res.StatusCode == 201 {
-		client.insertedCount += 1
-	}
-}

agent/pkg/entries/entries_provider.go

@@ -0,0 +1,103 @@
+package entries
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/agent/pkg/app"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/shared"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+type EntriesProvider interface {
+	GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error)
+	GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error)
+}
+
+type BasenineEntriesProvider struct{}
+
+func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error) {
+	data, _, lastMeta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
+		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
+		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var dataSlice []*tapApi.EntryWrapper
+
+	for _, row := range data {
+		var entry *tapApi.Entry
+		err = json.Unmarshal(row, &entry)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		protocol, ok := app.ProtocolsMap[entry.ProtocolId]
+		if !ok {
+			return nil, nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
+		}
+
+		extension, ok := app.ExtensionsMap[protocol.Name]
+		if !ok {
+			return nil, nil, fmt.Errorf("extension not found, extension: %v", protocol.Name)
+		}
+
+		base := extension.Dissector.Summarize(entry)
+
+		dataSlice = append(dataSlice, &tapApi.EntryWrapper{
+			Protocol: *protocol,
+			Data:     entry,
+			Base:     base,
+		})
+	}
+
+	var metadata *basenine.Metadata
+	err = json.Unmarshal(lastMeta, &metadata)
+	if err != nil {
+		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
+	}
+
+	return dataSlice, metadata, nil
+}
+
+func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error) {
+	var entry *tapApi.Entry
+	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, entryId, singleEntryRequest.Query)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(bytes, &entry)
+	if err != nil {
+		return nil, errors.New(string(bytes))
+	}
+
+	protocol, ok := app.ProtocolsMap[entry.ProtocolId]
+	if !ok {
+		return nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
+	}
+
+	extension, ok := app.ExtensionsMap[protocol.Name]
+	if !ok {
+		return nil, fmt.Errorf("extension not found, extension: %v", protocol.Name)
+	}
+
+	base := extension.Dissector.Summarize(entry)
+	var representation []byte
+	representation, err = extension.Dissector.Represent(entry.Request, entry.Response)
+	if err != nil {
+		return nil, err
+	}
+
+	return &tapApi.EntryWrapper{
+		Protocol:       *protocol,
+		Representation: string(representation),
+		Data:           entry,
+		Base:           base,
+	}, nil
+}

agent/pkg/har/types.go

@@ -2,9 +2,10 @@ package har
 
 import (
 	"encoding/base64"
-	"github.com/up9inc/mizu/shared/logger"
 	"time"
 	"unicode/utf8"
+
+	"github.com/up9inc/mizu/logger"
 )
 
 /*

agent/pkg/har/utils.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 // Keep it because we might want cookies in the future
@@ -124,8 +124,8 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
 
 	postData, _ := request["postData"].(map[string]interface{})
 	mimeType := postData["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
+	if mimeType == nil {
+		mimeType = ""
 	}
 	text := postData["text"]
 	postDataText := ""
@@ -177,8 +177,8 @@ func NewResponse(response map[string]interface{}) (harResponse *Response, err er
 
 	content, _ := response["content"].(map[string]interface{})
 	mimeType := content["mimeType"]
-	if mimeType == nil || len(mimeType.(string)) == 0 {
-		mimeType = "text/html"
+	if mimeType == nil {
+		mimeType = ""
 	}
 	encoding := content["encoding"]
 	text := content["text"]

agent/pkg/middlewares/auth.go

@@ -1,73 +0,0 @@
-package middlewares
-
-import (
-	"github.com/up9inc/mizu/agent/pkg/config"
-	"github.com/up9inc/mizu/agent/pkg/providers"
-
-	"github.com/gin-gonic/gin"
-	ory "github.com/ory/kratos-client-go"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-func RequiresAuth() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		// auth is irrelevant for ephermeral mizu
-		if !config.Config.StandaloneMode {
-			c.Next()
-			return
-		}
-
-		verifyKratosSessionForRequest(c)
-		if !c.IsAborted() {
-			c.Next()
-		}
-	}
-}
-
-func RequiresAdmin() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		// auth is irrelevant for ephermeral mizu
-		if !config.Config.StandaloneMode {
-			c.Next()
-			return
-		}
-
-		session := verifyKratosSessionForRequest(c)
-		if c.IsAborted() {
-			return
-		}
-
-		traits := session.Identity.Traits.(map[string]interface{})
-		username := traits["username"].(string)
-
-		isAdmin, err := providers.CheckIfUserHasSystemRole(username, providers.AdminRole)
-		if err != nil {
-			logger.Log.Errorf("error checking user role %v", err)
-			c.AbortWithStatusJSON(403, gin.H{"error": "unknown auth error occured"})
-		} else if !isAdmin {
-			logger.Log.Warningf("user %s attempted to call an admin only endpoint with insufficient privileges", username)
-			c.AbortWithStatusJSON(403, gin.H{"error": "unauthorized"})
-		} else {
-			c.Next()
-		}
-	}
-}
-
-func verifyKratosSessionForRequest(c *gin.Context) *ory.Session {
-	token := c.GetHeader("x-session-token")
-	if token == "" {
-		c.AbortWithStatusJSON(401, gin.H{"error": "token header is empty"})
-		return nil
-	}
-
-	if session, err := providers.VerifyToken(token, c.Request.Context()); err != nil {
-		logger.Log.Errorf("error verifying token %v", err)
-		c.AbortWithStatusJSON(401, gin.H{"error": "unknown auth error occured"})
-		return nil
-	} else if session == nil {
-		c.AbortWithStatusJSON(401, gin.H{"error": "invalid token"})
-		return nil
-	} else {
-		return session
-	}
-}

agent/pkg/middlewares/cors.go

@@ -7,7 +7,7 @@ func CORSMiddleware() gin.HandlerFunc {
 		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
 		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
 		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With, x-session-token")
-		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")
+		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE")
 
 		if c.Request.Method == "OPTIONS" {
 			c.AbortWithStatus(204)

agent/pkg/models/models.go

@@ -4,24 +4,14 @@ import (
 	"encoding/json"
 
 	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/agent/pkg/rules"
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap"
 )
 
-func GetEntry(r *tapApi.Entry, v tapApi.DataUnmarshaler) error {
-	return v.UnmarshalData(r)
-}
-
-type TapConfig struct {
-	TappedNamespaces map[string]bool `json:"tappedNamespaces"`
-}
-
 type EntriesRequest struct {
-	LeftOff   int    `form:"leftOff" validate:"required,min=-1"`
+	LeftOff   string `form:"leftOff" validate:"required"`
 	Direction int    `form:"direction" validate:"required,oneof='1' '-1'"`
 	Query     string `form:"query"`
 	Limit     int    `form:"limit" validate:"required,min=1"`
@@ -42,21 +32,16 @@ type WebSocketEntryMessage struct {
 	Data *tapApi.BaseEntry `json:"data,omitempty"`
 }
 
+type WebSocketFullEntryMessage struct {
+	*shared.WebSocketMessageMetadata
+	Data *tapApi.Entry `json:"data,omitempty"`
+}
+
 type WebSocketTappedEntryMessage struct {
 	*shared.WebSocketMessageMetadata
 	Data *tapApi.OutputChannelItem
 }
 
-type WebsocketOutboundLinkMessage struct {
-	*shared.WebSocketMessageMetadata
-	Data *tap.OutboundLink
-}
-
-type AuthStatus struct {
-	Email string `json:"email"`
-	Model string `json:"model"`
-}
-
 type ToastMessage struct {
 	Type      string `json:"type"`
 	AutoClose uint   `json:"autoClose"`
@@ -88,20 +73,20 @@ func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) {
 	return json.Marshal(message)
 }
 
-func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte, error) {
-	message := &WebSocketTappedEntryMessage{
+func CreateFullEntryWebSocketMessage(entry *tapApi.Entry) ([]byte, error) {
+	message := &WebSocketFullEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
-			MessageType: shared.WebSocketMessageTypeTappedEntry,
+			MessageType: shared.WebSocketMessageTypeFullEntry,
 		},
-		Data: base,
+		Data: entry,
 	}
 	return json.Marshal(message)
 }
 
-func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error) {
-	message := &WebsocketOutboundLinkMessage{
+func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte, error) {
+	message := &WebSocketTappedEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
-			MessageType: shared.WebsocketMessageTypeOutboundLink,
+			MessageType: shared.WebSocketMessageTypeTappedEntry,
 		},
 		Data: base,
 	}
@@ -157,13 +142,3 @@ type ExtendedCreator struct {
 	*har.Creator
 	Source *string `json:"_source"`
 }
-
-func RunValidationRulesState(harEntry har.Entry, service string) (tapApi.ApplicableRules, []rules.RulesMatched, bool) {
-	resultPolicyToSend, isEnabled := rules.MatchRequestPolicy(harEntry, service)
-	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend)
-	return tapApi.ApplicableRules{Status: statusPolicyToSend, Latency: latency, NumberOfRules: numberOfRules}, resultPolicyToSend, isEnabled
-}
-
-type InstallState struct {
-	Completed bool `json:"completed"`
-}

agent/pkg/oas/counters.go

@@ -17,6 +17,10 @@ type Counter struct {
 }
 
 func (c *Counter) addEntry(ts float64, rt float64, succ bool, dur float64) {
+	if dur < 0 {
+		panic("Duration cannot be negative")
+	}
+
 	c.Entries += 1
 	c.SumRT += rt
 	c.SumDuration += dur

agent/pkg/oas/feeder_test.go

@@ -4,8 +4,10 @@ import (
 	"bufio"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
+	"net/url"
 	"os"
 	"path/filepath"
 	"sort"
@@ -14,27 +16,40 @@ import (
 
 	"github.com/up9inc/mizu/agent/pkg/har"
 
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 func getFiles(baseDir string) (result []string, err error) {
 	result = make([]string, 0)
 	logger.Log.Infof("Reading files from tree: %s", baseDir)
 
+	inputs := []string{baseDir}
+
 	// https://yourbasic.org/golang/list-files-in-directory/
-	err = filepath.Walk(baseDir,
-		func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-
-			ext := strings.ToLower(filepath.Ext(path))
-			if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
-				result = append(result, path)
-			}
+	visitor := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
 
+		if info.Mode()&os.ModeSymlink != 0 {
+			path, _ = os.Readlink(path)
+			inputs = append(inputs, path)
 			return nil
-		})
+		}
+
+		ext := strings.ToLower(filepath.Ext(path))
+		if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
+			result = append(result, path)
+		}
+
+		return nil
+	}
+
+	for len(inputs) > 0 {
+		path := inputs[0]
+		inputs = inputs[1:]
+		err = filepath.Walk(path, visitor)
+	}
 
 	sort.SliceStable(result, func(i, j int) bool {
 		return fileSize(result[i]) < fileSize(result[j])
@@ -53,23 +68,23 @@ func fileSize(fname string) int64 {
 	return fi.Size()
 }
 
-func feedEntries(fromFiles []string, isSync bool) (count int, err error) {
+func feedEntries(fromFiles []string, isSync bool, gen *defaultOasGenerator) (count uint, err error) {
 	badFiles := make([]string, 0)
-	cnt := 0
+	cnt := uint(0)
 	for _, file := range fromFiles {
 		logger.Log.Info("Processing file: " + file)
 		ext := strings.ToLower(filepath.Ext(file))
-		eCnt := 0
+		eCnt := uint(0)
 		switch ext {
 		case ".har":
-			eCnt, err = feedFromHAR(file, isSync)
+			eCnt, err = feedFromHAR(file, isSync, gen)
 			if err != nil {
 				logger.Log.Warning("Failed processing file: " + err.Error())
 				badFiles = append(badFiles, file)
 				continue
 			}
 		case ".ldjson":
-			eCnt, err = feedFromLDJSON(file, isSync)
+			eCnt, err = feedFromLDJSON(file, isSync, gen)
 			if err != nil {
 				logger.Log.Warning("Failed processing file: " + err.Error())
 				badFiles = append(badFiles, file)
@@ -88,7 +103,7 @@ func feedEntries(fromFiles []string, isSync bool) (count int, err error) {
 	return cnt, nil
 }
 
-func feedFromHAR(file string, isSync bool) (int, error) {
+func feedFromHAR(file string, isSync bool, gen *defaultOasGenerator) (uint, error) {
 	fd, err := os.Open(file)
 	if err != nil {
 		panic(err)
@@ -107,16 +122,16 @@ func feedFromHAR(file string, isSync bool) (int, error) {
 		return 0, err
 	}
 
-	cnt := 0
+	cnt := uint(0)
 	for _, entry := range harDoc.Log.Entries {
 		cnt += 1
-		feedEntry(&entry, "", isSync, file)
+		feedEntry(&entry, "", file, gen, fmt.Sprintf("%024d", cnt))
 	}
 
 	return cnt, nil
 }
 
-func feedEntry(entry *har.Entry, source string, isSync bool, file string) {
+func feedEntry(entry *har.Entry, source string, file string, gen *defaultOasGenerator, cnt string) {
 	entry.Comment = file
 	if entry.Response.Status == 302 {
 		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
@@ -126,15 +141,16 @@ func feedEntry(entry *har.Entry, source string, isSync bool, file string) {
 		logger.Log.Debugf("Interesting: %s", entry.Request.URL)
 	}
 
-	ews := EntryWithSource{Entry: *entry, Source: source}
-	if isSync {
-		GetOasGeneratorInstance().entriesChan <- ews // blocking variant, right?
-	} else {
-		GetOasGeneratorInstance().PushEntry(&ews)
+	u, err := url.Parse(entry.Request.URL)
+	if err != nil {
+		logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
 	}
+
+	ews := EntryWithSource{Entry: *entry, Source: source, Destination: u.Host, Id: cnt}
+	gen.handleHARWithSource(&ews)
 }
 
-func feedFromLDJSON(file string, isSync bool) (int, error) {
+func feedFromLDJSON(file string, isSync bool, gen *defaultOasGenerator) (uint, error) {
 	fd, err := os.Open(file)
 	if err != nil {
 		panic(err)
@@ -146,7 +162,7 @@ func feedFromLDJSON(file string, isSync bool) (int, error) {
 
 	var meta map[string]interface{}
 	buf := strings.Builder{}
-	cnt := 0
+	cnt := uint(0)
 	source := ""
 	for {
 		substr, isPrefix, err := reader.ReadLine()
@@ -177,7 +193,7 @@ func feedFromLDJSON(file string, isSync bool) (int, error) {
 				logger.Log.Warningf("Failed decoding entry: %s", line)
 			} else {
 				cnt += 1
-				feedEntry(&entry, source, isSync, file)
+				feedEntry(&entry, source, file, gen, fmt.Sprintf("%024d", cnt))
 			}
 		}
 	}

agent/pkg/oas/ignores.go

@@ -9,7 +9,7 @@ var ignoredCtypes = []string{"application/javascript", "application/x-javascript
 
 var ignoredHeaders = []string{
 	"a-im", "accept",
-	"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-type", "cookie",
+	"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-range", "content-type", "cookie",
 	"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
 	"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
 	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests", "x-download-options",

agent/pkg/oas/oas-samples/carts.json

@@ -0,0 +1,96 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "http://carts",
+    "description": "Mizu observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
+    "version": "1.0"
+  },
+  "servers": [
+    {
+      "url": "http://carts"
+    }
+  ],
+  "paths": {
+    "/carts/{cartId}/items": {
+      "get": {
+        "summary": "/carts/{cartId}/items",
+        "description": "Mizu observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
+        "operationId": "84c9b926-1f73-4ab4-b381-3c124528959f",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": [
+                  {
+                    "id": "60fe98fb86c0fc000869a90c",
+                    "itemId": "3395a43e-2d88-40de-b95f-e00e1502085b",
+                    "quantity": 1,
+                    "unitPrice": 18
+                  }
+                ],
+                "x-sample-entry": "000000000000000000000010"
+              }
+            },
+            "x-sample-entry": "000000000000000000000010"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 3,
+            "failures": 0,
+            "firstSeen": 1627298058.3798368,
+            "lastSeen": 1627298065.2397773,
+            "sumRT": 0.05,
+            "sumDuration": 6.859940528869629
+          }
+        },
+        "x-counters-total": {
+          "entries": 3,
+          "failures": 0,
+          "firstSeen": 1627298058.3798368,
+          "lastSeen": 1627298065.2397773,
+          "sumRT": 0.05,
+          "sumDuration": 6.859940528869629
+        },
+        "x-last-seen-ts": 1627298065.2397773,
+        "x-sample-entry": "000000000000000000000010"
+      },
+      "parameters": [
+        {
+          "name": "cartId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "mHK0P7zTktmV1zv57iWAvCTd43FFMHap"
+            }
+          },
+          "x-sample-entry": "000000000000000000000010"
+        }
+      ]
+    }
+  },
+  "x-counters-per-source": {
+    "some-source": {
+      "entries": 3,
+      "failures": 0,
+      "firstSeen": 1627298058.3798368,
+      "lastSeen": 1627298065.2397773,
+      "sumRT": 0.05,
+      "sumDuration": 6.859940528869629
+    }
+  },
+  "x-counters-total": {
+    "entries": 3,
+    "failures": 0,
+    "firstSeen": 1627298058.3798368,
+    "lastSeen": 1627298065.2397773,
+    "sumRT": 0.05,
+    "sumDuration": 6.859940528869629
+  }
+}

agent/pkg/oas/oas-samples/catalogue.json

@@ -0,0 +1,485 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "Preloaded",
+    "description": "Test file for loading pre-existing OAS",
+    "version": "0.1"
+  },
+  "paths": {
+    "/catalogue": {
+      "get": {
+        "tags": [
+          "catalogue"
+        ],
+        "summary": "/catalogue",
+        "description": "Mizu observed 3 entries (0 failed), at 2.647 hits/s, average response time is 0.008 seconds",
+        "operationId": "dd6c3dbe-6b6b-4ddd-baed-757e237ddb8a",
+        "parameters": [
+          {
+            "name": "page",
+            "in": "query",
+            "required": false,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "1"
+              }
+            },
+            "x-sample-entry": "000000000000000000000002"
+          },
+          {
+            "name": "size",
+            "in": "query",
+            "required": true,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "6"
+              },
+              "example #1": {
+                "value": "3"
+              },
+              "example #2": {
+                "value": "5"
+              }
+            },
+            "x-sample-entry": "000000000000000000000011"
+          },
+          {
+            "name": "tags",
+            "in": "query",
+            "required": false,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": ""
+              },
+              "example #1": {
+                "value": "blue"
+              }
+            },
+            "x-sample-entry": "000000000000000000000007"
+          },
+          {
+            "name": "sort",
+            "in": "query",
+            "required": false,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "id"
+              }
+            },
+            "x-sample-entry": "000000000000000000000007"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": [
+                  {
+                    "count": 1,
+                    "description": "Socks fit for a Messiah. You too can experience walking in water with these special edition beauties. Each hole is lovingly proggled to leave smooth edges. The only sock approved by a higher power.",
+                    "id": "03fef6ac-1896-4ce8-bd69-b798f85c6e0b",
+                    "imageUrl": [
+                      "/catalogue/images/holy_1.jpeg",
+                      "/catalogue/images/holy_2.jpeg"
+                    ],
+                    "name": "Holy",
+                    "price": 99.99,
+                    "tag": [
+                      "action",
+                      "magic"
+                    ]
+                  },
+                  {
+                    "count": 438,
+                    "description": "proident occaecat irure et excepteur labore minim nisi amet irure",
+                    "id": "3395a43e-2d88-40de-b95f-e00e1502085b",
+                    "imageUrl": [
+                      "/catalogue/images/colourful_socks.jpg",
+                      "/catalogue/images/colourful_socks.jpg"
+                    ],
+                    "name": "Colourful",
+                    "price": 18,
+                    "tag": [
+                      "brown",
+                      "blue"
+                    ]
+                  },
+                  {
+                    "count": 820,
+                    "description": "Ready for action. Engineers: be ready to smash that next bug! Be ready, with these super-action-sport-masterpieces. This particular engineer was chased away from the office with a stick.",
+                    "id": "510a0d7e-8e83-4193-b483-e27e09ddc34d",
+                    "imageUrl": [
+                      "/catalogue/images/puma_1.jpeg",
+                      "/catalogue/images/puma_2.jpeg"
+                    ],
+                    "name": "SuperSport XL",
+                    "price": 15,
+                    "tag": [
+                      "sport",
+                      "formal",
+                      "black"
+                    ]
+                  },
+                  {
+                    "count": 738,
+                    "description": "A mature sock, crossed, with an air of nonchalance.",
+                    "id": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38",
+                    "imageUrl": [
+                      "/catalogue/images/cross_1.jpeg",
+                      "/catalogue/images/cross_2.jpeg"
+                    ],
+                    "name": "Crossed",
+                    "price": 17.32,
+                    "tag": [
+                      "blue",
+                      "action",
+                      "red",
+                      "formal"
+                    ]
+                  },
+                  {
+                    "count": 808,
+                    "description": "enim officia aliqua excepteur esse deserunt quis aliquip nostrud anim",
+                    "id": "819e1fbf-8b7e-4f6d-811f-693534916a8b",
+                    "imageUrl": [
+                      "/catalogue/images/WAT.jpg",
+                      "/catalogue/images/WAT2.jpg"
+                    ],
+                    "name": "Figueroa",
+                    "price": 14,
+                    "tag": [
+                      "green",
+                      "formal",
+                      "blue"
+                    ]
+                  },
+                  {
+                    "count": 175,
+                    "description": "consequat amet cupidatat minim laborum tempor elit ex consequat in",
+                    "id": "837ab141-399e-4c1f-9abc-bace40296bac",
+                    "imageUrl": [
+                      "/catalogue/images/catsocks.jpg",
+                      "/catalogue/images/catsocks2.jpg"
+                    ],
+                    "name": "Cat socks",
+                    "price": 15,
+                    "tag": [
+                      "brown",
+                      "formal",
+                      "green"
+                    ]
+                  }
+                ],
+                "x-sample-entry": "000000000000000000000011"
+              }
+            },
+            "x-sample-entry": "000000000000000000000011"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 3,
+            "failures": 0,
+            "firstSeen": 1627298057.7849188,
+            "lastSeen": 1627298065.7258668,
+            "sumRT": 0.024999999999999998,
+            "sumDuration": 7.940948009490967
+          }
+        },
+        "x-counters-total": {
+          "entries": 3,
+          "failures": 0,
+          "firstSeen": 1627298057.7849188,
+          "lastSeen": 1627298065.7258668,
+          "sumRT": 0.024999999999999998,
+          "sumDuration": 7.940948009490967
+        },
+        "x-last-seen-ts": 1627298065.7258668,
+        "x-sample-entry": "000000000000000000000011"
+      }
+    },
+    "/catalogue/size": {
+      "get": {
+        "tags": [
+          "catalogue"
+        ],
+        "summary": "/catalogue/size",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.013 seconds",
+        "operationId": "2315e69d-9d66-48cf-b3d3-fec9c30bd28b",
+        "parameters": [
+          {
+            "name": "tags",
+            "in": "query",
+            "required": true,
+            "style": "form",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": ""
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
+          },
+          {
+            "name": "x-some",
+            "in": "header",
+            "required": true,
+            "style": "simple",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "demo val"
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": {
+                  "err": null,
+                  "size": 9
+                },
+                "x-sample-entry": "000000000000000000000001"
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1627298057.7841518,
+            "lastSeen": 1627298057.7841518,
+            "sumRT": 0.013,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1627298057.7841518,
+          "lastSeen": 1627298057.7841518,
+          "sumRT": 0.013,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1627298057.7841518,
+        "x-sample-entry": "000000000000000000000001"
+      }
+    },
+    "/catalogue/{id}": {
+      "get": {
+        "tags": [
+          "catalogue"
+        ],
+        "summary": "/catalogue/{id}",
+        "description": "Mizu observed 4 entries (0 failed), at 1.899 hits/s, average response time is 0.003 seconds",
+        "parameters": [
+          {
+            "name": "non-required-header",
+            "in": "header",
+            "required": false,
+            "style": "simple",
+            "schema": {
+              "type": "string"
+            },
+            "example": "some-uuid-maybe"
+          },
+          {
+            "name": "x-some",
+            "in": "header",
+            "required": false,
+            "style": "simple",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "example #0": {
+                "value": "demoval"
+              }
+            },
+            "x-sample-entry": "000000000000000000000004"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": {
+                  "count": 438,
+                  "description": "proident occaecat irure et excepteur labore minim nisi amet irure",
+                  "id": "3395a43e-2d88-40de-b95f-e00e1502085b",
+                  "imageUrl": [
+                    "/catalogue/images/colourful_socks.jpg",
+                    "/catalogue/images/colourful_socks.jpg"
+                  ],
+                  "name": "Colourful",
+                  "price": 18,
+                  "tag": [
+                    "brown",
+                    "blue"
+                  ]
+                },
+                "x-sample-entry": "000000000000000000000012"
+              }
+            },
+            "x-sample-entry": "000000000000000000000012"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 4,
+            "failures": 0,
+            "firstSeen": 1627298058.1315014,
+            "lastSeen": 1627298065.7293031,
+            "sumRT": 0.013999999999999999,
+            "sumDuration": 7.597801685333252
+          }
+        },
+        "x-counters-total": {
+          "entries": 4,
+          "failures": 0,
+          "firstSeen": 1627298058.1315014,
+          "lastSeen": 1627298065.7293031,
+          "sumRT": 0.013999999999999999,
+          "sumDuration": 7.597801685333252
+        },
+        "x-last-seen-ts": 1627298065.7293031,
+        "x-sample-entry": "000000000000000000000012"
+      },
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "3395a43e-2d88-40de-b95f-e00e1502085b"
+            },
+            "example #1": {
+              "value": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38"
+            }
+          },
+          "example": "some-uuid-maybe",
+          "x-sample-entry": "000000000000000000000012"
+        }
+      ]
+    },
+    "/catalogue/{id}/details": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ]
+    },
+    "/tags": {
+      "get": {
+        "summary": "/tags",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.007 seconds",
+        "operationId": "c4d7d0ed-1a78-4370-a049-efe3abc631a6",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": {
+                  "err": null,
+                  "tags": [
+                    "brown",
+                    "geek",
+                    "formal",
+                    "blue",
+                    "skin",
+                    "red",
+                    "action",
+                    "sport",
+                    "black",
+                    "magic",
+                    "green"
+                  ]
+                },
+                "x-sample-entry": "000000000000000000000003"
+              }
+            },
+            "x-sample-entry": "000000000000000000000003"
+          }
+        },
+        "x-counters-per-source": {
+          "some-source": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1627298057.7841816,
+            "lastSeen": 1627298057.7841816,
+            "sumRT": 0.007,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1627298057.7841816,
+          "lastSeen": 1627298057.7841816,
+          "sumRT": 0.007,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1627298057.7841816,
+        "x-sample-entry": "000000000000000000000003"
+      }
+    }
+  },
+  "x-counters-per-source": {
+    "some-source": {
+      "entries": 9,
+      "failures": 0,
+      "firstSeen": 1627298057.7841518,
+      "lastSeen": 1627298065.7293031,
+      "sumRT": 0.05899999999999999,
+      "sumDuration": 15.538749694824219
+    }
+  },
+  "x-counters-total": {
+    "entries": 9,
+    "failures": 0,
+    "firstSeen": 1627298057.7841518,
+    "lastSeen": 1627298065.7293031,
+    "sumRT": 0.05899999999999999,
+    "sumDuration": 15.538749694824219
+  }
+}

agent/pkg/oas/oas-samples/httpbin.org.json

@@ -0,0 +1,897 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "https://httpbin.org",
+    "description": "Mizu observed 19 entries (0 failed), at 0.106 hits/s, average response time is 0.172 seconds",
+    "version": "1.0"
+  },
+  "servers": [
+    {
+      "url": "https://httpbin.org"
+    }
+  ],
+  "paths": {
+    "/appears-once": {
+      "get": {
+        "summary": "/appears-once",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "2d34623e-fde8-4720-8390-9a7439051755",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000004"
+              }
+            },
+            "x-sample-entry": "000000000000000000000004"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750580.0471218,
+            "lastSeen": 1567750580.0471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750580.0471218,
+          "lastSeen": 1567750580.0471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750580.0471218,
+        "x-sample-entry": "000000000000000000000004"
+      }
+    },
+    "/appears-twice": {
+      "get": {
+        "summary": "/appears-twice",
+        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds",
+        "operationId": "9c5330f3-8062-468b-b5a3-df1ad82b4846",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000006"
+              }
+            },
+            "x-sample-entry": "000000000000000000000006"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750580.7471218,
+            "lastSeen": 1567750581.7471218,
+            "sumRT": 1.26,
+            "sumDuration": 1
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750580.7471218,
+          "lastSeen": 1567750581.7471218,
+          "sumRT": 1.26,
+          "sumDuration": 1
+        },
+        "x-last-seen-ts": 1567750581.7471218,
+        "x-sample-entry": "000000000000000000000006"
+      }
+    },
+    "/body-optional": {
+      "post": {
+        "summary": "/body-optional",
+        "description": "Mizu observed 3 entries (0 failed), at 0.003 hits/s, average response time is 0.001 seconds",
+        "operationId": "34f3d66c-b1f7-4dca-9cab-987fcc8ae472",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000012"
+              }
+            },
+            "x-sample-entry": "000000000000000000000012"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 3,
+            "failures": 0,
+            "firstSeen": 1567750581.7471218,
+            "lastSeen": 1567750581.757122,
+            "sumRT": 0.003,
+            "sumDuration": 0.010000228881835938
+          }
+        },
+        "x-counters-total": {
+          "entries": 3,
+          "failures": 0,
+          "firstSeen": 1567750581.7471218,
+          "lastSeen": 1567750581.757122,
+          "sumRT": 0.003,
+          "sumDuration": 0.010000228881835938
+        },
+        "x-last-seen-ts": 1567750581.757122,
+        "x-sample-entry": "000000000000000000000012",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "application/json": {
+              "example": "{\"key\", \"val\"}",
+              "x-sample-entry": "000000000000000000000011"
+            }
+          },
+          "x-sample-entry": "000000000000000000000012"
+        }
+      }
+    },
+    "/body-required": {
+      "post": {
+        "summary": "/body-required",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "ff6add53-ab1c-4d4e-b590-0835fa318276",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000013"
+              }
+            },
+            "x-sample-entry": "000000000000000000000013"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750581.757122,
+            "lastSeen": 1567750581.757122,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750581.757122,
+          "lastSeen": 1567750581.757122,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750581.757122,
+        "x-sample-entry": "000000000000000000000013",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "": {
+              "example": "body exists",
+              "x-sample-entry": "000000000000000000000013"
+            }
+          },
+          "required": true,
+          "x-sample-entry": "000000000000000000000013"
+        }
+      }
+    },
+    "/form-multipart": {
+      "post": {
+        "summary": "/form-multipart",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "153f0925-9fc7-4e9f-9d33-f1470f25f0f7",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "example": {},
+                "x-sample-entry": "000000000000000000000009"
+              }
+            },
+            "x-sample-entry": "000000000000000000000009"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.7471218,
+            "lastSeen": 1567750582.7471218,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.7471218,
+          "lastSeen": 1567750582.7471218,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.7471218,
+        "x-sample-entry": "000000000000000000000009",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "multipart/form-data": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "file",
+                  "path"
+                ],
+                "properties": {
+                  "file": {
+                    "type": "string",
+                    "contentMediaType": "application/json",
+                    "examples": [
+                      "{\"functions\": 123}"
+                    ]
+                  },
+                  "path": {
+                    "type": "string",
+                    "examples": [
+                      "/content/components"
+                    ]
+                  }
+                }
+              },
+              "example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n",
+              "x-sample-entry": "000000000000000000000009"
+            }
+          },
+          "required": true,
+          "x-sample-entry": "000000000000000000000009"
+        }
+      }
+    },
+    "/form-urlencoded": {
+      "post": {
+        "summary": "/form-urlencoded",
+        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds",
+        "operationId": "c92189f5-5636-46eb-ac71-92b17941a568",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000008"
+              }
+            },
+            "x-sample-entry": "000000000000000000000008"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750580.7471218,
+            "lastSeen": 1567750581.7471218,
+            "sumRT": 0.002,
+            "sumDuration": 1
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750580.7471218,
+          "lastSeen": 1567750581.7471218,
+          "sumRT": 0.002,
+          "sumDuration": 1
+        },
+        "x-last-seen-ts": 1567750581.7471218,
+        "x-sample-entry": "000000000000000000000008",
+        "requestBody": {
+          "description": "Generic request body",
+          "content": {
+            "application/x-www-form-urlencoded": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "agent-id",
+                  "callback-url",
+                  "token"
+                ],
+                "properties": {
+                  "agent-id": {
+                    "type": "string",
+                    "examples": [
+                      "ade"
+                    ]
+                  },
+                  "callback-url": {
+                    "type": "string",
+                    "examples": [
+                      ""
+                    ]
+                  },
+                  "optional": {
+                    "type": "string",
+                    "examples": [
+                      "another"
+                    ]
+                  },
+                  "token": {
+                    "type": "string",
+                    "examples": [
+                      "sometoken",
+                      "sometoken-second-val"
+                    ]
+                  }
+                }
+              },
+              "example": "agent-id=ade\u0026callback-url=\u0026token=sometoken",
+              "x-sample-entry": "000000000000000000000008"
+            }
+          },
+          "required": true,
+          "x-sample-entry": "000000000000000000000008"
+        }
+      }
+    },
+    "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "85270437-7aae-4a5b-b988-3662092463d0",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000014"
+              }
+            },
+            "x-sample-entry": "000000000000000000000014"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582,
+            "lastSeen": 1567750582,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582,
+          "lastSeen": 1567750582,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582,
+        "x-sample-entry": "000000000000000000000014"
+      },
+      "parameters": [
+        {
+          "name": "prefixgibberishfineId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "234324"
+            }
+          },
+          "x-sample-entry": "000000000000000000000014"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}",
+        "description": "Mizu observed 2 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "da597734-1cf5-4d3b-917b-6b02dacf7b7b",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000018"
+              }
+            },
+            "x-sample-entry": "000000000000000000000018"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750582.000003,
+            "lastSeen": 1567750582.000004,
+            "sumRT": 0.002,
+            "sumDuration": 9.5367431640625e-7
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750582.000003,
+          "lastSeen": 1567750582.000004,
+          "sumRT": 0.002,
+          "sumDuration": 9.5367431640625e-7
+        },
+        "x-last-seen-ts": 1567750582.000004,
+        "x-sample-entry": "000000000000000000000018"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/1": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/1",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "e965a245-9cfc-48ed-94e1-f765eadb3960",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000015"
+              }
+            },
+            "x-sample-entry": "000000000000000000000015"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.000001,
+            "lastSeen": 1567750582.000001,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.000001,
+          "lastSeen": 1567750582.000001,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.000001,
+        "x-sample-entry": "000000000000000000000015"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/static": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/static",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "7af420dc-f8b7-450f-8f6f-18b039aa3cde",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000016"
+              }
+            },
+            "x-sample-entry": "000000000000000000000016"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.000002,
+            "lastSeen": 1567750582.000002,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.000002,
+          "lastSeen": 1567750582.000002,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.000002,
+        "x-sample-entry": "000000000000000000000016"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/{param1}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/{param1}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
+        "operationId": "02a1771d-2d50-4a8c-8be2-29c7e59b8435",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000019"
+              }
+            },
+            "x-sample-entry": "000000000000000000000019"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.000002,
+            "lastSeen": 1567750582.000002,
+            "sumRT": 0.001,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.000002,
+          "lastSeen": 1567750582.000002,
+          "sumRT": 0.001,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.000002,
+        "x-sample-entry": "000000000000000000000019"
+      },
+      "parameters": [
+        {
+          "name": "param1",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "23421"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        },
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/{Id}": {
+      "get": {
+        "summary": "/{Id}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "77ec4910-d47a-46a5-8234-fb80a11034b4",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000003"
+              }
+            },
+            "x-sample-entry": "000000000000000000000003"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750579.7471218,
+            "lastSeen": 1567750579.7471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750579.7471218,
+          "lastSeen": 1567750579.7471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750579.7471218,
+        "x-sample-entry": "000000000000000000000003"
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          },
+          "x-sample-entry": "000000000000000000000003"
+        }
+      ]
+    },
+    "/{Id}/sub1": {
+      "get": {
+        "summary": "/{Id}/sub1",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds",
+        "operationId": "198675eb-9faf-407b-83fa-0483a730bbbe",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "text/html": {
+                "x-sample-entry": "000000000000000000000001"
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750483.864529,
+            "lastSeen": 1567750483.864529,
+            "sumRT": 0.111,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750483.864529,
+          "lastSeen": 1567750483.864529,
+          "sumRT": 0.111,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750483.864529,
+        "x-sample-entry": "000000000000000000000001"
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          },
+          "x-sample-entry": "000000000000000000000003"
+        }
+      ]
+    },
+    "/{Id}/sub2": {
+      "get": {
+        "summary": "/{Id}/sub2",
+        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
+        "operationId": "31d880f1-152f-4dd6-84a7-463e13b694a5",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000002"
+              }
+            },
+            "x-sample-entry": "000000000000000000000002"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750578.7471218,
+            "lastSeen": 1567750578.7471218,
+            "sumRT": 0.63,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750578.7471218,
+          "lastSeen": 1567750578.7471218,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750578.7471218,
+        "x-sample-entry": "000000000000000000000002"
+      },
+      "parameters": [
+        {
+          "name": "Id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+            },
+            "example #1": {
+              "value": "952bea17-3776-11ea-9341-42010a84012a"
+            }
+          },
+          "x-sample-entry": "000000000000000000000003"
+        }
+      ]
+    }
+  },
+  "x-counters-per-source": {
+    "": {
+      "entries": 19,
+      "failures": 0,
+      "firstSeen": 1567750483.864529,
+      "lastSeen": 1567750582.7471218,
+      "sumRT": 3.273999999999999,
+      "sumDuration": 2.0100011825561523
+    }
+  },
+  "x-counters-total": {
+    "entries": 19,
+    "failures": 0,
+    "firstSeen": 1567750483.864529,
+    "lastSeen": 1567750582.7471218,
+    "sumRT": 3.273999999999999,
+    "sumDuration": 2.0100011825561523
+  }
+}

agent/pkg/oas/oas_generator.go

@@ -1,118 +1,142 @@
 package oas
 
 import (
-	"context"
 	"encoding/json"
 	"net/url"
 	"sync"
 
 	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
+	"github.com/up9inc/mizu/tap/api"
 )
 
 var (
 	syncOnce sync.Once
-	instance *oasGenerator
+	instance *defaultOasGenerator
 )
 
-func GetOasGeneratorInstance() *oasGenerator {
+type OasGeneratorSink interface {
+	HandleEntry(mizuEntry *api.Entry, protocol *api.Protocol)
+}
+
+type OasGenerator interface {
+	Start()
+	Stop()
+	IsStarted() bool
+	GetServiceSpecs() *sync.Map
+}
+
+type defaultOasGenerator struct {
+	started       bool
+	serviceSpecs  *sync.Map
+	maxExampleLen int
+}
+
+func GetDefaultOasGeneratorInstance(maxExampleLen int) *defaultOasGenerator {
 	syncOnce.Do(func() {
-		instance = newOasGenerator()
+		instance = NewDefaultOasGenerator(maxExampleLen)
 		logger.Log.Debug("OAS Generator Initialized")
 	})
 	return instance
 }
 
-func (g *oasGenerator) Start() {
-	if g.started {
-		return
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	g.cancel = cancel
-	g.ctx = ctx
-	g.entriesChan = make(chan EntryWithSource, 100) // buffer up to 100 entries for OAS processing
-	g.ServiceSpecs = &sync.Map{}
+func (g *defaultOasGenerator) Start() {
 	g.started = true
-	go instance.runGeneretor()
 }
 
-func (g *oasGenerator) runGeneretor() {
-	for {
-		select {
-		case <-g.ctx.Done():
-			logger.Log.Infof("OAS Generator was canceled")
-			return
-
-		case entryWithSource, ok := <-g.entriesChan:
-			if !ok {
-				logger.Log.Infof("OAS Generator - entries channel closed")
-				break
-			}
-			entry := entryWithSource.Entry
-			u, err := url.Parse(entry.Request.URL)
-			if err != nil {
-				logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
-			}
-
-			val, found := g.ServiceSpecs.Load(u.Host)
-			var gen *SpecGen
-			if !found {
-				gen = NewGen(u.Scheme + "://" + u.Host)
-				g.ServiceSpecs.Store(u.Host, gen)
-			} else {
-				gen = val.(*SpecGen)
-			}
-
-			opId, err := gen.feedEntry(entryWithSource)
-			if err != nil {
-				txt, suberr := json.Marshal(entry)
-				if suberr == nil {
-					logger.Log.Debugf("Problematic entry: %s", txt)
-				}
-
-				logger.Log.Warningf("Failed processing entry: %s", err)
-				continue
-			}
-
-			logger.Log.Debugf("Handled entry %s as opId: %s", entry.Request.URL, opId) // TODO: set opId back to entry?
-		}
-	}
-}
-
-func (g *oasGenerator) Reset() {
-	g.ServiceSpecs = &sync.Map{}
-}
-
-func (g *oasGenerator) PushEntry(entryWithSource *EntryWithSource) {
+func (g *defaultOasGenerator) Stop() {
 	if !g.started {
 		return
 	}
-	select {
-	case g.entriesChan <- *entryWithSource:
-	default:
-		logger.Log.Warningf("OAS Generator - entry wasn't sent to channel because the channel has no buffer or there is no receiver")
+
+	g.started = false
+
+	g.reset()
+}
+
+func (g *defaultOasGenerator) IsStarted() bool {
+	return g.started
+}
+
+func (g *defaultOasGenerator) HandleEntry(mizuEntry *api.Entry, protocol *api.Protocol) {
+	if !g.started {
+		return
+	}
+
+	if protocol.Name == "http" {
+		dest := mizuEntry.Destination.Name
+		if dest == "" {
+			logger.Log.Debugf("OAS: Unresolved entry %d", mizuEntry.Id)
+			return
+		}
+
+		entry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
+		if err != nil {
+			logger.Log.Warningf("Failed to turn MizuEntry %d into HAR Entry: %s", mizuEntry.Id, err)
+			return
+		}
+
+		entryWSource := &EntryWithSource{
+			Entry:       *entry,
+			Source:      mizuEntry.Source.Name,
+			Destination: dest,
+			Id:          mizuEntry.Id,
+		}
+
+		g.handleHARWithSource(entryWSource)
+	} else {
+		logger.Log.Debugf("OAS: Unsupported protocol in entry %d: %s", mizuEntry.Id, protocol.Name)
 	}
 }
 
-func newOasGenerator() *oasGenerator {
-	return &oasGenerator{
-		started:      false,
-		ctx:          nil,
-		cancel:       nil,
-		ServiceSpecs: nil,
-		entriesChan:  nil,
+func (g *defaultOasGenerator) handleHARWithSource(entryWSource *EntryWithSource) {
+	entry := entryWSource.Entry
+	gen := g.getGen(entryWSource.Destination, entry.Request.URL)
+
+	opId, err := gen.feedEntry(entryWSource)
+	if err != nil {
+		txt, suberr := json.Marshal(entry)
+		if suberr == nil {
+			logger.Log.Debugf("Problematic entry: %s", txt)
+		}
+
+		logger.Log.Warningf("Failed processing entry %d: %s", entryWSource.Id, err)
+		return
+	}
+
+	logger.Log.Debugf("Handled entry %s as opId: %s", entryWSource.Id, opId) // TODO: set opId back to entry?
+}
+
+func (g *defaultOasGenerator) getGen(dest string, urlStr string) *SpecGen {
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", urlStr, err)
+	}
+
+	val, found := g.serviceSpecs.Load(dest)
+	var gen *SpecGen
+	if !found {
+		gen = NewGen(u.Scheme + "://" + dest)
+		gen.MaxExampleLen = g.maxExampleLen
+		g.serviceSpecs.Store(dest, gen)
+	} else {
+		gen = val.(*SpecGen)
+	}
+	return gen
+}
+
+func (g *defaultOasGenerator) reset() {
+	g.serviceSpecs = &sync.Map{}
+}
+
+func (g *defaultOasGenerator) GetServiceSpecs() *sync.Map {
+	return g.serviceSpecs
+}
+
+func NewDefaultOasGenerator(maxExampleLen int) *defaultOasGenerator {
+	return &defaultOasGenerator{
+		started:       false,
+		serviceSpecs:  &sync.Map{},
+		maxExampleLen: maxExampleLen,
 	}
 }
-
-type EntryWithSource struct {
-	Source string
-	Entry  har.Entry
-}
-
-type oasGenerator struct {
-	started      bool
-	ctx          context.Context
-	cancel       context.CancelFunc
-	ServiceSpecs *sync.Map
-	entriesChan  chan EntryWithSource
-}

agent/pkg/oas/oas_generator_test.go

@@ -0,0 +1,45 @@
+package oas
+
+import (
+	"encoding/json"
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"testing"
+	"time"
+)
+
+func TestOASGen(t *testing.T) {
+	gen := GetDefaultOasGeneratorInstance(-1)
+
+	e := new(har.Entry)
+	err := json.Unmarshal([]byte(`{"startedDateTime": "20000101","request": {"url": "https://host/path", "method": "GET"}, "response": {"status": 200}}`), e)
+	if err != nil {
+		panic(err)
+	}
+
+	ews := &EntryWithSource{
+		Destination: "some",
+		Entry:       *e,
+	}
+
+	gen.Start()
+	gen.handleHARWithSource(ews)
+	g, ok := gen.serviceSpecs.Load("some")
+	if !ok {
+		panic("Failed")
+	}
+	sg := g.(*SpecGen)
+	spec, err := sg.GetSpec()
+	if err != nil {
+		panic(err)
+	}
+	specText, _ := json.Marshal(spec)
+	t.Log(string(specText))
+
+	if !gen.IsStarted() {
+		t.Errorf("Should be started")
+	}
+
+	time.Sleep(100 * time.Millisecond)
+
+	gen.Stop()
+}

agent/pkg/oas/specgen.go

@@ -3,10 +3,6 @@ package oas
 import (
 	"encoding/json"
 	"errors"
-	"github.com/chanced/openapi"
-	"github.com/google/uuid"
-	"github.com/nav-inc/datetime"
-	"github.com/up9inc/mizu/shared/logger"
 	"io"
 	"io/ioutil"
 	"mime"
@@ -18,6 +14,11 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/chanced/openapi"
+	"github.com/google/uuid"
+	"github.com/nav-inc/datetime"
+	"github.com/up9inc/mizu/logger"
+
 	"github.com/up9inc/mizu/agent/pkg/har"
 
 	"time"
@@ -26,6 +27,14 @@ import (
 const LastSeenTS = "x-last-seen-ts"
 const CountersTotal = "x-counters-total"
 const CountersPerSource = "x-counters-per-source"
+const SampleId = "x-sample-entry"
+
+type EntryWithSource struct {
+	Source      string
+	Destination string
+	Entry       har.Entry
+	Id          string
+}
 
 type reqResp struct { // hello, generics in Go
 	Req  *har.Request
@@ -33,6 +42,8 @@ type reqResp struct { // hello, generics in Go
 }
 
 type SpecGen struct {
+	MaxExampleLen int // -1 unlimited, 0 and above sets limit
+
 	oas  *openapi.OpenAPI
 	tree *Node
 	lock sync.Mutex
@@ -50,7 +61,11 @@ func NewGen(server string) *SpecGen {
 	spec.Servers = make([]*openapi.Server, 0)
 	spec.Servers = append(spec.Servers, &openapi.Server{URL: server})
 
-	gen := SpecGen{oas: spec, tree: new(Node)}
+	gen := SpecGen{
+		oas:           spec,
+		tree:          new(Node),
+		MaxExampleLen: -1,
+	}
 	return &gen
 }
 
@@ -59,7 +74,7 @@ func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
 	g.tree = new(Node)
 	for pathStr, pathObj := range oas.Paths.Items {
 		pathSplit := strings.Split(string(pathStr), "/")
-		g.tree.getOrSet(pathSplit, pathObj)
+		g.tree.getOrSet(pathSplit, pathObj, "")
 
 		// clean "last entry timestamp" markers from the past
 		for _, pathAndOp := range g.tree.listOps() {
@@ -68,11 +83,11 @@ func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
 	}
 }
 
-func (g *SpecGen) feedEntry(entryWithSource EntryWithSource) (string, error) {
+func (g *SpecGen) feedEntry(entryWithSource *EntryWithSource) (string, error) {
 	g.lock.Lock()
 	defer g.lock.Unlock()
 
-	opId, err := g.handlePathObj(&entryWithSource)
+	opId, err := g.handlePathObj(entryWithSource)
 	if err != nil {
 		return "", err
 	}
@@ -187,7 +202,7 @@ func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error
 	}
 
 	if entry.Request.Method == "OPTIONS" {
-		logger.Log.Debugf("Dropped traffic entry due to its method: %s", urlParsed.Path)
+		logger.Log.Debugf("Dropped traffic entry due to its method: %s %s", entry.Request.Method, urlParsed.Path)
 		return "", nil
 	}
 
@@ -218,8 +233,8 @@ func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error
 	} else {
 		split = strings.Split(urlParsed.Path, "/")
 	}
-	node := g.tree.getOrSet(split, new(openapi.PathObj))
-	opObj, err := handleOpObj(entryWithSource, node.pathObj)
+	node := g.tree.getOrSet(split, new(openapi.PathObj), entryWithSource.Id)
+	opObj, err := handleOpObj(entryWithSource, node.pathObj, g.MaxExampleLen)
 
 	if opObj != nil {
 		return opObj.OperationID, err
@@ -228,7 +243,7 @@ func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error
 	return "", err
 }
 
-func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj) (*openapi.Operation, error) {
+func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj, limit int) (*openapi.Operation, error) {
 	entry := entryWithSource.Entry
 	isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
 	opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
@@ -241,12 +256,12 @@ func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj) (*o
 		return nil, nil
 	}
 
-	err = handleRequest(&entry.Request, opObj, isSuccess)
+	err = handleRequest(&entry.Request, opObj, isSuccess, entryWithSource.Id, limit)
 	if err != nil {
 		return nil, err
 	}
 
-	err = handleResponse(&entry.Response, opObj, isSuccess)
+	err = handleResponse(&entry.Response, opObj, isSuccess, entryWithSource.Id, limit)
 	if err != nil {
 		return nil, err
 	}
@@ -256,6 +271,8 @@ func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj) (*o
 		return nil, err
 	}
 
+	setSampleID(&opObj.Extensions, entryWithSource.Id)
+
 	return opObj, nil
 }
 
@@ -306,7 +323,7 @@ func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *Ent
 	rt := float64(entryWithSource.Entry.Time) / 1000
 
 	dur := 0.0
-	if prevTs != 0 {
+	if prevTs != 0 && ts >= prevTs {
 		dur = ts - prevTs
 	}
 
@@ -331,16 +348,31 @@ func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *Ent
 	return nil
 }
 
-func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error {
+func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, sampleId string, limit int) error {
 	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
+	urlParsed, err := url.Parse(req.URL)
+	if err != nil {
+		return err
+	}
+
+	qs := make([]har.NVP, 0)
+	for name, vals := range urlParsed.Query() {
+		for _, val := range vals {
+			qs = append(qs, har.NVP{Name: name, Value: val})
+		}
+	}
+
+	if len(qs) != len(req.QueryString) {
+		logger.Log.Warningf("QStr params in HAR do not match URL: %s", req.URL)
+	}
 
 	qstrGW := nvParams{
 		In:             openapi.InQuery,
-		Pairs:          req.QueryString,
+		Pairs:          qs,
 		IsIgnored:      func(name string) bool { return false },
 		GeneralizeName: func(name string) string { return name },
 	}
-	handleNameVals(qstrGW, &opObj.Parameters)
+	handleNameVals(qstrGW, &opObj.Parameters, false, sampleId)
 
 	hdrGW := nvParams{
 		In:             openapi.InHeader,
@@ -348,7 +380,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e
 		IsIgnored:      isHeaderIgnored,
 		GeneralizeName: strings.ToLower,
 	}
-	handleNameVals(hdrGW, &opObj.Parameters)
+	handleNameVals(hdrGW, &opObj.Parameters, true, sampleId)
 
 	if isSuccess {
 		reqBody, err := getRequestBody(req, opObj)
@@ -357,12 +389,14 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e
 		}
 
 		if reqBody != nil {
+			setSampleID(&reqBody.Extensions, sampleId)
+
 			if req.PostData.Text == "" {
 				reqBody.Required = false
 			} else {
 
 				reqCtype, _ := getReqCtype(req)
-				reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype)
+				reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, sampleId, limit)
 				if err != nil {
 					return err
 				}
@@ -374,18 +408,20 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e
 	return nil
 }
 
-func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool) error {
+func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool, sampleId string, limit int) error {
 	// TODO: we don't support "default" response
 	respObj, err := getResponseObj(resp, opObj, isSuccess)
 	if err != nil {
 		return err
 	}
 
-	handleRespHeaders(resp.Headers, respObj)
+	setSampleID(&respObj.Extensions, sampleId)
+
+	handleRespHeaders(resp.Headers, respObj, sampleId)
 
 	respCtype := getRespCtype(resp)
 	respContent := respObj.Content
-	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype)
+	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, sampleId, limit)
 	if err != nil {
 		return err
 	}
@@ -393,7 +429,7 @@ func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool
 	return nil
 }
 
-func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj) {
+func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sampleId string) {
 	visited := map[string]*openapi.HeaderObj{}
 	for _, pair := range reqHeaders {
 		if isHeaderIgnored(pair.Name) {
@@ -415,6 +451,8 @@ func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj) {
 			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
 		}
 		visited[nameGeneral] = param
+
+		setSampleID(&param.Extensions, sampleId)
 	}
 
 	// maintain "required" flag
@@ -435,13 +473,15 @@ func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj) {
 	}
 }
 
-func fillContent(reqResp reqResp, respContent openapi.Content, ctype string) (*openapi.MediaType, error) {
+func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, sampleId string, limit int) (*openapi.MediaType, error) {
 	content, found := respContent[ctype]
 	if !found {
 		respContent[ctype] = &openapi.MediaType{}
 		content = respContent[ctype]
 	}
 
+	setSampleID(&content.Extensions, sampleId)
+
 	var text string
 	var isBinary bool
 	if reqResp.Req != nil {
@@ -453,10 +493,10 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string) (*o
 	if !isBinary && text != "" {
 		var exampleMsg []byte
 		// try treating it as json
-		any, isJSON := anyJSON(text)
+		anyVal, isJSON := anyJSON(text)
 		if isJSON {
 			// re-marshal with forced indent
-			if msg, err := json.MarshalIndent(any, "", "\t"); err != nil {
+			if msg, err := json.MarshalIndent(anyVal, "", "\t"); err != nil {
 				panic("Failed to re-marshal value, super-strange")
 			} else {
 				exampleMsg = msg
@@ -476,7 +516,7 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string) (*o
 			handleFormDataMultipart(text, content, params)
 		}
 
-		if content.Example == nil && len(exampleMsg) > len(content.Example) {
+		if len(exampleMsg) > len(content.Example) && (limit < 0 || len(exampleMsg) <= limit) {
 			content.Example = exampleMsg
 		}
 	}

agent/pkg/oas/specgen_test.go

@@ -2,23 +2,23 @@ package oas
 
 import (
 	"encoding/json"
-	"github.com/chanced/openapi"
-	"github.com/op/go-logging"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/wI2L/jsondiff"
 	"io/ioutil"
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
+	"github.com/chanced/openapi"
 	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/logger"
+	"github.com/wI2L/jsondiff"
 )
 
 // if started via env, write file into subdir
 func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string {
-	content, err := json.MarshalIndent(spec, "", "\t")
+	content, err := json.MarshalIndent(spec, "", "  ")
 	if err != nil {
 		panic(err)
 	}
@@ -41,25 +41,42 @@ func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string {
 }
 
 func TestEntries(t *testing.T) {
-	logger.InitLoggerStderrOnly(logging.INFO)
+	//logger.InitLoggerStd(logging.INFO) causes race condition
 	files, err := getFiles("./test_artifacts/")
 	if err != nil {
 		t.Log(err)
 		t.FailNow()
 	}
-	GetOasGeneratorInstance().Start()
-	loadStartingOAS()
 
-	cnt, err := feedEntries(files, true)
+	gen := NewDefaultOasGenerator(-1)
+	gen.serviceSpecs = new(sync.Map)
+	loadStartingOAS("test_artifacts/catalogue.json", "catalogue", gen.serviceSpecs)
+	loadStartingOAS("test_artifacts/trcc.json", "trcc-api-service", gen.serviceSpecs)
+
+	go func() {
+		for {
+			time.Sleep(1 * time.Second)
+			gen.serviceSpecs.Range(func(key, val interface{}) bool {
+				svc := key.(string)
+				t.Logf("Getting spec for %s", svc)
+				gen := val.(*SpecGen)
+				_, err := gen.GetSpec()
+				if err != nil {
+					t.Error(err)
+				}
+				return true
+			})
+		}
+	}()
+
+	cnt, err := feedEntries(files, true, gen)
 	if err != nil {
 		t.Log(err)
 		t.Fail()
 	}
 
-	waitQueueProcessed()
-
 	svcs := strings.Builder{}
-	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+	gen.serviceSpecs.Range(func(key, val interface{}) bool {
 		gen := val.(*SpecGen)
 		svc := key.(string)
 		svcs.WriteString(svc + ",")
@@ -81,7 +98,7 @@ func TestEntries(t *testing.T) {
 		return true
 	})
 
-	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+	gen.serviceSpecs.Range(func(key, val interface{}) bool {
 		svc := key.(string)
 		gen := val.(*SpecGen)
 		spec, err := gen.GetSpec()
@@ -105,20 +122,18 @@ func TestEntries(t *testing.T) {
 }
 
 func TestFileSingle(t *testing.T) {
-	GetOasGeneratorInstance().Start()
-	GetOasGeneratorInstance().Reset()
+	gen := NewDefaultOasGenerator(-1)
+	gen.serviceSpecs = new(sync.Map)
 	// loadStartingOAS()
 	file := "test_artifacts/params.har"
 	files := []string{file}
-	cnt, err := feedEntries(files, true)
+	cnt, err := feedEntries(files, true, gen)
 	if err != nil {
 		logger.Log.Warning("Failed processing file: " + err.Error())
 		t.Fail()
 	}
 
-	waitQueueProcessed()
-
-	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
+	gen.serviceSpecs.Range(func(key, val interface{}) bool {
 		svc := key.(string)
 		gen := val.(*SpecGen)
 		spec, err := gen.GetSpec()
@@ -154,6 +169,13 @@ func TestFileSingle(t *testing.T) {
 			t.FailNow()
 		}
 
+		if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
+			err = ioutil.WriteFile(file+".spec.json", []byte(specText), 0644)
+			if err != nil {
+				panic(err)
+			}
+		}
+
 		if len(diff) > 0 {
 			t.Errorf("Generated spec does not match expected:\n%s", diff.String())
 		}
@@ -164,19 +186,7 @@ func TestFileSingle(t *testing.T) {
 	logger.Log.Infof("Processed entries: %d", cnt)
 }
 
-func waitQueueProcessed() {
-	for {
-		time.Sleep(100 * time.Millisecond)
-		queue := len(GetOasGeneratorInstance().entriesChan)
-		logger.Log.Infof("Queue: %d", queue)
-		if queue < 1 {
-			break
-		}
-	}
-}
-
-func loadStartingOAS() {
-	file := "test_artifacts/catalogue.json"
+func loadStartingOAS(file string, label string, specs *sync.Map) {
 	fd, err := os.Open(file)
 	if err != nil {
 		panic(err)
@@ -195,15 +205,17 @@ func loadStartingOAS() {
 		panic(err)
 	}
 
-	gen := NewGen("catalogue")
+	gen := NewGen(label)
 	gen.StartFromSpec(doc)
 
-	GetOasGeneratorInstance().ServiceSpecs.Store("catalogue", gen)
+	specs.Store(label, gen)
 }
 
 func TestEntriesNegative(t *testing.T) {
+	gen := NewDefaultOasGenerator(-1)
+	gen.serviceSpecs = new(sync.Map)
 	files := []string{"invalid"}
-	_, err := feedEntries(files, false)
+	_, err := feedEntries(files, false, gen)
 	if err == nil {
 		t.Logf("Should have failed")
 		t.Fail()
@@ -211,8 +223,10 @@ func TestEntriesNegative(t *testing.T) {
 }
 
 func TestEntriesPositive(t *testing.T) {
+	gen := NewDefaultOasGenerator(-1)
+	gen.serviceSpecs = new(sync.Map)
 	files := []string{"test_artifacts/params.har"}
-	_, err := feedEntries(files, false)
+	_, err := feedEntries(files, false, gen)
 	if err != nil {
 		t.Logf("Failed")
 		t.Fail()

agent/pkg/oas/test_artifacts/params.har

@@ -538,6 +538,252 @@
           "wait": -1,
           "receive": 1
         }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:22.000000+00:00",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/param-patterns/prefix-gibberish-fine/234324",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:22.000001+00:00",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/param-patterns/prefix-gibberish-sfdlasdfkadf87sd93284q24r/1",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+     {
+        "startedDateTime": "2019-09-06T06:16:22.000002+00:00",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/param-patterns/prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf/static",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+     {
+        "startedDateTime": "2019-09-06T06:16:22.000003+00:00",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/param-patterns/prefix-gibberish-4jk5l2345h2452l4352435jlk45",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+     {
+        "startedDateTime": "2019-09-06T06:16:22.000004+00:00",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/param-patterns/prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:22.000002+00:00",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/param-patterns/prefix-gibberish-afterwards/23421",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "queryString": [],
+          "headersSize": -1,
+          "bodySize": -1,
+          "postData": {
+            "mimeType": "",
+            "text": ""
+          }
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "",
+          "cookies": [],
+          "headers": [
+          ],
+          "content": {
+            "size": 0,
+            "mimeType": "",
+            "text": ""
+          },
+          "redirectURL": "",
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": 1
+        }
       }
     ]
   }

agent/pkg/oas/test_artifacts/params.har.spec.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "https://httpbin.org",
-    "description": "Mizu observed 13 entries (0 failed), at 0.154 hits/s, average response time is 0.251 seconds",
+    "description": "Mizu observed 19 entries (0 failed), at 0.10 hits/s, average response time is 0.17 seconds",
     "version": "1.0"
   },
   "servers": [
@@ -14,194 +14,215 @@
     "/appears-once": {
       "get": {
         "summary": "/appears-once",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
-        "operationId": "8dfc5a75-5f9b-4b57-ad88-77154a016201",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
+        "operationId": "<UUID4>",
         "responses": {
           "200": {
             "description": "Successful call with status 200",
             "content": {
               "application/json": {
-                "example": null
+                "example": null,
+                "x-sample-entry": "000000000000000000000004"
               }
-            }
+            },
+            "x-sample-entry": "000000000000000000000004"
           }
         },
         "x-counters-per-source": {
           "": {
             "entries": 1,
             "failures": 0,
-            "firstSeen": 1567750580.0471218,
-            "lastSeen": 1567750580.0471218,
+            "firstSeen": 1567750580.04,
+            "lastSeen": 1567750580.04,
             "sumRT": 0.63,
             "sumDuration": 0
           }
         },
-        "x-last-seen-ts": 1567750580.0471218,
         "x-counters-total": {
           "entries": 1,
           "failures": 0,
-          "firstSeen": 1567750580.0471218,
-          "lastSeen": 1567750580.0471218,
+          "firstSeen": 1567750580.04,
+          "lastSeen": 1567750580.04,
           "sumRT": 0.63,
           "sumDuration": 0
-        }
+        },
+        "x-last-seen-ts": 1567750580.04,
+        "x-sample-entry": "000000000000000000000004"
       }
     },
     "/appears-twice": {
       "get": {
         "summary": "/appears-twice",
-        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds",
-        "operationId": "844ae464-fcee-4bd7-97cf-aaa096196c02",
+        "description": "Mizu observed 2 entries (0 failed), at 0.50 hits/s, average response time is 0.63 seconds",
+        "operationId": "<UUID4>",
         "responses": {
           "200": {
             "description": "Successful call with status 200",
             "content": {
               "application/json": {
-                "example": null
+                "example": null,
+                "x-sample-entry": "000000000000000000000006"
               }
-            }
+            },
+            "x-sample-entry": "000000000000000000000006"
           }
         },
         "x-counters-per-source": {
           "": {
             "entries": 2,
             "failures": 0,
-            "firstSeen": 1567750580.7471218,
-            "lastSeen": 1567750581.7471218,
+            "firstSeen": 1567750580.74,
+            "lastSeen": 1567750581.74,
             "sumRT": 1.26,
             "sumDuration": 1
           }
         },
-        "x-last-seen-ts": 1567750581.7471218,
         "x-counters-total": {
           "entries": 2,
           "failures": 0,
-          "firstSeen": 1567750580.7471218,
-          "lastSeen": 1567750581.7471218,
+          "firstSeen": 1567750580.74,
+          "lastSeen": 1567750581.74,
           "sumRT": 1.26,
           "sumDuration": 1
-        }
+        },
+        "x-last-seen-ts": 1567750581.74,
+        "x-sample-entry": "000000000000000000000006"
       }
     },
     "/body-optional": {
       "post": {
         "summary": "/body-optional",
-        "description": "Mizu observed 3 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
-        "operationId": "6e78e253-1f15-4ecd-9147-ba407c6fbeac",
+        "description": "Mizu observed 3 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
         "responses": {
           "200": {
             "description": "Successful call with status 200",
             "content": {
-              "": {}
-            }
+              "": {
+                "x-sample-entry": "000000000000000000000012"
+              }
+            },
+            "x-sample-entry": "000000000000000000000012"
           }
         },
-        "x-last-seen-ts": 1567750581.757122,
-        "x-counters-total": {
-          "entries": 3,
-          "failures": 0,
-          "firstSeen": 1567750581.7471218,
-          "lastSeen": 1567750581.757122,
-          "sumRT": 0.003,
-          "sumDuration": 0
-        },
         "x-counters-per-source": {
           "": {
             "entries": 3,
             "failures": 0,
-            "firstSeen": 1567750581.7471218,
-            "lastSeen": 1567750581.757122,
-            "sumRT": 0.003,
-            "sumDuration": 0
+            "firstSeen": 1567750581.74,
+            "lastSeen": 1567750581.75,
+            "sumRT": 0.00,
+            "sumDuration": 0.01
           }
         },
+        "x-counters-total": {
+          "entries": 3,
+          "failures": 0,
+          "firstSeen": 1567750581.74,
+          "lastSeen": 1567750581.75,
+          "sumRT": 0.00,
+          "sumDuration": 0.01
+        },
+        "x-last-seen-ts": 1567750581.75,
+        "x-sample-entry": "000000000000000000000012",
         "requestBody": {
           "description": "Generic request body",
           "content": {
             "application/json": {
-              "example": "{\"key\", \"val\"}"
+              "example": "{\"key\", \"val\"}",
+              "x-sample-entry": "000000000000000000000011"
             }
-          }
+          },
+          "x-sample-entry": "000000000000000000000012"
         }
       }
     },
     "/body-required": {
       "post": {
         "summary": "/body-required",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
-        "operationId": "1eefebc8-28c0-43d8-877f-4c5f67ff479d",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
         "responses": {
           "200": {
             "description": "Successful call with status 200",
             "content": {
-              "": {}
-            }
+              "": {
+                "x-sample-entry": "000000000000000000000013"
+              }
+            },
+            "x-sample-entry": "000000000000000000000013"
           }
         },
-        "x-last-seen-ts": 1567750581.757122,
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750581.757122,
-          "lastSeen": 1567750581.757122,
-          "sumRT": 0.001,
-          "sumDuration": 0
-        },
         "x-counters-per-source": {
           "": {
             "entries": 1,
             "failures": 0,
-            "firstSeen": 1567750581.757122,
-            "lastSeen": 1567750581.757122,
-            "sumRT": 0.001,
+            "firstSeen": 1567750581.75,
+            "lastSeen": 1567750581.75,
+            "sumRT": 0.00,
             "sumDuration": 0
           }
         },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750581.75,
+          "lastSeen": 1567750581.75,
+          "sumRT": 0.00,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750581.75,
+        "x-sample-entry": "000000000000000000000013",
         "requestBody": {
           "description": "Generic request body",
           "content": {
             "": {
-              "example": "body exists"
+              "example": "body exists",
+              "x-sample-entry": "000000000000000000000013"
             }
           },
-          "required": true
+          "required": true,
+          "x-sample-entry": "000000000000000000000013"
         }
       }
     },
     "/form-multipart": {
       "post": {
         "summary": "/form-multipart",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
-        "operationId": "fc1eb6a3-3d55-4b4a-bfd0-c2ae85d590d5",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
         "responses": {
           "200": {
             "description": "Successful call with status 200",
             "content": {
               "": {
-                "example": {}
+                "example": {},
+                "x-sample-entry": "000000000000000000000009"
               }
-            }
+            },
+            "x-sample-entry": "000000000000000000000009"
           }
         },
         "x-counters-per-source": {
           "": {
             "entries": 1,
             "failures": 0,
-            "firstSeen": 1567750582.7471218,
-            "lastSeen": 1567750582.7471218,
-            "sumRT": 0.001,
+            "firstSeen": 1567750582.74,
+            "lastSeen": 1567750582.74,
+            "sumRT": 0.00,
             "sumDuration": 0
           }
         },
-        "x-last-seen-ts": 1567750582.7471218,
         "x-counters-total": {
           "entries": 1,
           "failures": 0,
-          "firstSeen": 1567750582.7471218,
-          "lastSeen": 1567750582.7471218,
-          "sumRT": 0.001,
+          "firstSeen": 1567750582.74,
+          "lastSeen": 1567750582.74,
+          "sumRT": 0.00,
           "sumDuration": 0
         },
+        "x-last-seen-ts": 1567750582.74,
+        "x-sample-entry": "000000000000000000000009",
         "requestBody": {
           "description": "Generic request body",
           "content": {
@@ -228,45 +249,51 @@
                   }
                 }
               },
-              "example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n"
+              "example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n",
+              "x-sample-entry": "000000000000000000000009"
             }
           },
-          "required": true
+          "required": true,
+          "x-sample-entry": "000000000000000000000009"
         }
       }
     },
     "/form-urlencoded": {
       "post": {
         "summary": "/form-urlencoded",
-        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds",
-        "operationId": "b416c98b-4ba8-4739-bd2a-6c3e31478bc9",
+        "description": "Mizu observed 2 entries (0 failed), at 0.50 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
         "responses": {
           "200": {
             "description": "Successful call with status 200",
             "content": {
-              "": {}
-            }
+              "": {
+                "x-sample-entry": "000000000000000000000008"
+              }
+            },
+            "x-sample-entry": "000000000000000000000008"
           }
         },
-        "x-counters-total": {
-          "entries": 2,
-          "failures": 0,
-          "firstSeen": 1567750580.7471218,
-          "lastSeen": 1567750581.7471218,
-          "sumRT": 0.002,
-          "sumDuration": 1
-        },
         "x-counters-per-source": {
           "": {
             "entries": 2,
             "failures": 0,
-            "firstSeen": 1567750580.7471218,
-            "lastSeen": 1567750581.7471218,
-            "sumRT": 0.002,
+            "firstSeen": 1567750580.74,
+            "lastSeen": 1567750581.74,
+            "sumRT": 0.00,
             "sumDuration": 1
           }
         },
-        "x-last-seen-ts": 1567750581.7471218,
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750580.74,
+          "lastSeen": 1567750581.74,
+          "sumRT": 0.00,
+          "sumDuration": 1
+        },
+        "x-last-seen-ts": 1567750581.74,
+        "x-sample-entry": "000000000000000000000008",
         "requestBody": {
           "description": "Generic request body",
           "content": {
@@ -306,47 +333,409 @@
                   }
                 }
               },
-              "example": "agent-id=ade\u0026callback-url=\u0026token=sometoken"
+              "example": "agent-id=ade\u0026callback-url=\u0026token=sometoken-second-val\u0026optional=another",
+              "x-sample-entry": "000000000000000000000008"
             }
           },
-          "required": true
+          "required": true,
+          "x-sample-entry": "000000000000000000000008"
         }
       }
     },
-    "/{Id}": {
+    "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}": {
       "get": {
-        "summary": "/{Id}",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
-        "operationId": "d8b4de17-4c0e-401c-891b-36a7cee390bf",
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
         "responses": {
           "200": {
             "description": "Successful call with status 200",
             "content": {
-              "application/json": {
-                "example": null
+              "": {
+                "x-sample-entry": "000000000000000000000014"
               }
-            }
+            },
+            "x-sample-entry": "000000000000000000000014"
           }
         },
-        "x-last-seen-ts": 1567750579.7471218,
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750579.7471218,
-          "lastSeen": 1567750579.7471218,
-          "sumRT": 0.63,
-          "sumDuration": 0
-        },
         "x-counters-per-source": {
           "": {
             "entries": 1,
             "failures": 0,
-            "firstSeen": 1567750579.7471218,
-            "lastSeen": 1567750579.7471218,
+            "firstSeen": 1567750582,
+            "lastSeen": 1567750582,
+            "sumRT": 0.00,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582,
+          "lastSeen": 1567750582,
+          "sumRT": 0.00,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582,
+        "x-sample-entry": "000000000000000000000014"
+      },
+      "parameters": [
+        {
+          "name": "prefixgibberishfineId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "234324"
+            }
+          },
+          "x-sample-entry": "000000000000000000000014"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}",
+        "description": "Mizu observed 2 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000018"
+              }
+            },
+            "x-sample-entry": "000000000000000000000018"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 2,
+            "failures": 0,
+            "firstSeen": 1567750582.00,
+            "lastSeen": 1567750582.00,
+            "sumRT": 0.00,
+            "sumDuration": 9.53e-7
+          }
+        },
+        "x-counters-total": {
+          "entries": 2,
+          "failures": 0,
+          "firstSeen": 1567750582.00,
+          "lastSeen": 1567750582.00,
+          "sumRT": 0.00,
+          "sumDuration": 9.53e-7
+        },
+        "x-last-seen-ts": 1567750582.00,
+        "x-sample-entry": "000000000000000000000018"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/1": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/1",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000015"
+              }
+            },
+            "x-sample-entry": "000000000000000000000015"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.00,
+            "lastSeen": 1567750582.00,
+            "sumRT": 0.00,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.00,
+          "lastSeen": 1567750582.00,
+          "sumRT": 0.00,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.00,
+        "x-sample-entry": "000000000000000000000015"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/static": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/static",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000016"
+              }
+            },
+            "x-sample-entry": "000000000000000000000016"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.00,
+            "lastSeen": 1567750582.00,
+            "sumRT": 0.00,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.00,
+          "lastSeen": 1567750582.00,
+          "sumRT": 0.00,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.00,
+        "x-sample-entry": "000000000000000000000016"
+      },
+      "parameters": [
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/param-patterns/{parampatternId}/{param1}": {
+      "get": {
+        "tags": [
+          "param-patterns"
+        ],
+        "summary": "/param-patterns/{parampatternId}/{param1}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "": {
+                "x-sample-entry": "000000000000000000000019"
+              }
+            },
+            "x-sample-entry": "000000000000000000000019"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750582.00,
+            "lastSeen": 1567750582.00,
+            "sumRT": 0.00,
+            "sumDuration": 0
+          }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750582.00,
+          "lastSeen": 1567750582.00,
+          "sumRT": 0.00,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750582.00,
+        "x-sample-entry": "000000000000000000000019"
+      },
+      "parameters": [
+        {
+          "name": "param1",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string"
+          },
+          "examples": {
+            "example #0": {
+              "value": "23421"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        },
+        {
+          "name": "parampatternId",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "^prefix-gibberish-.+"
+          },
+          "examples": {
+            "example #0": {
+              "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
+            },
+            "example #1": {
+              "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
+            },
+            "example #2": {
+              "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
+            },
+            "example #3": {
+              "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
+            },
+            "example #4": {
+              "value": "prefix-gibberish-afterwards"
+            }
+          },
+          "x-sample-entry": "000000000000000000000019"
+        }
+      ]
+    },
+    "/{Id}": {
+      "get": {
+        "summary": "/{Id}",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
+        "operationId": "<UUID4>",
+        "responses": {
+          "200": {
+            "description": "Successful call with status 200",
+            "content": {
+              "application/json": {
+                "example": null,
+                "x-sample-entry": "000000000000000000000003"
+              }
+            },
+            "x-sample-entry": "000000000000000000000003"
+          }
+        },
+        "x-counters-per-source": {
+          "": {
+            "entries": 1,
+            "failures": 0,
+            "firstSeen": 1567750579.74,
+            "lastSeen": 1567750579.74,
             "sumRT": 0.63,
             "sumDuration": 0
           }
-        }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750579.74,
+          "lastSeen": 1567750579.74,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750579.74,
+        "x-sample-entry": "000000000000000000000003"
       },
       "parameters": [
         {
@@ -359,47 +748,52 @@
           },
           "examples": {
             "example #0": {
-              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+              "value": "<UUID4>"
             },
             "example #1": {
-              "value": "952bea17-3776-11ea-9341-42010a84012a"
+              "value": "<UUID4>"
             }
-          }
+          },
+          "x-sample-entry": "000000000000000000000003"
         }
       ]
     },
     "/{Id}/sub1": {
       "get": {
         "summary": "/{Id}/sub1",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds",
-        "operationId": "edacc275-1761-4011-af9b-3a3ed7c3d2b8",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.11 seconds",
+        "operationId": "<UUID4>",
         "responses": {
           "200": {
             "description": "Successful call with status 200",
             "content": {
-              "text/html": {}
-            }
+              "text/html": {
+                "x-sample-entry": "000000000000000000000001"
+              }
+            },
+            "x-sample-entry": "000000000000000000000001"
           }
         },
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750483.864529,
-          "lastSeen": 1567750483.864529,
-          "sumRT": 0.111,
-          "sumDuration": 0
-        },
         "x-counters-per-source": {
           "": {
             "entries": 1,
             "failures": 0,
-            "firstSeen": 1567750483.864529,
-            "lastSeen": 1567750483.864529,
-            "sumRT": 0.111,
+            "firstSeen": 1567750483.86,
+            "lastSeen": 1567750483.86,
+            "sumRT": 0.11,
             "sumDuration": 0
           }
         },
-        "x-last-seen-ts": 1567750483.864529
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750483.86,
+          "lastSeen": 1567750483.86,
+          "sumRT": 0.11,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750483.86,
+        "x-sample-entry": "000000000000000000000001"
       },
       "parameters": [
         {
@@ -412,49 +806,53 @@
           },
           "examples": {
             "example #0": {
-              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+              "value": "<UUID4>"
             },
             "example #1": {
-              "value": "952bea17-3776-11ea-9341-42010a84012a"
+              "value": "<UUID4>"
             }
-          }
+          },
+          "x-sample-entry": "000000000000000000000003"
         }
       ]
     },
     "/{Id}/sub2": {
       "get": {
         "summary": "/{Id}/sub2",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
-        "operationId": "61702da5-e4e7-4d95-8071-a6208f3d19d4",
+        "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
+        "operationId": "<UUID4>",
         "responses": {
           "200": {
             "description": "Successful call with status 200",
             "content": {
               "application/json": {
-                "example": null
+                "example": null,
+                "x-sample-entry": "000000000000000000000002"
               }
-            }
+            },
+            "x-sample-entry": "000000000000000000000002"
           }
         },
-        "x-last-seen-ts": 1567750578.7471218,
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750578.7471218,
-          "lastSeen": 1567750578.7471218,
-          "sumRT": 0.63,
-          "sumDuration": 0
-        },
         "x-counters-per-source": {
           "": {
             "entries": 1,
             "failures": 0,
-            "firstSeen": 1567750578.7471218,
-            "lastSeen": 1567750578.7471218,
+            "firstSeen": 1567750578.74,
+            "lastSeen": 1567750578.74,
             "sumRT": 0.63,
             "sumDuration": 0
           }
-        }
+        },
+        "x-counters-total": {
+          "entries": 1,
+          "failures": 0,
+          "firstSeen": 1567750578.74,
+          "lastSeen": 1567750578.74,
+          "sumRT": 0.63,
+          "sumDuration": 0
+        },
+        "x-last-seen-ts": 1567750578.74,
+        "x-sample-entry": "000000000000000000000002"
       },
       "parameters": [
         {
@@ -467,32 +865,33 @@
           },
           "examples": {
             "example #0": {
-              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
+              "value": "<UUID4>"
             },
             "example #1": {
-              "value": "952bea17-3776-11ea-9341-42010a84012a"
+              "value": "<UUID4>"
             }
-          }
+          },
+          "x-sample-entry": "000000000000000000000003"
         }
       ]
     }
   },
   "x-counters-per-source": {
     "": {
-      "entries": 13,
+      "entries": 19,
       "failures": 0,
-      "firstSeen": 1567750483.864529,
-      "lastSeen": 1567750582.7471218,
-      "sumRT": 3.268,
-      "sumDuration": 2
+      "firstSeen": 1567750483.86,
+      "lastSeen": 1567750582.74,
+      "sumRT": 3.27,
+      "sumDuration": 2.01
     }
   },
   "x-counters-total": {
-    "entries": 13,
+    "entries": 19,
     "failures": 0,
-    "firstSeen": 1567750483.864529,
-    "lastSeen": 1567750582.7471218,
-    "sumRT": 3.268,
-    "sumDuration": 2
+    "firstSeen": 1567750483.86,
+    "lastSeen": 1567750582.74,
+    "sumRT": 3.27,
+    "sumDuration": 2.01
   }
 }

agent/pkg/oas/test_artifacts/trcc.json

@@ -0,0 +1,50 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "Preloaded TRCC",
+    "version": "0.1",
+    "description": "Test file for loading pre-existing OAS"
+  },
+  "paths": {
+    "/models/{id}": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": ".+(_|-|\\.).+"
+          },
+          "example": "some-uuid-maybe"
+        }
+      ]
+    },
+    "/models/{id}/{id2}": {
+      "parameters": [
+        {
+          "name": "id",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": ".+(_|-|\\.).+"
+          },
+          "example": "some-uuid-maybe"
+        },
+        {
+          "name": "id2",
+          "in": "path",
+          "required": true,
+          "style": "simple",
+          "schema": {
+            "type": "string",
+            "pattern": "\\d+"
+          }
+        }
+      ]
+    }
+  }
+}

agent/pkg/oas/tree.go

@@ -1,12 +1,14 @@
 package oas
 
 import (
+	"encoding/json"
 	"net/url"
+	"regexp"
 	"strconv"
 	"strings"
 
 	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 type NodePath = []string
@@ -19,7 +21,7 @@ type Node struct {
 	children  []*Node
 }
 
-func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node *Node) {
+func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj, sampleId string) (node *Node) {
 	if existingPathObj == nil {
 		panic("Invalid function call")
 	}
@@ -49,8 +51,8 @@ func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node *
 		node = n.searchInConstants(pathChunk)
 	}
 
-	if node == nil {
-		node = n.searchInParams(paramObj, chunkIsGibberish)
+	if node == nil && pathChunk != "" {
+		node = n.searchInParams(paramObj, pathChunk, chunkIsGibberish)
 	}
 
 	// still no node found, should create it
@@ -69,6 +71,10 @@ func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node *
 		}
 	}
 
+	if node.pathParam != nil {
+		setSampleID(&node.pathParam.Extensions, sampleId)
+	}
+
 	// add example if it's a gibberish chunk
 	if node.pathParam != nil && !chunkIsParam {
 		exmp := &node.pathParam.Examples
@@ -76,11 +82,15 @@ func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node *
 		if err != nil {
 			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
 		}
+
+		if len(*exmp) >= 3 && node.pathParam.Schema.Pattern == nil { // is it enough to decide on 2 samples?
+			node.pathParam.Schema.Pattern = getPatternFromExamples(exmp)
+		}
 	}
 
 	// TODO: eat up trailing slash, in a smart way: node.pathObj!=nil && path[1]==""
 	if len(path) > 1 {
-		return node.getOrSet(path[1:], existingPathObj)
+		return node.getOrSet(path[1:], existingPathObj, sampleId)
 	} else if node.pathObj == nil {
 		node.pathObj = existingPathObj
 	}
@@ -88,6 +98,57 @@ func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node *
 	return node
 }
 
+func getPatternFromExamples(exmp *openapi.Examples) *openapi.Regexp {
+	allInts := true
+	strs := make([]string, 0)
+	for _, example := range *exmp {
+		exampleObj, err := example.ResolveExample(exampleResolver)
+		if err != nil {
+			continue
+		}
+
+		var value string
+		err = json.Unmarshal(exampleObj.Value, &value)
+		if err != nil {
+			logger.Log.Warningf("Failed decoding parameter example into string: %s", err)
+			continue
+		}
+		strs = append(strs, value)
+
+		if _, err := strconv.Atoi(value); err != nil {
+			allInts = false
+		}
+	}
+
+	if allInts {
+		re := new(openapi.Regexp)
+		re.Regexp = regexp.MustCompile(`\d+`)
+		return re
+	} else {
+		prefix := longestCommonXfixStr(strs, true)
+		suffix := longestCommonXfixStr(strs, false)
+
+		pat := ""
+		separators := "-._/:|*,+" // TODO: we could also cut prefix till the last separator
+		if len(prefix) > 0 && strings.Contains(separators, string(prefix[len(prefix)-1])) {
+			pat = "^" + regexp.QuoteMeta(prefix)
+		}
+
+		pat += ".+"
+
+		if len(suffix) > 0 && strings.Contains(separators, string(suffix[0])) {
+			pat += regexp.QuoteMeta(suffix) + "$"
+		}
+
+		if pat != ".+" {
+			re := new(openapi.Regexp)
+			re.Regexp = regexp.MustCompile(pat)
+			return re
+		}
+	}
+	return nil
+}
+
 func (n *Node) createParam() *openapi.ParameterObj {
 	name := "param"
 
@@ -118,23 +179,30 @@ func (n *Node) createParam() *openapi.ParameterObj {
 	return newParam
 }
 
-func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunkIsGibberish bool) *Node {
+func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunk string, chunkIsGibberish bool) *Node {
 	// look among params
-	if paramObj != nil || chunkIsGibberish {
-		for _, subnode := range n.children {
-			if subnode.constant != nil {
-				continue
-			}
-
-			// TODO: check the regex pattern of param? for exceptions etc
-
-			if paramObj != nil {
-				// TODO: mergeParam(subnode.pathParam, paramObj)
-				return subnode
-			} else {
-				return subnode
-			}
+	for _, subnode := range n.children {
+		if subnode.constant != nil {
+			continue
 		}
+
+		if paramObj != nil {
+			// TODO: mergeParam(subnode.pathParam, paramObj)
+			return subnode
+		} else if subnode.pathParam.Schema.Pattern != nil { // it has defined param pattern, have to respect it
+			// TODO: and not in exceptions
+			if subnode.pathParam.Schema.Pattern.Match([]byte(chunk)) {
+				return subnode
+			} else if chunkIsGibberish {
+				// TODO: what to do if gibberish chunk does not match the pattern and not in exceptions?
+				return nil
+			} else {
+				return nil
+			}
+		} else if chunkIsGibberish {
+			return subnode
+		}
+
 	}
 	return nil
 }

agent/pkg/oas/tree_test.go

@@ -1,9 +1,11 @@
 package oas
 
 import (
-	"github.com/chanced/openapi"
+	"fmt"
 	"strings"
 	"testing"
+
+	"github.com/chanced/openapi"
 )
 
 func TestTree(t *testing.T) {
@@ -19,10 +21,10 @@ func TestTree(t *testing.T) {
 	}
 
 	tree := new(Node)
-	for _, tc := range testCases {
+	for i, tc := range testCases {
 		split := strings.Split(tc.inp, "/")
 		pathObj := new(openapi.PathObj)
-		node := tree.getOrSet(split, pathObj)
+		node := tree.getOrSet(split, pathObj, fmt.Sprintf("%024d", i))
 
 		fillPathParams(node, pathObj)
 

agent/pkg/oas/utils.go

@@ -9,7 +9,7 @@ import (
 	"github.com/up9inc/mizu/agent/pkg/har"
 
 	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/logger"
 )
 
 func exampleResolver(ref string) (*openapi.ExampleObj, error) {
@@ -115,10 +115,10 @@ type nvParams struct {
 	GeneralizeName func(name string) string
 }
 
-func handleNameVals(gw nvParams, params **openapi.ParameterList) {
+func handleNameVals(gw nvParams, params **openapi.ParameterList, checkIgnore bool, sampleId string) {
 	visited := map[string]*openapi.ParameterObj{}
 	for _, pair := range gw.Pairs {
-		if gw.IsIgnored(pair.Name) {
+		if (checkIgnore && gw.IsIgnored(pair.Name)) || pair.Name == "" {
 			continue
 		}
 
@@ -137,6 +137,8 @@ func handleNameVals(gw nvParams, params **openapi.ParameterList) {
 			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
 		}
 		visited[nameGeneral] = param
+
+		setSampleID(&param.Extensions, sampleId)
 	}
 
 	// maintain "required" flag
@@ -290,6 +292,53 @@ func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.c
 	return xfix
 }
 
+func longestCommonXfixStr(strs []string, pre bool) string { // https://github.com/jpillora/longestcommon
+	//short-circuit empty list
+	if len(strs) == 0 {
+		return ""
+	}
+	xfix := strs[0]
+	//short-circuit single-element list
+	if len(strs) == 1 {
+		return xfix
+	}
+	//compare first to rest
+	for _, str := range strs[1:] {
+		xfixl := len(xfix)
+		strl := len(str)
+		//short-circuit empty strings
+		if xfixl == 0 || strl == 0 {
+			return ""
+		}
+		//maximum possible length
+		maxl := xfixl
+		if strl < maxl {
+			maxl = strl
+		}
+		//compare letters
+		if pre {
+			//prefix, iterate left to right
+			for i := 0; i < maxl; i++ {
+				if xfix[i] != str[i] {
+					xfix = xfix[:i]
+					break
+				}
+			}
+		} else {
+			//suffix, iternate right to left
+			for i := 0; i < maxl; i++ {
+				xi := xfixl - i - 1
+				si := strl - i - 1
+				if xfix[xi] != str[si] {
+					xfix = xfix[xi+1:]
+					break
+				}
+			}
+		}
+	}
+	return xfix
+}
+
 func getSimilarPrefix(strs []string) string {
 	chunked := make([][]string, 0)
 	for _, item := range strs {
@@ -427,3 +476,15 @@ func intersectSliceWithMap(required []string, names map[string]struct{}) []strin
 	}
 	return required
 }
+
+func setSampleID(extensions *openapi.Extensions, id string) {
+	if id != "" {
+		if *extensions == nil {
+			*extensions = openapi.Extensions{}
+		}
+		err := (extensions).SetExtension(SampleId, id)
+		if err != nil {
+			logger.Log.Warningf("Failed to set sample ID: %s", err)
+		}
+	}
+}

agent/pkg/oas/utils_test.go

@@ -29,7 +29,7 @@ func TestAnyJSON(t *testing.T) {
 		} else if tc.inp == "null" && any != nil {
 			t.Errorf("null has to parse as nil (but got %s)", any)
 		} else {
-			t.Logf("%s => %s", tc.inp, any)
+			t.Logf("%s => %v", tc.inp, any)
 		}
 	}
 }