Merge remote-tracking branch 'origin/develop'

Develop -> Main #patch

.dockerignore

@@ -2,6 +2,7 @@
 .dockerignore
 .editorconfig
 .gitignore
+**/.env*
 Dockerfile
 Makefile
 LICENSE

.github/workflows/acceptance_tests.yml

@@ -8,15 +8,19 @@ on:
     branches:
       - 'develop'
 
+concurrency:
+  group: mizu-acceptance-tests-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   run-acceptance-tests:
     name: Run acceptance tests
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.17
+      - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: '^1.17'
+          go-version: '^1.16'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
@@ -26,15 +30,3 @@ jobs:
 
       - name: Test
         run: make acceptance-test
-
-      - name: Slack notification on failure
-        uses: ravsamhq/notify-slack-action@v1
-        if: always()
-        with:
-          status: ${{ job.status }}
-          notification_title: 'Mizu {workflow} has {status_message}'
-          message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit: <{commit_url}|{commit_sha}>'
-          footer: 'Linked Repo <{repo_url}|{repo}>'
-          notify_when: 'failure'
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/inactive-issues-close.yaml

@@ -1,22 +0,0 @@
-name: Close inactive issues
-on:
-  schedule:
-    - cron: "0 0 * * *"
-
-jobs:
-  close-issues:
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-    steps:
-      - uses: actions/stale@v3
-        with:
-          days-before-issue-stale: 30
-          days-before-issue-close: 14
-          stale-issue-label: "stale"
-          stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
-          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
-          days-before-pr-stale: -1
-          days-before-pr-close: -1
-          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pr_validation.yml

@@ -1,4 +1,4 @@
-name: Build
+name: PR validation
 
 on:
   pull_request:
@@ -12,13 +12,13 @@ concurrency:
 
 jobs:
   build-cli:
-    name: CLI executable build
+    name: Build CLI
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.17
+      - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: '1.17'
+          go-version: '1.16'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
@@ -27,11 +27,20 @@ jobs:
         run: make cli
 
   build-agent:
-    name: Agent docker image build
+    name: Build Agent
     runs-on: ubuntu-latest
     steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
+      - shell: bash
+        run: |
+          sudo apt-get install libpcap-dev
+
       - name: Build Agent
-        run: make agent-docker
+        run: make agent

.github/workflows/publish.yml

@@ -0,0 +1,90 @@
+name: publish
+
+on:
+  push:
+    branches:
+      - 'develop'
+      - 'main'
+
+concurrency:
+  group: mizu-publish-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
+        with:
+          service_account_key: ${{ secrets.GCR_JSON_KEY }}
+          export_default_credentials: true
+      - uses: haya14busa/action-cond@v1
+        id: condval
+        with:
+          cond: ${{ github.ref == 'refs/heads/main' }}
+          if_true: "minor"
+          if_false: "patch"
+      - name: Auto Increment Semver Action
+        uses: MCKanpolat/auto-semver-action@1.0.5
+        id: versioning
+        with:
+          releaseType: ${{ steps.condval.outputs.value }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Get version parameters
+        shell: bash
+        run: |
+          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
+          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: version_parameters
+      - name: Get base image name
+        shell: bash
+        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
+        id: base_image_step
+      - name: Docker meta
+        id: meta
+        uses: crazy-max/ghaction-docker-meta@v2
+        with:
+          images: ${{ steps.base_image_step.outputs.image }}
+          tags: |
+            type=sha
+            type=raw,${{ github.sha }}
+            type=raw,${{ steps.versioning.outputs.version }}
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          registry: gcr.io
+          username: _json_key
+          password: ${{ secrets.GCR_JSON_KEY }}
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            SEM_VER=${{ steps.versioning.outputs.version }}
+            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
+            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
+            COMMIT_HASH=${{ github.sha }}
+      - name: Build and Push CLI
+        run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
+      - shell: bash
+        run: |
+          echo '${{ steps.versioning.outputs.version }}' >> cli/bin/version.txt
+      - name: publish
+        uses: ncipollo/release-action@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          artifacts: "cli/bin/*"
+          commit: ${{ steps.version_parameters.outputs.branch }}
+          tag: ${{ steps.versioning.outputs.version }}
+          prerelease: ${{ github.ref != 'refs/heads/main' }}
+          bodyFile: 'cli/bin/README.md'

.github/workflows/release.yml

@@ -1,287 +0,0 @@
-name: Release
-
-on:
-  push:
-    branches:
-      - 'develop'
-      - 'main'
-
-concurrency:
-  group: mizu-publish-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  docker-registry:
-    name: Push Docker image to Docker Hub
-    runs-on: ubuntu-latest
-
-    strategy:
-      max-parallel: 2
-      fail-fast: false
-      matrix:
-        target:
-          - amd64
-          - arm64v8
-
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v2
-
-      - name: Determine versioning strategy
-        uses: haya14busa/action-cond@v1
-        id: condval
-        with:
-          cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "stable"
-          if_false: "dev"
-
-      - name: Auto Increment Ver Action
-        uses: docker://igorgov/auto-inc-ver:v2.0.0
-        id: versioning
-        with:
-          mode: ${{ steps.condval.outputs.value }}
-          suffix: 'dev'
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Get version parameters
-        shell: bash
-        run: |
-          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
-          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: version_parameters
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            up9inc/mizu
-          tags: |
-            type=raw,${{ steps.versioning.outputs.version }}
-          flavor: |
-            latest=auto
-            prefix=
-            suffix=-${{ matrix.target }},onlatest=true
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_PASS }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            TARGETARCH=${{ matrix.target }}
-            VER=${{ steps.versioning.outputs.version }}
-            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
-            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
-            COMMIT_HASH=${{ github.sha }}
-
-  gcp-registry:
-    name: Push Docker image to GCR
-    runs-on: ubuntu-latest
-
-    strategy:
-      max-parallel: 2
-      fail-fast: false
-      matrix:
-        target:
-          - amd64
-          - arm64v8
-
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v2
-
-      - id: 'auth'
-        uses: 'google-github-actions/auth@v0'
-        with:
-          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
-
-      - name: 'Set up Cloud SDK'
-        uses: 'google-github-actions/setup-gcloud@v0'
-
-      - name: Determine versioning strategy
-        uses: haya14busa/action-cond@v1
-        id: condval
-        with:
-          cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "stable"
-          if_false: "dev"
-
-      - name: Auto Increment Ver Action
-        uses: docker://igorgov/auto-inc-ver:v2.0.0
-        id: versioning
-        with:
-          mode: ${{ steps.condval.outputs.value }}
-          suffix: 'dev'
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Get version parameters
-        shell: bash
-        run: |
-          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
-          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: version_parameters
-
-      - name: Get base image name
-        shell: bash
-        run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
-        id: base_image_step
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            ${{ steps.base_image_step.outputs.image }}
-          tags: |
-            type=raw,${{ steps.versioning.outputs.version }}
-          flavor: |
-            latest=auto
-            prefix=
-            suffix=-${{ matrix.target }},onlatest=true
-
-      - name: Login to GCR
-        uses: docker/login-action@v1
-        with:
-          registry: gcr.io
-          username: _json_key
-          password: ${{ secrets.GCR_JSON_KEY }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            TARGETARCH=${{ matrix.target }}
-            VER=${{ steps.versioning.outputs.version }}
-            BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
-            GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
-            COMMIT_HASH=${{ github.sha }}
-
-  docker-manifest:
-    name: Create and Push a Docker Manifest
-    runs-on: ubuntu-latest
-    needs: [docker-registry]
-    steps:
-      - name: Determine versioning strategy
-        uses: haya14busa/action-cond@v1
-        id: condval
-        with:
-          cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "stable"
-          if_false: "dev"
-
-      - name: Auto Increment Ver Action
-        uses: docker://igorgov/auto-inc-ver:v2.0.0
-        id: versioning
-        with:
-          mode: ${{ steps.condval.outputs.value }}
-          suffix: 'dev'
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Get version parameters
-        shell: bash
-        run: |
-          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
-          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: version_parameters
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            up9inc/mizu
-          tags: |
-            type=raw,${{ steps.versioning.outputs.version }}
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_PASS }}
-
-      - name: Create manifest
-        run: |
-          while IFS= read -r line; do
-            docker manifest create $line --amend $line-amd64 --amend $line-arm64v8
-          done <<< "${{ steps.meta.outputs.tags }}"
-
-      - name: Push manifest
-        run: |
-          while IFS= read -r line; do
-            docker manifest push $line
-          done <<< "${{ steps.meta.outputs.tags }}"
-
-  cli:
-    name: Build the CLI and publish
-    runs-on: ubuntu-latest
-    needs: [docker-manifest, gcp-registry]
-    steps:
-      - name: Set up Go 1.17
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.17'
-
-      - name: Check out the repo
-        uses: actions/checkout@v2
-
-      - id: 'auth'
-        uses: 'google-github-actions/auth@v0'
-        with:
-          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
-
-      - name: 'Set up Cloud SDK'
-        uses: 'google-github-actions/setup-gcloud@v0'
-
-      - name: Determine versioning strategy
-        uses: haya14busa/action-cond@v1
-        id: condval
-        with:
-          cond: ${{ github.ref == 'refs/heads/main' }}
-          if_true: "stable"
-          if_false: "dev"
-
-      - name: Auto Increment Ver Action
-        uses: docker://igorgov/auto-inc-ver:v2.0.0
-        id: versioning
-        with:
-          mode: ${{ steps.condval.outputs.value }}
-          suffix: 'dev'
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Get version parameters
-        shell: bash
-        run: |
-          echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
-          echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: version_parameters
-
-      - name: Build and Push CLI
-        run: make push-cli VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
-
-      - name: Log the version into a .txt file
-        shell: bash
-        run: |
-          echo '${{ steps.versioning.outputs.version }}' >> cli/bin/version.txt
-
-      - name: Release
-        uses: ncipollo/release-action@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          artifacts: "cli/bin/*"
-          commit: ${{ steps.version_parameters.outputs.branch }}
-          tag: ${{ steps.versioning.outputs.version }}
-          prerelease: ${{ github.ref != 'refs/heads/main' }}
-          bodyFile: 'cli/bin/README.md'

.github/workflows/security_validation.yml

@@ -0,0 +1,25 @@
+name: Security validation
+
+on:
+  pull_request:
+    branches:
+      - 'develop'
+      - 'main'
+
+jobs:
+  security:
+    name: Check for vulnerabilities
+    runs-on: ubuntu-latest
+    env:
+      SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: snyk/actions/setup@master
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+
+      - name: Run snyl on all projects
+        run: snyk test --all-projects

.github/workflows/static_code_analysis.yml

@@ -1,90 +0,0 @@
-name: Static code analysis
-
-on:
-  pull_request:
-    branches:
-      - 'develop'
-      - 'main'
-
-permissions:
-  contents: read
-
-jobs:
-  golangci:
-    name: Go lint
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-        with:
-            go-version: '^1.17'
-
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y libpcap-dev
-
-      - name: Go lint - agent
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: latest
-          working-directory: agent
-          args: --timeout=3m
-
-      - name: Go lint - shared
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: latest
-          working-directory: shared
-          args: --timeout=3m
-
-      - name: Go lint - tap
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: latest
-          working-directory: tap
-          args: --timeout=3m
-
-      - name: Go lint - CLI
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: latest
-          working-directory: cli
-          args: --timeout=3m
-
-      - name: Go lint - acceptanceTests
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: latest
-          working-directory: acceptanceTests
-          args: --timeout=3m
-
-      - name: Go lint - tap/api
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: latest
-          working-directory: tap/api
-
-      - name: Go lint - tap/extensions/amqp
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: latest
-          working-directory: tap/extensions/amqp
-
-      - name: Go lint - tap/extensions/http
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: latest
-          working-directory: tap/extensions/http
-
-      - name: Go lint - tap/extensions/kafka
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: latest
-          working-directory: tap/extensions/kafka
-
-      - name: Go lint - tap/extensions/redis
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: latest
-          working-directory: tap/extensions/redis

.github/workflows/tests_validation.yml

@@ -1,11 +1,11 @@
-name: Test
+name: tests validation
 
 on:
   pull_request:
     branches:
       - 'develop'
       - 'main'
-  push: # needed to upload test coverage report to codecov
+  push:
     branches:
       - 'develop'
       - 'main'
@@ -15,43 +15,42 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run-unit-tests:
-    name: Unit Tests
+  run-tests-cli:
+    name: Run CLI tests
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.17
+      - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: '^1.17'
+          go-version: '^1.16'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
-      - name: Install libpcap
-        shell: bash
-        run: |
-          sudo apt-get install libpcap-dev
-
-      - id: 'auth'
-        uses: 'google-github-actions/auth@v0'
-        with:
-          credentials_json: '${{ secrets.GCR_JSON_KEY }}'
-
-      - name: 'Set up Cloud SDK'
-        uses: 'google-github-actions/setup-gcloud@v0'
-
-      - name: CLI Test
+      - name: Test
         run: make test-cli
 
-      - name: Agent Test
-        run: make test-agent
-
-      - name: Shared Test
-        run: make test-shared
-
-      - name: Extensions Test
-        run: make test-extensions
-
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
 
+  run-tests-agent:
+    name: Run Agent tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - shell: bash
+        run: |
+          sudo apt-get install libpcap-dev
+
+      - name: Test
+        run: make test-agent
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2

.gitignore

@@ -15,6 +15,7 @@
 # vendor/
 .idea/
 build
+*.db
 
 # Mac OS
 .DS_Store
@@ -28,23 +29,3 @@ build
 
 # pprof
 pprof/*
-
-# Database Files
-*.bin
-*.gob
-
-# Nohup Files - https://man7.org/linux/man-pages/man1/nohup.1p.html
-nohup.*
-
-# Cypress tests
-cypress.env.json
-*/cypress/downloads
-*/cypress/fixtures
-*/cypress/plugins
-*/cypress/screenshots
-*/cypress/videos
-*/cypress/support
-
-# Ignore test data in extensions
-tap/extensions/*/bin
-tap/extensions/*/expect

Dockerfile

@@ -1,8 +1,4 @@
-ARG BUILDARCH=amd64
-ARG TARGETARCH=amd64
-
-### Front-end
-FROM node:16 AS front-end
+FROM node:14-slim AS site-build
 
 WORKDIR /app/ui-build
 
@@ -11,95 +7,56 @@ COPY ui/package-lock.json .
 RUN npm i
 COPY ui .
 RUN npm run build
-RUN npm run build-ent
-
-### Base builder image for native builds architecture
-FROM golang:1.17-alpine AS builder-native-base
-ENV CGO_ENABLED=1 GOOS=linux
-RUN apk add libpcap-dev g++ perl-utils
 
 
-### Intermediate builder image for x86-64 to x86-64 native builds
-FROM builder-native-base AS builder-from-amd64-to-amd64
-ENV GOARCH=amd64
+FROM golang:1.16-alpine AS builder
+# Set necessary environment variables needed for our image.
+ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 
+RUN apk add libpcap-dev gcc g++ make bash
 
-### Intermediate builder image for AArch64 to AArch64 native builds
-FROM builder-native-base AS builder-from-arm64v8-to-arm64v8
-ENV GOARCH=arm64
-
-
-### Builder image for x86-64 to AArch64 cross-compilation
-FROM up9inc/linux-arm64-musl-go-libpcap AS builder-from-amd64-to-arm64v8
-ENV CGO_ENABLED=1 GOOS=linux
-ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap"
-
-
-### Final builder image where the build happens
-# Possible build strategies:
-# BUILDARCH=amd64 TARGETARCH=amd64
-# BUILDARCH=arm64v8 TARGETARCH=arm64v8
-# BUILDARCH=amd64 TARGETARCH=arm64v8
-ARG BUILDARCH=amd64
-ARG TARGETARCH=amd64
-FROM builder-from-${BUILDARCH}-to-${TARGETARCH} AS builder
-
-# Move to agent working directory (/agent-build)
+# Move to agent working directory (/agent-build).
 WORKDIR /app/agent-build
 
 COPY agent/go.mod agent/go.sum ./
 COPY shared/go.mod shared/go.mod ../shared/
 COPY tap/go.mod tap/go.mod ../tap/
-COPY tap/api/go.mod ../tap/api/
-COPY tap/extensions/amqp/go.mod ../tap/extensions/amqp/
-COPY tap/extensions/http/go.mod ../tap/extensions/http/
-COPY tap/extensions/kafka/go.mod ../tap/extensions/kafka/
-COPY tap/extensions/redis/go.mod ../tap/extensions/redis/
+COPY tap/api/go.* ../tap/api/
 RUN go mod download
-# cheap trick to make the build faster (as long as go.mod did not change)
-RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
+# cheap trick to make the build faster (As long as go.mod wasn't changes)
+RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
+
+ARG COMMIT_HASH
+ARG GIT_BRANCH
+ARG BUILD_TIMESTAMP
+ARG SEM_VER=0.0.0
 
 # Copy and build agent code
 COPY shared ../shared
 COPY tap ../tap
 COPY agent .
+RUN go build -ldflags="-s -w \
+     -X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
+     -X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
+     -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
+     -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
-ARG COMMIT_HASH
-ARG GIT_BRANCH
-ARG BUILD_TIMESTAMP
-ARG VER=0.0
+COPY devops/build_extensions.sh ..
+RUN cd .. && /bin/bash build_extensions.sh
 
-WORKDIR /app/agent-build
+FROM alpine:3.14
 
-RUN go build -ldflags="-extldflags=-static -s -w \
-    -X 'github.com/up9inc/mizu/agent/pkg/version.GitCommitHash=${COMMIT_HASH}' \
-    -X 'github.com/up9inc/mizu/agent/pkg/version.Branch=${GIT_BRANCH}' \
-    -X 'github.com/up9inc/mizu/agent/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
-    -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
-
-# Download Basenine executable, verify the sha1sum
-ADD https://github.com/up9inc/basenine/releases/download/v0.4.16/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
-ADD https://github.com/up9inc/basenine/releases/download/v0.4.16/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
-RUN shasum -a 256 -c basenine_linux_${GOARCH}.sha256
-RUN chmod +x ./basenine_linux_${GOARCH}
-RUN mv ./basenine_linux_${GOARCH} ./basenine
-
-
-### The shipped image
-ARG TARGETARCH=amd64
-FROM ${TARGETARCH}/busybox:latest
-
-# gin-gonic runs in debug mode without this
-ENV GIN_MODE=release
-
-WORKDIR /app/data/
+RUN apk add bash libpcap-dev tcpdump
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/app/agent-build/mizuagent", "."]
-COPY --from=builder ["/app/agent-build/basenine", "/usr/local/bin/basenine"]
-COPY --from=front-end ["/app/ui-build/build", "site"]
-COPY --from=front-end ["/app/ui-build/build-ent", "site-standalone"]
+COPY --from=builder ["/app/agent/build/extensions", "extensions"]
+COPY --from=site-build ["/app/ui-build/build", "site"]
+RUN mkdir /app/data/
+
+# gin-gonic runs in debug mode without this
+ENV GIN_MODE=release
 
 # this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
-ENTRYPOINT ["/app/mizuagent"]
+ENTRYPOINT "/app/mizuagent"

Makefile

@@ -8,7 +8,7 @@ SHELL=/bin/bash
 # HELP
 # This will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-.PHONY: help ui agent agent-debug cli tap docker
+.PHONY: help ui agent cli tap docker
 
 help: ## This help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -19,7 +19,7 @@ help: ## This help.
 TS_SUFFIX="$(shell date '+%s')"
 GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
 BUCKET_PATH=static.up9.io/mizu/$(GIT_BRANCH)
-export VER?=0.0
+export SEM_VER?=0.0.0
 
 ui: ## Build UI.
 	@(cd ui; npm i ; npm run build; )
@@ -28,35 +28,28 @@ ui: ## Build UI.
 cli: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build
 
-cli-debug: ## Build CLI.
-	@echo "building cli"; cd cli && $(MAKE) build-debug
-
 build-cli-ci: ## Build CLI for CI.
 	@echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci
 
 agent: ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
-	@ls -l agent/build
-
-agent-debug: ## Build agent for debug.
-	@(echo "building mizu agent for debug.." )
-	@(cd agent; go build -gcflags="all=-N -l" -o build/mizuagent main.go)
+	${MAKE} extensions
 	@ls -l agent/build
 
 docker: ## Build and publish agent docker image.
 	$(MAKE) push-docker
 
-agent-docker: ## Build agent docker image.
-	@echo "Building agent docker image"
-	@docker build -t up9inc/mizu:devlatest .
-
 push: push-docker push-cli ## Build and publish agent docker image & CLI.
 
 push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
 	devops/build-push-featurebranch.sh
 
+push-docker-debug:
+	@echo "publishing debug Docker image .. "
+	devops/build-push-featurebranch-debug.sh
+
 build-docker-ci: ## Build agent docker image for CI.
 	@echo "building docker image for ci"
 	devops/build-agent-ci.sh
@@ -80,29 +73,17 @@ clean-agent: ## Clean agent.
 clean-cli:  ## Clean CLI.
 	@(cd cli; make clean ; echo "CLI cleanup done" )
 
-clean-docker:  ## Run clen docker
+clean-docker:
 	@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
 
-test-lint:  ## Run lint on all modules
-	cd agent && golangci-lint run
-	cd shared && golangci-lint run
-	cd tap && golangci-lint run
-	cd cli && golangci-lint run
-	cd acceptanceTests && golangci-lint run
-	cd tap/api && golangci-lint run
-	cd tap/extensions/ && for D in */; do cd $$D && golangci-lint run && cd ..; done
+extensions:
+	devops/build_extensions.sh
 
-test-cli:  ## Run cli tests
+test-cli:
 	@echo "running cli tests"; cd cli && $(MAKE) test
 
-test-agent:  ## Run agent tests
+test-agent:
 	@echo "running agent tests"; cd agent && $(MAKE) test
 
-test-shared:  ## Run shared tests
-	@echo "running shared tests"; cd shared && $(MAKE) test
-
-test-extensions:  ## Run extensions tests
-	@echo "running http tests"; cd tap/extensions/http && $(MAKE) test
-
-acceptance-test:  ## Run acceptance tests
+acceptance-test:
 	@echo "running acceptance tests"; cd acceptanceTests && $(MAKE) test

README.md

@@ -1,42 +1,19 @@
 ![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
 
-<p align="center">
-    <a href="https://github.com/up9inc/mizu/blob/main/LICENSE">
-        <img alt="GitHub License" src="https://img.shields.io/github/license/up9inc/mizu?logo=GitHub&style=flat-square">
-    </a>
-    <a href="https://github.com/up9inc/mizu/releases/latest">
-        <img alt="GitHub Latest Release" src="https://img.shields.io/github/v/release/up9inc/mizu?logo=GitHub&style=flat-square">
-    </a>
-    <a href="https://hub.docker.com/r/up9inc/mizu">
-      <img alt="Docker pulls" src="https://img.shields.io/docker/pulls/up9inc/mizu?color=%23099cec">
-    </a>
-    <a href="https://hub.docker.com/r/up9inc/mizu">
-      <img alt="Image size" src="https://img.shields.io/docker/image-size/up9inc/mizu/latest">
-    </a>
-    <a href="https://join.slack.com/t/up9/shared_invite/zt-tfjnduli-QzlR8VV4Z1w3YnPIAJfhlQ">
-      <img alt="Slack" src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social">
-    </a>
-</p>
-
 # The API Traffic Viewer for Kubernetes
 
 A simple-yet-powerful API traffic viewer for Kubernetes enabling you to view all API communication between microservices to help your debug and troubleshoot regressions.
 
-Think TCPDump and Wireshark re-invented for Kubernetes.
+Think TCPDump and Chrome Dev Tools combined.
 
 ![Simple UI](assets/mizu-ui.png)
 
 ## Features
 
 - Simple and powerful CLI
-- Monitoring network traffic in real-time. Supported protocols:
-  - [HTTP/1.x](https://datatracker.ietf.org/doc/html/rfc2616) (REST, GraphQL, SOAP, etc.)
-  - [HTTP/2](https://datatracker.ietf.org/doc/html/rfc7540) (gRPC)
-  - [AMQP](https://www.rabbitmq.com/amqp-0-9-1-reference.html) (RabbitMQ, Apache Qpid, etc.)
-  - [Apache Kafka](https://kafka.apache.org/protocol)
-  - [Redis](https://redis.io/topics/protocol)
-- Works with Kubernetes APIs. No installation or code instrumentation
-- Rich filtering
+- Real-time view of all HTTP requests, REST and gRPC API calls
+- No installation or code instrumentation
+- Works completely on premises
 
 ## Requirements
 
@@ -67,11 +44,20 @@ SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/
 ### Development (unstable) Build
 Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page
 
+## Kubeconfig & Permissions
+While `mizu`most often works out of the box, you can influence its behavior:
+
+1. [OPTIONAL] Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
+2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination)
+
+For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document
+
+
 ## How to Run
 
 1. Find pods you'd like to tap to in your Kubernetes cluster
-2. Run `mizu tap` or `mizu tap PODNAME`
-3. Open browser on `http://localhost:8899` **or** as instructed in the CLI
+2. Run `mizu tap` or `mizu tap PODNAME`  
+3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI
 4. Watch the API traffic flowing
 5. Type ^C to stop
 
@@ -97,7 +83,7 @@ To tap all pods in current namespace -
 ```
 
 
-### To tap specific pod
+To tap specific pod - 
 ```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
@@ -110,7 +96,7 @@ To tap all pods in current namespace -
  ^C
 ```
 
-### To tap multiple pods using regex
+To tap multiple pods using regex - 
 ```bash
  $ kubectl get pods 
  NAME                            READY   STATUS    RESTARTS   AGE
@@ -128,22 +114,21 @@ To tap all pods in current namespace -
 
 ## Configuration
 
-Mizu can optionally work with a config file that can be provided as a CLI argument (using `--set config-path=<PATH>`) or if not provided, will be stored at ${HOME}/.mizu/config.yaml 
+Mizu can work with config file which should be stored in ${HOME}/.mizu/config.yaml (macOS: ~/.mizu/config.yaml) <br />
+In case no config file found, defaults will be used <br />
 In case of partial configuration defined, all other fields will be used with defaults <br />
 You can always override the defaults or config file with CLI flags
 
 To get the default config params run `mizu config` <br />
 To generate a new config file with default values use `mizu config -r`
 
+### Telemetry
+
+By default, mizu reports usage telemetry. It can be disabled by adding a line of `telemetry: false` in the `${HOME}/.mizu/config.yaml` file
+
 
 ## Advanced Usage
 
-### Kubeconfig
-
-It is possible to change the kubeconfig path using `KUBECONFIG` environment variable or the command like flag
-with `--set kube-config-path=<PATH>`. </br >
-If both are not set - Mizu assumes that configuration is at `${HOME}/.kube/config`
-
 ### Namespace-Restricted Mode
 
 Some users have permission to only manage resources in one particular namespace assigned to them
@@ -157,8 +142,6 @@ using the `--namespace` flag or by setting `tap.namespaces` in the config file
 
 Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
 
-For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document
-
 ### User agent filtering
 
 User-agent filtering (like health checks) - can be configured using command-line options:
@@ -190,14 +173,32 @@ Please see [CONTRACT MONITORING](docs/CONTRACT_MONITORING.md) page for more deta
 
 ### Configure proxy host 
 
-By default, mizu will be accessible via local host: 'http://localhost:8899', it is possible to change the host, for
-instance, to '0.0.0.0' which can grant access via machine IP address. This setting can be changed via command line
-flag `--set tap.proxy-host=<value>` or via config file:
-tap proxy-host: 0.0.0.0 and when changed it will support accessing by IP
+By default, mizu will be accessible via local host: 'http://localhost:8899/mizu/', it is possible to change the host,
+for instance, to '0.0.0.0' which can grant access via machine IP address.
+This setting can be changed via command line flag `--set tap.proxy-host=<value>` or via config file:
+tap
+    proxy-host: 0.0.0.0
+and when changed it will support accessing by IP
 
-### Install Mizu standalone
+### Run in daemon mode
 
-Mizu can be run detached from the cli using the install command: `mizu install`. This type of mizu instance will run
-indefinitely in the cluster.
+Mizu can be ran detached from the cli using the daemon flag: `mizu tap --daemon`. This type of mizu instance will run indefinitely in the cluster.
 
-For more information please refer to [INSTALL STANDALONE](docs/INSTALL_STANDALONE.md)
+Please note that daemon mode requires you to have RBAC creation permissions, see the [permissions](docs/PERMISSIONS.md) doc for more details.
+
+In order to access a daemon mizu you will have to run `mizu view` after running the `tap --daemon` command.
+
+To stop the detached mizu instance and clean all cluster side resources, run `mizu clean`
+
+
+## How to Run local UI
+
+- run from mizu/agent `go run main.go --hars-read --hars-dir <folder>`
+
+- copy Har files into the folder from last command
+
+- change `MizuWebsocketURL` and `apiURL` in `api.js` file
+
+- run from mizu/ui - `npm run start`
+
+- open browser on `localhost:3000`

acceptanceTests/.snyk

@@ -1,6 +0,0 @@
-# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
-version: v1.14.0
-ignore:
-  SNYK-GOLANG-GITHUBCOMPKGSFTP-569475:
-    - '*':
-      reason: None Given

acceptanceTests/Makefile

@@ -1,2 +1,2 @@
 test: ## Run acceptance tests.
-	@go test ./... -timeout 1h -v
+	@go test ./... -timeout 1h

acceptanceTests/config_test.go

@@ -2,12 +2,11 @@ package acceptanceTests
 
 import (
 	"fmt"
+	"gopkg.in/yaml.v3"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"testing"
-
-	"gopkg.in/yaml.v3"
 )
 
 type tapConfig struct {

acceptanceTests/cypress.json

@@ -1,30 +0,0 @@
-{
-  "watchForFileChanges":false,
-  "viewportWidth": 1920,
-  "viewportHeight": 1080,
-  "video": false,
-  "screenshotOnRunFailure": false,
-  "defaultCommandTimeout": 6000,
-  "testFiles": [
-    "tests/GuiPort.js",
-    "tests/MultipleNamespaces.js",
-    "tests/Redact.js",
-    "tests/NoRedact.js",
-    "tests/Regex.js",
-    "tests/RegexMasking.js",
-    "tests/IgnoredUserAgents.js",
-    "tests/UiTest.js",
-    "tests/Redis.js"
-  ],
-
-  "env": {
-    "testUrl": "http://localhost:8899/",
-    "redactHeaderContent": "User-Header[REDACTED]",
-    "redactBodyContent": "{ \"User\": \"[REDACTED]\" }",
-    "regexMaskingBodyContent": "[REDACTED]",
-    "minimumEntries": 25,
-    "greenFilterColor": "rgb(210, 250, 210)",
-    "redFilterColor": "rgb(250, 214, 220)",
-    "bodyJsonClass": ".hljs"
-  }
-}

acceptanceTests/cypress/integration/testHelpers/StatusBarHelper.js

@@ -1,46 +0,0 @@
-const columns = {podName : 1, namespace : 2, tapping : 3};
-const greenStatusImageSrc = '/static/media/success.662997eb.svg';
-
-function getDomPathInStatusBar(line, column) {
-    return `.expandedStatusBar > :nth-child(2) > > :nth-child(2) > :nth-child(${line}) > :nth-child(${column})`;
-}
-
-export function checkLine(line, expectedValues) {
-    cy.get(getDomPathInStatusBar(line, columns.podName)).invoke('text').then(podValue => {
-        const podName = getOnlyPodName(podValue);
-        expect(podName).to.equal(expectedValues.podName);
-
-        cy.get(getDomPathInStatusBar(line, columns.namespace)).invoke('text').then(namespaceValue => {
-            expect(namespaceValue).to.equal(expectedValues.namespace);
-            cy.get(getDomPathInStatusBar(line, columns.tapping)).children().should('have.attr', 'src', greenStatusImageSrc);
-        });
-    });
-}
-
-export function findLineAndCheck(expectedValues) {
-    cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(1)').then(pods => {
-        cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) > > :nth-child(2)').then(namespaces => {
-            // organizing namespaces array
-            const podObjectsArray = Object.values(pods ?? {});
-            const namespacesObjectsArray = Object.values(namespaces ?? {});
-            let lineNumber = -1;
-            namespacesObjectsArray.forEach((namespaceObj, index) => {
-                const currentLine = index + 1;
-                lineNumber = (namespaceObj.getAttribute && namespaceObj.innerHTML === expectedValues.namespace && (getOnlyPodName(podObjectsArray[index].innerHTML)) === expectedValues.podName) ? currentLine : lineNumber;
-            });
-            lineNumber === -1 ? throwError(expectedValues) : checkLine(lineNumber, expectedValues);
-        });
-    });
-}
-
-function throwError(expectedValues) {
-    throw new Error(`The pod named ${expectedValues.podName} doesn't match any namespace named ${expectedValues.namespace}`);
-}
-
-export function getExpectedDetailsDict(podName, namespace) {
-    return {podName : podName, namespace : namespace};
-}
-
-function getOnlyPodName(podElementFullStr) {
-    return podElementFullStr.substring(0, podElementFullStr.indexOf('-'));
-}

acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js

@@ -1,55 +0,0 @@
-export function isValueExistsInElement(shouldInclude, content, domPathToContainer){
-    it(`should ${shouldInclude ? '' : 'not'} include '${content}'`, function () {
-        cy.get(domPathToContainer).then(htmlText => {
-            const allTextString = htmlText.text();
-            if (allTextString.includes(content) !== shouldInclude)
-                throw new Error(`One of the containers part contains ${content}`)
-        });
-    });
-}
-
-export function resizeToHugeMizu() {
-    cy.viewport(1920, 3500);
-}
-
-export function resizeToNormalMizu() {
-    cy.viewport(1920, 1080);
-}
-
-export function verifyMinimumEntries() {
-    const minimumEntries = Cypress.env('minimumEntries');
-    it(`Making sure that mizu shows at least ${minimumEntries} entries`, async function () {
-        cy.get('#total-entries').then(number => {
-            const getNum = () => {
-                const numOfEntries = number.text();
-                return parseInt(numOfEntries);
-            };
-            cy.wrap({there: getNum}).invoke('there').should('be.gte', minimumEntries);
-        });
-    });
-}
-
-export function leftTextCheck(entryNum, path, expectedText) {
-    cy.get(`#list #entry-${entryNum} ${path}`).invoke('text').should('eq', expectedText);
-}
-
-export function leftOnHoverCheck(entryNum, path, filterName) {
-    cy.get(`#list #entry-${entryNum} ${path}`).trigger('mouseover');
-    cy.get(`#list #entry-${entryNum} .Queryable-Tooltip`).invoke('text').should('match', new RegExp(filterName));
-}
-
-export function rightTextCheck(path, expectedText) {
-    cy.get(`#rightSideContainer ${path}`).should('have.text', expectedText);
-}
-
-export function rightOnHoverCheck(path, expectedText) {
-    cy.get(`#rightSideContainer ${path}`).trigger('mouseover');
-    cy.get(`#rightSideContainer .Queryable-Tooltip`).invoke('text').should('match', new RegExp(expectedText));
-}
-
-export function checkThatAllEntriesShown() {
-    cy.get('#entries-length').then(number => {
-        if (number.text() === '1')
-            cy.get('[title="Fetch old records"]').click();
-    });
-}

acceptanceTests/cypress/integration/tests/GuiPort.js

@@ -1,16 +0,0 @@
-import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
-import {verifyMinimumEntries} from "../testHelpers/TrafficHelper";
-
-it('check', function () {
-    const podName = Cypress.env('name'), namespace = Cypress.env('namespace');
-    const port = Cypress.env('port');
-    cy.intercept('GET', `http://localhost:${port}/status/tap`).as('statusTap');
-
-    cy.visit(`http://localhost:${port}`);
-    cy.wait('@statusTap').its('response.statusCode').should('match', /^2\d{2}/);
-
-    verifyMinimumEntries();
-
-    cy.get('.podsCount').trigger('mouseover');
-    findLineAndCheck(getExpectedDetailsDict(podName, namespace));
-});

acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js

@@ -1,32 +0,0 @@
-import {
-    checkThatAllEntriesShown,
-    isValueExistsInElement,
-    resizeToHugeMizu,
-    verifyMinimumEntries
-} from "../testHelpers/TrafficHelper";
-
-it('Loading Mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-verifyMinimumEntries();
-
-checkEntries();
-
-function checkEntries() {
-    it('checking all entries', function () {
-        checkThatAllEntriesShown();
-        resizeToHugeMizu();
-
-        cy.get('#total-entries').then(number => {
-            const numOfEntries = parseInt(number.text());
-            [...Array(numOfEntries).keys()].map(checkEntry);
-        });
-    });
-}
-
-function checkEntry(entryIndex) {
-    cy.get(`#entry-${entryIndex}`).click();
-    cy.get('#tbody-Headers').should('be.visible');
-    isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers');
-}

acceptanceTests/cypress/integration/tests/MultipleNamespaces.js

@@ -1,17 +0,0 @@
-import {findLineAndCheck, getExpectedDetailsDict} from '../testHelpers/StatusBarHelper';
-
-it('opening', function () {
-    cy.visit(Cypress.env('testUrl'));
-    cy.get('.podsCount').trigger('mouseover');
-});
-
-[1, 2, 3].map(doItFunc);
-
-function doItFunc(number) {
-    const podName = Cypress.env(`name${number}`);
-    const namespace = Cypress.env(`namespace${number}`);
-
-    it(`verifying the pod (${podName}, ${namespace})`, function () {
-        findLineAndCheck(getExpectedDetailsDict(podName, namespace));
-    });
-}

acceptanceTests/cypress/integration/tests/NoRedact.js

@@ -1,10 +0,0 @@
-import {isValueExistsInElement, verifyMinimumEntries} from '../testHelpers/TrafficHelper';
-
-it('Loading Mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-verifyMinimumEntries();
-
-isValueExistsInElement(false, Cypress.env('redactHeaderContent'), '#tbody-Headers');
-isValueExistsInElement(false, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/integration/tests/Redact.js

@@ -1,10 +0,0 @@
-import {isValueExistsInElement, verifyMinimumEntries} from '../testHelpers/TrafficHelper';
-
-it('Loading Mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-verifyMinimumEntries();
-
-isValueExistsInElement(true, Cypress.env('redactHeaderContent'), '#tbody-Headers');
-isValueExistsInElement(true, Cypress.env('redactBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/integration/tests/Redis.js

@@ -1,155 +0,0 @@
-import {
-    leftOnHoverCheck,
-    leftTextCheck,
-    rightOnHoverCheck,
-    rightTextCheck,
-} from "../testHelpers/TrafficHelper";
-
-const valueTabs = {
-    response: 'RESPONSE',
-    request: 'REQUEST',
-    none: null
-}
-
-it('opening mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-checkRedisFilterByMethod({
-    method: 'PING',
-    shouldCheckSummary: false,
-    valueTab: valueTabs.none
-});
-
-checkRedisFilterByMethod({
-    method: 'SET',
-    shouldCheckSummary: true,
-    valueTab: valueTabs.request,
-    valueRegex: /^\[value, keepttl]$/mg
-});
-
-checkRedisFilterByMethod({
-    method: 'EXISTS',
-    shouldCheckSummary: true,
-    valueTab: valueTabs.response,
-    valueRegex: /^1$/mg
-});
-
-checkRedisFilterByMethod({
-    method: 'GET',
-    shouldCheckSummary: true,
-    valueTab: valueTabs.response,
-    valueRegex: /^value$/mg
-});
-
-checkRedisFilterByMethod({
-    method: 'DEL',
-    shouldCheckSummary: true,
-    valueTab: valueTabs.response,
-    valueRegex: /^1$|^0$/mg
-});
-
-function checkRedisFilterByMethod(funcDict) {
-    const {method, shouldCheckSummary} = funcDict
-    const summaryDict = getSummeryDict();
-    const methodDict = getMethodDict(method);
-    const protocolDict = getProtocolDict();
-
-    it(`Testing the method: ${method}`, function () {
-        // applying filter
-        cy.get('.w-tc-editor-text').clear().type(`method == "${method}"`);
-        cy.get('[type="submit"]').click();
-        cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
-
-        cy.get('#entries-length').then(number => {
-            // if the entries list isn't expanded it expands here
-            if (number.text() === '0' || number.text() === '1') // todo change when TRA-4262 is fixed
-                cy.get('[title="Fetch old records"]').click();
-
-            cy.get('#entries-length').should('not.have.text', '0').and('not.have.text', '1').then(() => {
-                cy.get(`#list [id]`).then(elements => {
-                   const listElmWithIdAttr = Object.values(elements);
-                   let doneCheckOnFirst = false;
-
-                   listElmWithIdAttr.forEach(entry => {
-                       if (entry?.id && entry.id.match(RegExp(/entry-(\d{2}|\d{1})$/gm))) {
-                           const entryNum = getEntryNumById(entry.id);
-
-                           leftTextCheck(entryNum, methodDict.pathLeft, methodDict.expectedText);
-                           leftTextCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedTextLeft);
-                           if (shouldCheckSummary)
-                               leftTextCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedText);
-
-                           if (!doneCheckOnFirst) {
-                               deepCheck(funcDict, protocolDict, methodDict, summaryDict, entry);
-                               doneCheckOnFirst = true;
-                           }
-                       }
-                   });
-                });
-            });
-        });
-    });
-}
-
-function deepCheck(generalDict, protocolDict, methodDict, summaryDict, entry) {
-    const entryNum = getEntryNumById(entry.id);
-    const {shouldCheckSummary, valueTab, valueRegex} = generalDict;
-
-    leftOnHoverCheck(entryNum, methodDict.pathLeft, methodDict.expectedOnHover);
-    leftOnHoverCheck(entryNum, protocolDict.pathLeft, protocolDict.expectedOnHover);
-    if (shouldCheckSummary)
-        leftOnHoverCheck(entryNum, summaryDict.pathLeft, summaryDict.expectedOnHover);
-
-    cy.get(`#${entry.id}`).click();
-
-    rightTextCheck(methodDict.pathRight, methodDict.expectedText);
-    rightTextCheck(protocolDict.pathRight, protocolDict.expectedTextRight);
-    if (shouldCheckSummary)
-        rightTextCheck(summaryDict.pathRight, summaryDict.expectedText);
-
-    rightOnHoverCheck(methodDict.pathRight, methodDict.expectedOnHover);
-    rightOnHoverCheck(protocolDict.pathRight, protocolDict.expectedOnHover);
-    if (shouldCheckSummary)
-        rightOnHoverCheck(summaryDict.pathRight, summaryDict.expectedOnHover);
-
-    if (valueTab) {
-        if (valueTab === valueTabs.response)
-            cy.contains('Response').click();
-        cy.get(Cypress.env('bodyJsonClass')).then(text => {
-            expect(text.text()).to.match(valueRegex)
-        });
-    }
-}
-
-function getSummeryDict() {
-    return {
-        pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
-        pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
-        expectedText: 'key',
-        expectedOnHover: `summary == "key"`
-    };
-}
-
-function getMethodDict(method) {
-    return {
-        pathLeft: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
-        pathRight: '> :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
-        expectedText: method,
-        expectedOnHover: `method == "${method}"`
-    };
-}
-
-function getProtocolDict() {
-    return {
-        pathLeft: '> :nth-child(1) > :nth-child(1)',
-        pathRight: '> :nth-child(1) > :nth-child(1) > :nth-child(1) > :nth-child(1)',
-        expectedTextLeft: 'REDIS',
-        expectedTextRight: 'Redis Serialization Protocol',
-        expectedOnHover: `redis`
-    };
-}
-
-function getEntryNumById (id) {
-    return parseInt(id.split('-')[1]);
-}

acceptanceTests/cypress/integration/tests/Regex.js

@@ -1,11 +0,0 @@
-import {getExpectedDetailsDict, checkLine} from '../testHelpers/StatusBarHelper';
-
-
-it('opening', function () {
-    cy.visit(Cypress.env('testUrl'));
-    cy.get('.podsCount').trigger('mouseover');
-
-    cy.get('.expandedStatusBar > :nth-child(2) > > :nth-child(2) >').should('have.length', 1); // one line
-
-    checkLine(1, getExpectedDetailsDict(Cypress.env('name'), Cypress.env('namespace')));
-});

acceptanceTests/cypress/integration/tests/RegexMasking.js

@@ -1,9 +0,0 @@
-import {isValueExistsInElement, verifyMinimumEntries} from "../testHelpers/TrafficHelper";
-
-it('Loading Mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-verifyMinimumEntries();
-
-isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -1,322 +0,0 @@
-import {findLineAndCheck, getExpectedDetailsDict} from "../testHelpers/StatusBarHelper";
-import {
-    leftTextCheck,
-    resizeToHugeMizu,
-    resizeToNormalMizu,
-    rightOnHoverCheck,
-    leftOnHoverCheck,
-    rightTextCheck,
-    verifyMinimumEntries
-} from "../testHelpers/TrafficHelper";
-const refreshWaitTimeout = 10000;
-
-it('opening mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-verifyMinimumEntries();
-
-it('top bar check', function () {
-    const podName1 = 'httpbin', namespace1 = 'mizu-tests';
-    const podName2 = 'httpbin2', namespace2 = 'mizu-tests';
-
-    cy.get('.podsCount').trigger('mouseover');
-    findLineAndCheck(getExpectedDetailsDict(podName1, namespace1));
-    findLineAndCheck(getExpectedDetailsDict(podName2, namespace2));
-    cy.reload();
-});
-
-it('filtering guide check', function () {
-    cy.get('[title="Open Filtering Guide (Cheatsheet)"]').click();
-    cy.get('#modal-modal-title').should('be.visible');
-    cy.get('[lang="en"]').click(0, 0);
-    cy.get('#modal-modal-title').should('not.exist');
-});
-
-it('right side sanity test', function () {
-    cy.get('#entryDetailedTitleBodySize').then(sizeTopLine => {
-        const sizeOnTopLine = sizeTopLine.text().replace(' B', '');
-        cy.contains('Response').click();
-        cy.contains('Body Size (bytes)').parent().next().then(size => {
-            const bodySizeByDetails = size.text();
-            expect(sizeOnTopLine).to.equal(bodySizeByDetails, 'The body size in the top line should match the details in the response');
-
-            if (parseInt(bodySizeByDetails) < 0) {
-                throw new Error(`The body size cannot be negative. got the size: ${bodySizeByDetails}`)
-            }
-
-            cy.get('#entryDetailedTitleElapsedTime').then(timeInMs => {
-                const time = timeInMs.text();
-                if (time < '0ms') {
-                    throw new Error(`The time in the top line cannot be negative ${time}`);
-                }
-
-                cy.get('#rightSideContainer [title="Status Code"]').then(status => {
-                    const statusCode = status.text();
-                    cy.contains('Status').parent().next().then(statusInDetails => {
-                        const statusCodeInDetails = statusInDetails.text();
-
-                        expect(statusCode).to.equal(statusCodeInDetails, 'The status code in the top line should match the status code in details');
-                    });
-                });
-            });
-        });
-    });
-});
-
-checkIllegalFilter('invalid filter');
-
-checkFilter({
-    name: 'http',
-    leftSidePath: '> :nth-child(1) > :nth-child(1)',
-    leftSideExpectedText: 'HTTP',
-    rightSidePath: '[title=HTTP]',
-    rightSideExpectedText: 'Hypertext Transfer Protocol -- HTTP/1.1',
-    applyByEnter: true
-});
-
-checkFilter({
-    name: 'response.status == 200',
-    leftSidePath: '[title="Status Code"]',
-    leftSideExpectedText: '200',
-    rightSidePath: '> :nth-child(2) [title="Status Code"]',
-    rightSideExpectedText: '200',
-    applyByEnter: false
-});
-
-checkFilter({
-    name: 'src.name == ""',
-    leftSidePath: '[title="Source Name"]',
-    leftSideExpectedText: '[Unresolved]',
-    rightSidePath: '> :nth-child(2) [title="Source Name"]',
-    rightSideExpectedText: '[Unresolved]',
-    applyByEnter: false
-});
-
-checkFilter({
-    name: 'method == "GET"',
-    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
-    leftSideExpectedText: 'GET',
-    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(1) > :nth-child(2)',
-    rightSideExpectedText: 'GET',
-    applyByEnter: true
-});
-
-checkFilter({
-    name: 'summary == "/get"',
-    leftSidePath: '> :nth-child(3) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
-    leftSideExpectedText: '/get',
-    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(1) > :nth-child(2) > :nth-child(2)',
-    rightSideExpectedText: '/get',
-    applyByEnter: false
-});
-
-checkFilter({
-    name: 'dst.name == "httpbin.mizu-tests"',
-    leftSidePath: '> :nth-child(3) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
-    leftSideExpectedText: 'httpbin.mizu-tests',
-    rightSidePath: '> :nth-child(2) > :nth-child(2) > :nth-child(2) > :nth-child(3) > :nth-child(2)',
-    rightSideExpectedText: 'httpbin.mizu-tests',
-    applyByEnter: false
-});
-
-checkFilter({
-    name: 'src.ip == "127.0.0.1"',
-    leftSidePath: '[title="Source IP"]',
-    leftSideExpectedText: '127.0.0.1',
-    rightSidePath: '> :nth-child(2) [title="Source IP"]',
-    rightSideExpectedText: '127.0.0.1',
-    applyByEnter: false
-});
-
-checkFilterNoResults('method == "POST"');
-
-function checkFilterNoResults(filterName) {
-    it(`checking the filter: ${filterName}. Expecting no results`, function () {
-        cy.get('#total-entries').then(number => {
-            const totalEntries = number.text();
-
-            // applying the filter
-            cy.get('.w-tc-editor-text').type(filterName);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
-            cy.get('[type="submit"]').click();
-
-            // waiting for the entries number to load
-            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-
-            // the DOM should show 0 entries
-            cy.get('#entries-length').should('have.text', '0');
-
-            // going through every potential entry and verifies that it doesn't exist
-            [...Array(parseInt(totalEntries)).keys()].map(shouldNotExist);
-
-            cy.get('[title="Fetch old records"]').click();
-            cy.get('#noMoreDataTop', {timeout: refreshWaitTimeout}).should('be.visible');
-            cy.get('#entries-length').should('have.text', '0'); // after loading all entries there should still be 0 entries
-
-            // reloading then waiting for the entries number to load
-            cy.reload();
-            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-        });
-    });
-}
-
-function shouldNotExist(entryNum) {
-    cy.get(`entry-${entryNum}`).should('not.exist');
-}
-
-function checkIllegalFilter(illegalFilterName) {
-    it(`should show red search bar with the input: ${illegalFilterName}`, function () {
-        cy.get('#total-entries').then(number => {
-            const totalEntries = number.text();
-
-            cy.get('.w-tc-editor-text').type(illegalFilterName);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('redFilterColor'));
-            cy.get('[type="submit"]').click();
-
-            cy.get('[role="alert"]').should('be.visible');
-            cy.get('.w-tc-editor-text').clear();
-
-            // reloading then waiting for the entries number to load
-            cy.reload();
-            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-        });
-    });
-}
-function checkFilter(filterDetails){
-    const {name, leftSidePath, rightSidePath, rightSideExpectedText, leftSideExpectedText, applyByEnter} = filterDetails;
-    const entriesForDeeperCheck = 5;
-
-    it(`checking the filter: ${name}`, function () {
-        cy.get('#total-entries').then(number => {
-            const totalEntries = number.text();
-
-            // checks the hover on the last entry (the only one in DOM at the beginning)
-            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
-
-            // applying the filter with alt+enter or with the button
-            cy.get('.w-tc-editor-text').type(`${name}${applyByEnter ? '{alt+enter}' : ''}`);
-            cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor'));
-            if (!applyByEnter)
-                cy.get('[type="submit"]').click();
-
-            // only one entry in DOM after filtering, checking all checks on it
-            leftTextCheck(totalEntries - 1, leftSidePath, leftSideExpectedText);
-            leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
-            rightTextCheck(rightSidePath, rightSideExpectedText);
-            rightOnHoverCheck(rightSidePath, name);
-            checkRightSideResponseBody();
-
-            cy.get('[title="Fetch old records"]').click();
-            resizeToHugeMizu();
-
-            // waiting for the entries number to load
-            cy.get('#entries-length', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-
-            // checking only 'leftTextCheck' on all entries because the rest of the checks require more time
-            [...Array(parseInt(totalEntries)).keys()].forEach(entryNum => {
-                leftTextCheck(entryNum, leftSidePath, leftSideExpectedText);
-            });
-
-            // making the other 3 checks on the first X entries (longer time for each check)
-            deeperChcek(leftSidePath, rightSidePath, name, leftSideExpectedText, rightSideExpectedText, entriesForDeeperCheck);
-
-            // reloading then waiting for the entries number to load
-            resizeToNormalMizu();
-            cy.reload();
-            cy.get('#total-entries', {timeout: refreshWaitTimeout}).should('have.text', totalEntries);
-        });
-    });
-}
-
-function deeperChcek(leftSidePath, rightSidePath, filterName, leftSideExpectedText, rightSideExpectedText, entriesNumToCheck) {
-    [...Array(entriesNumToCheck).keys()].forEach(entryNum => {
-        leftOnHoverCheck(entryNum, leftSidePath, filterName);
-
-        cy.get(`#list #entry-${entryNum}`).click();
-        rightTextCheck(rightSidePath, rightSideExpectedText);
-        rightOnHoverCheck(rightSidePath, filterName);
-    });
-}
-
-function checkRightSideResponseBody() {
-    cy.contains('Response').click();
-    clickCheckbox('Decode Base64');
-
-    cy.get(`${Cypress.env('bodyJsonClass')}`).then(value => {
-        const encodedBody = value.text();
-        const decodedBody = atob(encodedBody);
-        const responseBody = JSON.parse(decodedBody);
-
-        const expectdJsonBody = {
-            args: RegExp({}),
-            url: RegExp('http://.*/get'),
-            headers: {
-                "User-Agent": RegExp('[REDACTED]'),
-                "Accept-Encoding": RegExp('gzip'),
-                "X-Forwarded-Uri": RegExp('/api/v1/namespaces/.*/services/.*/proxy/get')
-            }
-        };
-
-        expect(responseBody.args).to.match(expectdJsonBody.args);
-        expect(responseBody.url).to.match(expectdJsonBody.url);
-        expect(responseBody.headers['User-Agent']).to.match(expectdJsonBody.headers['User-Agent']);
-        expect(responseBody.headers['Accept-Encoding']).to.match(expectdJsonBody.headers['Accept-Encoding']);
-        expect(responseBody.headers['X-Forwarded-Uri']).to.match(expectdJsonBody.headers['X-Forwarded-Uri']);
-
-        cy.get(`${Cypress.env('bodyJsonClass')}`).should('have.text', encodedBody);
-        clickCheckbox('Decode Base64');
-
-        cy.get(`${Cypress.env('bodyJsonClass')} > `).its('length').should('be.gt', 1).then(linesNum => {
-            cy.get(`${Cypress.env('bodyJsonClass')} > >`).its('length').should('be.gt', linesNum).then(jsonItemsNum => {
-                checkPrettyAndLineNums(jsonItemsNum, decodedBody);
-
-                clickCheckbox('Line numbers');
-                checkPrettyOrNothing(jsonItemsNum, decodedBody);
-
-                clickCheckbox('Pretty');
-                checkPrettyOrNothing(jsonItemsNum, decodedBody);
-
-                clickCheckbox('Line numbers');
-                checkOnlyLineNumberes(jsonItemsNum, decodedBody);
-            });
-        });
-    });
-}
-
-function clickCheckbox(type) {
-    cy.contains(`${type}`).prev().children().click();
-}
-
-function checkPrettyAndLineNums(jsonItemsLen, decodedBody) {
-    decodedBody = decodedBody.replaceAll(' ', '');
-    cy.get(`${Cypress.env('bodyJsonClass')} >`).then(elements => {
-        const lines = Object.values(elements);
-        lines.forEach((line, index) => {
-            if (line.getAttribute) {
-                const cleanLine = getCleanLine(line);
-                const currentLineFromDecodedText = decodedBody.substring(0, cleanLine.length);
-
-                expect(cleanLine).to.equal(currentLineFromDecodedText, `expected the text in line number ${index + 1} to match the text that generated by the base64 decoding`)
-
-                decodedBody = decodedBody.substring(cleanLine.length);
-            }
-        });
-    });
-}
-
-function getCleanLine(lineElement) {
-    return (lineElement.innerText.substring(0, lineElement.innerText.length - 1)).replaceAll(' ', '');
-}
-
-function checkPrettyOrNothing(jsonItems, decodedBody) {
-    cy.get(`${Cypress.env('bodyJsonClass')} > `).should('have.length', jsonItems).then(text => {
-        const json = text.text();
-        expect(json).to.equal(decodedBody);
-    });
-}
-
-function checkOnlyLineNumberes(jsonItems, decodedText) {
-    cy.get(`${Cypress.env('bodyJsonClass')} >`).should('have.length', 1).and('have.text', decodedText);
-    cy.get(`${Cypress.env('bodyJsonClass')} > >`).should('have.length', jsonItems)
-}

acceptanceTests/extensions_test.go

@@ -1,101 +0,0 @@
-package acceptanceTests
-
-import (
-	"context"
-	"fmt"
-	"github.com/go-redis/redis/v8"
-	"os/exec"
-	"testing"
-)
-
-func TestRedis(t *testing.T) {
-	if testing.Short() {
-		t.Skip("ignored acceptance test")
-	}
-
-	cliPath, cliPathErr := getCliPath()
-	if cliPathErr != nil {
-		t.Errorf("failed to get cli path, err: %v", cliPathErr)
-		return
-	}
-
-	tapCmdArgs := getDefaultTapCommandArgs()
-
-	tapNamespace := getDefaultTapNamespace()
-	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-
-	tapCmd := exec.Command(cliPath, tapCmdArgs...)
-	t.Logf("running command: %v", tapCmd.String())
-
-	t.Cleanup(func() {
-		if err := cleanupCommand(tapCmd); err != nil {
-			t.Logf("failed to cleanup tap command, err: %v", err)
-		}
-	})
-
-	if err := tapCmd.Start(); err != nil {
-		t.Errorf("failed to start tap command, err: %v", err)
-		return
-	}
-
-	apiServerUrl := getApiServerUrl(defaultApiServerPort)
-
-	if err := waitTapPodsReady(apiServerUrl); err != nil {
-		t.Errorf("failed to start tap pods on time, err: %v", err)
-		return
-	}
-
-	ctx := context.Background()
-
-	redisExternalIp, err := getServiceExternalIp(ctx, defaultNamespaceName, "redis")
-	if err != nil {
-		t.Errorf("failed to get redis external ip, err: %v", err)
-		return
-	}
-
-	rdb := redis.NewClient(&redis.Options{
-		Addr: fmt.Sprintf("%v:6379", redisExternalIp),
-	})
-
-	for i := 0; i < defaultEntriesCount/5; i++ {
-		requestErr := rdb.Ping(ctx).Err()
-		if requestErr != nil {
-			t.Errorf("failed to send redis request, err: %v", requestErr)
-			return
-		}
-	}
-
-	for i := 0; i < defaultEntriesCount/5; i++ {
-		requestErr := rdb.Set(ctx, "key", "value", -1).Err()
-		if requestErr != nil {
-			t.Errorf("failed to send redis request, err: %v", requestErr)
-			return
-		}
-	}
-
-	for i := 0; i < defaultEntriesCount/5; i++ {
-		requestErr := rdb.Exists(ctx, "key").Err()
-		if requestErr != nil {
-			t.Errorf("failed to send redis request, err: %v", requestErr)
-			return
-		}
-	}
-
-	for i := 0; i < defaultEntriesCount/5; i++ {
-		requestErr := rdb.Get(ctx, "key").Err()
-		if requestErr != nil {
-			t.Errorf("failed to send redis request, err: %v", requestErr)
-			return
-		}
-	}
-
-	for i := 0; i < defaultEntriesCount/5; i++ {
-		requestErr := rdb.Del(ctx, "key").Err()
-		if requestErr != nil {
-			t.Errorf("failed to send redis request, err: %v", requestErr)
-			return
-		}
-	}
-
-	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/Redis.js\"")
-}

acceptanceTests/go.mod

@@ -1,49 +1,10 @@
 module github.com/up9inc/mizu/tests
 
-go 1.17
+go 1.16
 
 require (
-	github.com/go-redis/redis/v8 v8.11.4
 	github.com/up9inc/mizu/shared v0.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
-	k8s.io/apimachinery v0.23.3
-	k8s.io/client-go v0.23.3
-)
-
-require (
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/go-logr/logr v1.2.2 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.7 // indirect
-	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/api v0.23.3 // indirect
-	k8s.io/klog/v2 v2.40.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf // indirect
-	k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
-	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared

acceptanceTests/go.sum

@@ -9,44 +9,26 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T
 cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
 cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
 cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
-cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
-cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
-cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
-cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
-github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
 github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
-github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
 github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -75,19 +57,13 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -99,12 +75,9 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -113,25 +86,17 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
-github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
@@ -146,9 +111,6 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs=
-github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
 github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
 github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
@@ -161,7 +123,6 @@ github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwds
 github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
@@ -191,38 +152,28 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
 github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
 github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
-github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg=
-github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0=
 github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU=
-github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -231,46 +182,25 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
-github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
-github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -279,11 +209,7 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
-github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
-github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
-github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -312,16 +238,11 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
-github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
-github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
@@ -366,37 +287,22 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
-github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
-github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
-github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c=
-github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -406,7 +312,6 @@ github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -450,21 +355,16 @@ github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJ
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -473,11 +373,8 @@ github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
@@ -486,9 +383,6 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -505,7 +399,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -528,7 +421,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
@@ -538,9 +430,6 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -560,7 +449,6 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -570,52 +458,21 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg=
-golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -634,12 +491,10 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -650,58 +505,25 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220207234003-57398862261d h1:Bm7BNOQt2Qv7ZqysjeLjgCBanX+88Z/OtdvsrEv1Djc=
-golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M=
-golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -738,32 +560,14 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
-golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
-golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
@@ -774,26 +578,12 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
-google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
-google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
-google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
-google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -811,48 +601,16 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx
 google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
-google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -861,12 +619,7 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -875,11 +628,9 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8X
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -888,11 +639,8 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
@@ -903,54 +651,29 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU=
-k8s.io/api v0.23.3 h1:KNrME8KHGr12Ozjf8ytOewKzZh6hl/hHUZeHddT3a38=
-k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ=
 k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM=
-k8s.io/apimachinery v0.23.3 h1:7IW6jxNzrXTsP0c8yXz2E5Yx/WTzVPTsHIx/2Vm0cIk=
-k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
 k8s.io/cli-runtime v0.21.2/go.mod h1:8u/jFcM0QpoI28f6sfrAAIslLCXUYKD5SsPPMWiHYrI=
 k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA=
-k8s.io/client-go v0.23.3 h1:23QYUmCQ/W6hW78xIwm3XqZrrKZM+LWDqW2zfo+szJs=
-k8s.io/client-go v0.23.3/go.mod h1:47oMd+YvAOqZM7pcQ6neJtBiFH7alOyfunYN48VsmwE=
 k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U=
 k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc=
 k8s.io/component-helpers v0.21.2/go.mod h1:DbyFt/A0p6Cv+R5+QOGSJ5f5t4xDfI8Yb89a57DgJlQ=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
-k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/klog/v2 v2.40.1 h1:P4RRucWk/lFOlDdkAr3mc7iWFkgKrZY9qZMAgek06S4=
-k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
-k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
-k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf h1:M9XBsiMslw2lb2ZzglC0TOkBPK5NQi0/noUrdnoFwUg=
-k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
 k8s.io/kubectl v0.21.2/go.mod h1:PgeUclpG8VVmmQIl8zpLar3IQEpFc9mrmvlwY3CK1xo=
 k8s.io/metrics v0.21.2/go.mod h1:wzlOINZMCtWq8dR9gHlyaOemmYlOpAoldEIXE82gAhI=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20220127004650-9b3446523e65 h1:ONWS0Wgdg5wRiQIAui7L/023aC9+IxrIrydY7l8llsE=
-k8s.io/utils v0.0.0-20220127004650-9b3446523e65/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
-sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y=
-sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY=
 sigs.k8s.io/kustomize/api v0.8.8/go.mod h1:He1zoK0nk43Pc6NlV085xDXDXTNprtcyKZVm3swsdNY=
 sigs.k8s.io/kustomize/cmd/config v0.9.10/go.mod h1:Mrby0WnRH7hA6OwOYnYpfpiY0WJIMgYrEDfwOeFdMK0=
 sigs.k8s.io/kustomize/kustomize/v4 v4.1.2/go.mod h1:PxBvo4WGYlCLeRPL+ziT64wBXqbgfcalOS/SXa/tcyo=
 sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4cPcfUVt5Hg=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

acceptanceTests/logs_test.go

@@ -81,16 +81,11 @@ func TestLogs(t *testing.T) {
 		logsFileNames = append(logsFileNames, file.Name)
 	}
 
-	if !Contains(logsFileNames, "mizu.mizu-api-server.mizu-api-server.log") {
+	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
 		t.Errorf("api server logs not found")
 		return
 	}
 
-	if !Contains(logsFileNames, "mizu.mizu-api-server.basenine.log") {
-		t.Errorf("basenine logs not found")
-		return
-	}
-
 	if !Contains(logsFileNames, "mizu_cli.log") {
 		t.Errorf("cli logs not found")
 		return
@@ -179,16 +174,11 @@ func TestLogsPath(t *testing.T) {
 		logsFileNames = append(logsFileNames, file.Name)
 	}
 
-	if !Contains(logsFileNames, "mizu.mizu-api-server.mizu-api-server.log") {
+	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
 		t.Errorf("api server logs not found")
 		return
 	}
 
-	if !Contains(logsFileNames, "mizu.mizu-api-server.basenine.log") {
-		t.Errorf("basenine logs not found")
-		return
-	}
-
 	if !Contains(logsFileNames, "mizu_cli.log") {
 		t.Errorf("cli logs not found")
 		return

acceptanceTests/setup.sh

@@ -36,24 +36,15 @@ kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests
 
 kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2
 
-echo "Creating redis deployment"
-kubectl create deployment redis --image=redis -n mizu-tests
-
 echo "Creating httpbin services"
 kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
 kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests
 
 kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2
 
-echo "Creating redis service"
-kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests
-
 echo "Starting proxy"
 kubectl proxy --port=8080 &
 
-echo "Starting tunnel"
-minikube tunnel &
-
 echo "Setting minikube docker env"
 eval $(minikube docker-env)
 

acceptanceTests/tap_test.go

@@ -3,6 +3,7 @@ package acceptanceTests
 import (
 	"archive/zip"
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -62,7 +63,38 @@ func TestTap(t *testing.T) {
 				}
 			}
 
-			runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/UiTest.js\"")
+			entriesCheckFunc := func() error {
+				timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+				entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, entriesCount, timestamp)
+				requestResult, requestErr := executeHttpGetRequest(entriesUrl)
+				if requestErr != nil {
+					return fmt.Errorf("failed to get entries, err: %v", requestErr)
+				}
+
+				entries := requestResult.([]interface{})
+				if len(entries) == 0 {
+					return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+				}
+
+				entry :=  entries[0].(map[string]interface{})
+
+				entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"])
+				requestResult, requestErr = executeHttpGetRequest(entryUrl)
+				if requestErr != nil {
+					return fmt.Errorf("failed to get entry, err: %v", requestErr)
+				}
+
+				if requestResult == nil {
+					return fmt.Errorf("unexpected nil entry result")
+				}
+
+				return nil
+			}
+			if err := retriesExecute(shortRetriesCount, entriesCheckFunc); err != nil {
+				t.Errorf("%v", err)
+				return
+			}
 		})
 	}
 }
@@ -109,17 +141,6 @@ func TestTapGuiPort(t *testing.T) {
 				t.Errorf("failed to start tap pods on time, err: %v", err)
 				return
 			}
-
-			proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
-			for i := 0; i < defaultEntriesCount; i++ {
-				if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil {
-					t.Errorf("failed to send proxy request, err: %v", requestErr)
-					return
-				}
-			}
-
-			runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env name=%v,namespace=%v,port=%d",
-				"httpbin", "mizu-tests", guiPort))
 		})
 	}
 }
@@ -129,9 +150,11 @@ func TestTapAllNamespaces(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	expectedPods := []PodDescriptor{
+	expectedPods := []struct{
+		Name      string
+		Namespace string
+	}{
 		{Name: "httpbin", Namespace: "mizu-tests"},
-		{Name: "httpbin2", Namespace: "mizu-tests"},
 		{Name: "httpbin", Namespace: "mizu-tests2"},
 	}
 
@@ -165,8 +188,37 @@ func TestTapAllNamespaces(t *testing.T) {
 		return
 	}
 
-	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
-		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
+	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
+	requestResult, requestErr := executeHttpGetRequest(podsUrl)
+	if requestErr != nil {
+		t.Errorf("failed to get tap status, err: %v", requestErr)
+		return
+	}
+
+	pods, err := getPods(requestResult)
+	if err != nil {
+		t.Errorf("failed to get pods, err: %v", err)
+		return
+	}
+
+	for _, expectedPod := range expectedPods {
+		podFound := false
+
+		for _, pod := range pods {
+			podNamespace :=  pod["namespace"].(string)
+			podName := pod["name"].(string)
+
+			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
+				podFound = true
+				break
+			}
+		}
+
+		if !podFound {
+			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
+			return
+		}
+	}
 }
 
 func TestTapMultipleNamespaces(t *testing.T) {
@@ -174,7 +226,10 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		t.Skip("ignored acceptance test")
 	}
 
-	expectedPods := []PodDescriptor{
+	expectedPods := []struct{
+		Name      string
+		Namespace string
+	}{
 		{Name: "httpbin", Namespace: "mizu-tests"},
 		{Name: "httpbin2", Namespace: "mizu-tests"},
 		{Name: "httpbin", Namespace: "mizu-tests2"},
@@ -214,8 +269,42 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		return
 	}
 
-	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/MultipleNamespaces.js\" --env name1=%v,name2=%v,name3=%v,namespace1=%v,namespace2=%v,namespace3=%v",
-		expectedPods[0].Name, expectedPods[1].Name, expectedPods[2].Name, expectedPods[0].Namespace, expectedPods[1].Namespace, expectedPods[2].Namespace))
+	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
+	requestResult, requestErr := executeHttpGetRequest(podsUrl)
+	if requestErr != nil {
+		t.Errorf("failed to get tap status, err: %v", requestErr)
+		return
+	}
+
+	pods, err := getPods(requestResult)
+	if err != nil {
+		t.Errorf("failed to get pods, err: %v", err)
+		return
+	}
+
+	if len(expectedPods) != len(pods) {
+		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
+		return
+	}
+
+	for _, expectedPod := range expectedPods {
+		podFound := false
+
+		for _, pod := range pods {
+			podNamespace :=  pod["namespace"].(string)
+			podName := pod["name"].(string)
+
+			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
+				podFound = true
+				break
+			}
+		}
+
+		if !podFound {
+			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
+			return
+		}
+	}
 }
 
 func TestTapRegex(t *testing.T) {
@@ -224,7 +313,10 @@ func TestTapRegex(t *testing.T) {
 	}
 
 	regexPodName := "httpbin2"
-	expectedPods := []PodDescriptor{
+	expectedPods := []struct{
+		Name      string
+		Namespace string
+	}{
 		{Name: regexPodName, Namespace: "mizu-tests"},
 	}
 
@@ -260,8 +352,42 @@ func TestTapRegex(t *testing.T) {
 		return
 	}
 
-	runCypressTests(t, fmt.Sprintf("npx cypress run --spec  \"cypress/integration/tests/Regex.js\" --env name=%v,namespace=%v",
-		expectedPods[0].Name, expectedPods[0].Namespace))
+	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
+	requestResult, requestErr := executeHttpGetRequest(podsUrl)
+	if requestErr != nil {
+		t.Errorf("failed to get tap status, err: %v", requestErr)
+		return
+	}
+
+	pods, err := getPods(requestResult)
+	if err != nil {
+		t.Errorf("failed to get pods, err: %v", err)
+		return
+	}
+
+	if len(expectedPods) != len(pods) {
+		t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods))
+		return
+	}
+
+	for _, expectedPod := range expectedPods {
+		podFound := false
+
+		for _, pod := range pods {
+			podNamespace :=  pod["namespace"].(string)
+			podName := pod["name"].(string)
+
+			if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
+				podFound = true
+				break
+			}
+		}
+
+		if !podFound {
+			t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
+			return
+		}
+	}
 }
 
 func TestTapDryRun(t *testing.T) {
@@ -305,7 +431,7 @@ func TestTapDryRun(t *testing.T) {
 		resultChannel <- "fail"
 	}()
 
-	testResult := <-resultChannel
+	testResult := <- resultChannel
 	if testResult != "success" {
 		t.Errorf("unexpected result - dry run cmd not done")
 	}
@@ -349,16 +475,79 @@ func TestTapRedact(t *testing.T) {
 	}
 
 	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
-	requestHeaders := map[string]string{"User-Header": "Mizu"}
 	requestBody := map[string]string{"User": "Mizu"}
 	for i := 0; i < defaultEntriesCount; i++ {
-		if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
+		if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/Redact.js\"")
+	redactCheckFunc := func() error {
+		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
+		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		}
+
+		entries := requestResult.([]interface{})
+		if len(entries) == 0 {
+			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		}
+
+		firstEntry :=  entries[0].(map[string]interface{})
+
+		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
+		requestResult, requestErr = executeHttpGetRequest(entryUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entry, err: %v", requestErr)
+		}
+
+		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+		entryJson := data["entry"].(string)
+
+		var entry map[string]interface{}
+		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
+			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
+		}
+
+		entryRequest := entry["request"].(map[string]interface{})
+		entryPayload := entryRequest["payload"].(map[string]interface{})
+		entryDetails := entryPayload["details"].(map[string]interface{})
+
+		headers :=  entryDetails["headers"].([]interface{})
+		for _, headerInterface := range headers {
+			header := headerInterface.(map[string]interface{})
+			if header["name"].(string) != "User-Agent" {
+				continue
+			}
+
+			userAgent := header["value"].(string)
+			if userAgent != "[REDACTED]" {
+				return fmt.Errorf("unexpected result - user agent is not redacted")
+			}
+		}
+
+		postData := entryDetails["postData"].(map[string]interface{})
+		textDataStr := postData["text"].(string)
+
+		var textData map[string]string
+		if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
+			return fmt.Errorf("failed to parse text data, err: %v", parseErr)
+		}
+
+		if textData["User"] != "[REDACTED]" {
+			return fmt.Errorf("unexpected result - user in body is not redacted")
+		}
+
+		return nil
+	}
+	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
+		t.Errorf("%v", err)
+		return
+	}
 }
 
 func TestTapNoRedact(t *testing.T) {
@@ -401,16 +590,79 @@ func TestTapNoRedact(t *testing.T) {
 	}
 
 	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
-	requestHeaders := map[string]string{"User-Header": "Mizu"}
 	requestBody := map[string]string{"User": "Mizu"}
 	for i := 0; i < defaultEntriesCount; i++ {
-		if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
+		if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
 			return
 		}
 	}
 
-	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/NoRedact.js\"")
+	redactCheckFunc := func() error {
+		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
+		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		}
+
+		entries := requestResult.([]interface{})
+		if len(entries) == 0 {
+			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		}
+
+		firstEntry :=  entries[0].(map[string]interface{})
+
+		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
+		requestResult, requestErr = executeHttpGetRequest(entryUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entry, err: %v", requestErr)
+		}
+
+		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+		entryJson := data["entry"].(string)
+
+		var entry map[string]interface{}
+		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
+			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
+		}
+
+		entryRequest := entry["request"].(map[string]interface{})
+		entryPayload := entryRequest["payload"].(map[string]interface{})
+		entryDetails := entryPayload["details"].(map[string]interface{})
+
+		headers :=  entryDetails["headers"].([]interface{})
+		for _, headerInterface := range headers {
+			header := headerInterface.(map[string]interface{})
+			if header["name"].(string) != "User-Agent" {
+				continue
+			}
+
+			userAgent := header["value"].(string)
+			if userAgent == "[REDACTED]" {
+				return fmt.Errorf("unexpected result - user agent is redacted")
+			}
+		}
+
+		postData := entryDetails["postData"].(map[string]interface{})
+		textDataStr := postData["text"].(string)
+
+		var textData map[string]string
+		if parseErr := json.Unmarshal([]byte(textDataStr), &textData); parseErr != nil {
+			return fmt.Errorf("failed to parse text data, err: %v", parseErr)
+		}
+
+		if textData["User"] == "[REDACTED]" {
+			return fmt.Errorf("unexpected result - user in body is redacted")
+		}
+
+		return nil
+	}
+	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
+		t.Errorf("%v", err)
+		return
+	}
 }
 
 func TestTapRegexMasking(t *testing.T) {
@@ -461,8 +713,53 @@ func TestTapRegexMasking(t *testing.T) {
 		}
 	}
 
-	runCypressTests(t, "npx cypress run --spec \"cypress/integration/tests/RegexMasking.js\"")
+	redactCheckFunc := func() error {
+		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
+		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
+		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		}
+
+		entries := requestResult.([]interface{})
+		if len(entries) == 0 {
+			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		}
+
+		firstEntry :=  entries[0].(map[string]interface{})
+
+		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
+		requestResult, requestErr = executeHttpGetRequest(entryUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entry, err: %v", requestErr)
+		}
+
+		data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+		entryJson := data["entry"].(string)
+
+		var entry map[string]interface{}
+		if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
+			return fmt.Errorf("failed to parse entry, err: %v", parseErr)
+		}
+
+		entryRequest := entry["request"].(map[string]interface{})
+		entryPayload := entryRequest["payload"].(map[string]interface{})
+		entryDetails := entryPayload["details"].(map[string]interface{})
+
+		postData := entryDetails["postData"].(map[string]interface{})
+		textData := postData["text"].(string)
+
+		if textData != "[REDACTED]" {
+			return fmt.Errorf("unexpected result - body is not redacted")
+		}
+
+		return nil
+	}
+	if err := retriesExecute(shortRetriesCount, redactCheckFunc); err != nil {
+		t.Errorf("%v", err)
+		return
+	}
 }
 
 func TestTapIgnoredUserAgents(t *testing.T) {
@@ -508,7 +805,7 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 	proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
 
 	ignoredUserAgentCustomHeader := "Ignored-User-Agent"
-	headers := map[string]string{"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
+	headers := map[string]string {"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
 	for i := 0; i < defaultEntriesCount; i++ {
 		if _, requestErr := executeHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil {
 			t.Errorf("failed to send proxy request, err: %v", requestErr)
@@ -523,7 +820,56 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 		}
 	}
 
-	runCypressTests(t, "npx cypress run --spec  \"cypress/integration/tests/IgnoredUserAgents.js\"")
+	ignoredUserAgentsCheckFunc := func() error {
+		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount * 2, timestamp)
+		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
+		if requestErr != nil {
+			return fmt.Errorf("failed to get entries, err: %v", requestErr)
+		}
+
+		entries := requestResult.([]interface{})
+		if len(entries) == 0 {
+			return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
+		}
+
+		for _, entryInterface := range entries {
+			entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface.(map[string]interface{})["id"])
+			requestResult, requestErr = executeHttpGetRequest(entryUrl)
+			if requestErr != nil {
+				return fmt.Errorf("failed to get entry, err: %v", requestErr)
+			}
+
+			data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
+			entryJson := data["entry"].(string)
+
+			var entry map[string]interface{}
+			if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
+				return fmt.Errorf("failed to parse entry, err: %v", parseErr)
+			}
+
+			entryRequest := entry["request"].(map[string]interface{})
+			entryPayload := entryRequest["payload"].(map[string]interface{})
+			entryDetails := entryPayload["details"].(map[string]interface{})
+
+			entryHeaders :=  entryDetails["headers"].([]interface{})
+			for _, headerInterface := range entryHeaders {
+				header := headerInterface.(map[string]interface{})
+				if header["name"].(string) != ignoredUserAgentCustomHeader {
+					continue
+				}
+
+				return fmt.Errorf("unexpected result - user agent is not ignored")
+			}
+		}
+
+		return nil
+	}
+	if err := retriesExecute(shortRetriesCount, ignoredUserAgentsCheckFunc); err != nil {
+		t.Errorf("%v", err)
+		return
+	}
 }
 
 func TestTapDumpLogs(t *testing.T) {
@@ -576,21 +922,21 @@ func TestTapDumpLogs(t *testing.T) {
 		return
 	}
 
-	var dumpLogsPath string
+	var dumpsLogsPath string
 	for _, file := range files {
 		fileName := file.Name()
 		if strings.Contains(fileName, "mizu_logs") {
-			dumpLogsPath = path.Join(mizuFolderPath, fileName)
+			dumpsLogsPath = path.Join(mizuFolderPath, fileName)
 			break
 		}
 	}
 
-	if dumpLogsPath == "" {
+	if dumpsLogsPath == "" {
 		t.Errorf("dump logs file not found")
 		return
 	}
 
-	zipReader, zipError := zip.OpenReader(dumpLogsPath)
+	zipReader, zipError := zip.OpenReader(dumpsLogsPath)
 	if zipError != nil {
 		t.Errorf("failed to get zip reader, err: %v", zipError)
 		return
@@ -607,16 +953,11 @@ func TestTapDumpLogs(t *testing.T) {
 		logsFileNames = append(logsFileNames, file.Name)
 	}
 
-	if !Contains(logsFileNames, "mizu.mizu-api-server.mizu-api-server.log") {
+	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
 		t.Errorf("api server logs not found")
 		return
 	}
 
-	if !Contains(logsFileNames, "mizu.mizu-api-server.basenine.log") {
-		t.Errorf("basenine logs not found")
-		return
-	}
-
 	if !Contains(logsFileNames, "mizu_cli.log") {
 		t.Errorf("cli logs not found")
 		return

acceptanceTests/testsUtils.go

@@ -2,22 +2,15 @@ package acceptanceTests
 
 import (
 	"bytes"
-	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/util/homedir"
 	"net/http"
 	"os"
 	"os/exec"
 	"path"
-	"path/filepath"
 	"strings"
 	"syscall"
-	"testing"
 	"time"
 
 	"github.com/up9inc/mizu/shared"
@@ -33,11 +26,6 @@ const (
 	waitAfterTapPodsReady = 3 * time.Second
 )
 
-type PodDescriptor struct {
-	Name      string
-	Namespace string
-}
-
 func getCliPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {
@@ -71,46 +59,16 @@ func getProxyUrl(namespace string, service string) string {
 }
 
 func getApiServerUrl(port uint16) string {
-	return fmt.Sprintf("http://localhost:%v", port)
-}
-
-func getServiceExternalIp(ctx context.Context, namespace string, service string) (string, error) {
-	home := homedir.HomeDir()
-	configLoadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(home, ".kube", "config")}
-	clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
-		configLoadingRules,
-		&clientcmd.ConfigOverrides{
-			CurrentContext: "",
-		},
-	)
-
-	restClientConfig, err := clientConfig.ClientConfig()
-	if err != nil {
-		return "", err
-	}
-
-	clientSet, err := kubernetes.NewForConfig(restClientConfig)
-	if err != nil {
-		return "", err
-	}
-
-	serviceObj, err := clientSet.CoreV1().Services(namespace).Get(ctx, service, metav1.GetOptions{})
-	if err != nil {
-		return "", err
-	}
-
-	externalIp := serviceObj.Status.LoadBalancer.Ingress[0].IP
-	return externalIp, nil
+	return fmt.Sprintf("http://localhost:%v/mizu", port)
 }
 
 func getDefaultCommandArgs() []string {
 	setFlag := "--set"
 	telemetry := "telemetry=false"
-	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0"
-	imagePullPolicy := "image-pull-policy=IfNotPresent"
-	headless := "headless=true"
+	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
+	imagePullPolicy := "image-pull-policy=Never"
 
-	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
+	return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy}
 }
 
 func getDefaultTapCommandArgs() []string {
@@ -145,17 +103,6 @@ func getDefaultConfigCommandArgs() []string {
 	return append([]string{configCommand}, defaultCmdArgs...)
 }
 
-func runCypressTests(t *testing.T, cypressRunCmd string) {
-	cypressCmd := exec.Command("bash", "-c", cypressRunCmd)
-	t.Logf("running command: %v", cypressCmd.String())
-	out, err := cypressCmd.Output()
-	if err != nil {
-		t.Errorf("%s", out)
-		return
-	}
-	t.Logf("%s", out)
-}
-
 func retriesExecute(retriesCount int, executeFunc func() error) error {
 	var lastError interface{}
 
@@ -184,16 +131,16 @@ func tryExecuteFunc(executeFunc func() error) (err interface{}) {
 }
 
 func waitTapPodsReady(apiServerUrl string) error {
-	resolvingUrl := fmt.Sprintf("%v/status/connectedTappersCount", apiServerUrl)
+	resolvingUrl := fmt.Sprintf("%v/status/tappersCount", apiServerUrl)
 	tapPodsReadyFunc := func() error {
 		requestResult, requestErr := executeHttpGetRequest(resolvingUrl)
 		if requestErr != nil {
 			return requestErr
 		}
 
-		connectedTappersCount := requestResult.(float64)
-		if connectedTappersCount == 0 {
-			return fmt.Errorf("no connected tappers running")
+		tappersCount := requestResult.(float64)
+		if tappersCount == 0 {
+			return fmt.Errorf("no tappers running")
 		}
 		time.Sleep(waitAfterTapPodsReady)
 		return nil
@@ -248,24 +195,13 @@ func executeHttpGetRequest(url string) (interface{}, error) {
 	return executeHttpRequest(response, requestErr)
 }
 
-func executeHttpPostRequestWithHeaders(url string, headers map[string]string, body interface{}) (interface{}, error) {
+func executeHttpPostRequest(url string, body interface{}) (interface{}, error) {
 	requestBody, jsonErr := json.Marshal(body)
 	if jsonErr != nil {
 		return nil, jsonErr
 	}
 
-	request, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(requestBody))
-	if err != nil {
-		return nil, err
-	}
-
-	request.Header.Add("Content-Type", "application/json")
-	for headerKey, headerValue := range headers {
-		request.Header.Add(headerKey, headerValue)
-	}
-
-	client := &http.Client{}
-	response, requestErr := client.Do(request)
+	response, requestErr := http.Post(url, "application/json", bytes.NewBuffer(requestBody))
 	return executeHttpRequest(response, requestErr)
 }
 
@@ -281,6 +217,18 @@ func cleanupCommand(cmd *exec.Cmd) error {
 	return nil
 }
 
+func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) {
+	tapStatus := tapStatusInterface.(map[string]interface{})
+	podsInterface := tapStatus["pods"].([]interface{})
+
+	var pods []map[string]interface{}
+	for _, podInterface := range podsInterface {
+		pods = append(pods, podInterface.(map[string]interface{}))
+	}
+
+	return pods, nil
+}
+
 func getLogsPath() (string, error) {
 	dir, filePathErr := os.Getwd()
 	if filePathErr != nil {

agent/README.md

@@ -0,0 +1,20 @@
+# mizu agent
+Agent for MIZU (API server and tapper)
+Basic APIs:
+* /stats - retrieve statistics of collected data
+* /viewer - web ui
+
+## Remote Debugging
+### Setup remote debugging
+1. Run `go get github.com/go-delve/delve/cmd/dlv`
+2. Create a "Go Remote" run/debug configuration in Intellij, set to localhost:2345
+3. Build and push a debug image using
+   `docker build . -t  gcr.io/up9-docker-hub/mizu/debug:latest -f debug.Dockerfile && docker push gcr.io/up9-docker-hub/mizu/debug:latest`
+
+### Connecting
+1. Start mizu using the cli with the debug
+   image `mizu tap --set agent-image=gcr.io/up9-docker-hub/mizu/debug:latest {tapped_pod_name}`
+2. Forward the debug port using `kubectl port-forward -n default mizu-api-server 2345:2345`
+3. Run the run/debug configuration you've created earlier in Intellij.
+
+<small>Do note that dlv won't start the api until a debugger connects to it.</small>

agent/go.mod

@@ -1,152 +1,31 @@
-module github.com/up9inc/mizu/agent
+module mizuserver
 
-go 1.17
+go 1.16
 
 require (
-	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
-	github.com/chanced/openapi v0.0.7
-	github.com/djherbis/atime v1.1.0
-	github.com/elastic/go-elasticsearch/v7 v7.17.0
-	github.com/getkin/kin-openapi v0.89.0
+	github.com/djherbis/atime v1.0.0
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/getkin/kin-openapi v0.76.0
 	github.com/gin-contrib/static v0.0.1
-	github.com/gin-gonic/gin v1.7.7
-	github.com/go-playground/locales v0.14.0
-	github.com/go-playground/universal-translator v0.18.0
-	github.com/go-playground/validator/v10 v10.10.0
-	github.com/google/uuid v1.3.0
+	github.com/gin-gonic/gin v1.7.2
+	github.com/go-playground/locales v0.13.0
+	github.com/go-playground/universal-translator v0.17.0
+	github.com/go-playground/validator/v10 v10.5.0
+	github.com/google/martian v2.1.0+incompatible
 	github.com/gorilla/websocket v1.4.2
-	github.com/nav-inc/datetime v0.1.3
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
-	github.com/orcaman/concurrent-map v1.0.0
-	github.com/ory/keto-client-go v0.7.0-alpha.1
-	github.com/ory/kratos-client-go v0.8.2-alpha.1
+	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/stretchr/testify v1.7.0
-	github.com/up9inc/basenine/client/go v0.0.0-20220125035757-926e42208705
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
-	github.com/up9inc/mizu/tap/extensions/amqp v0.0.0
-	github.com/up9inc/mizu/tap/extensions/http v0.0.0
-	github.com/up9inc/mizu/tap/extensions/kafka v0.0.0
-	github.com/up9inc/mizu/tap/extensions/redis v0.0.0
-	github.com/wI2L/jsondiff v0.1.1
 	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
-	k8s.io/api v0.23.3
-	k8s.io/apimachinery v0.23.3
-	k8s.io/client-go v0.23.3
-)
-
-require (
-	cloud.google.com/go/compute v1.2.0 // indirect
-	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
-	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.24 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
-	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
-	github.com/Azure/go-autorest/logger v0.2.1 // indirect
-	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/MakeNowJust/heredoc v1.0.0 // indirect
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
-	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
-	github.com/beevik/etree v1.1.0 // indirect
-	github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 // indirect
-	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
-	github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
-	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
-	github.com/fatih/camelcase v1.0.0 // indirect
-	github.com/fvbommel/sortorder v1.0.2 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/go-errors/errors v1.4.2 // indirect
-	github.com/go-logr/logr v1.2.2 // indirect
-	github.com/go-openapi/analysis v0.21.2 // indirect
-	github.com/go-openapi/errors v0.20.2 // indirect
-	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.19.6 // indirect
-	github.com/go-openapi/loads v0.21.1 // indirect
-	github.com/go-openapi/runtime v0.23.0 // indirect
-	github.com/go-openapi/spec v0.20.4 // indirect
-	github.com/go-openapi/strfmt v0.21.2 // indirect
-	github.com/go-openapi/swag v0.21.1 // indirect
-	github.com/go-openapi/validate v0.20.3 // indirect
-	github.com/go-stack/stack v1.8.1 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/btree v1.0.1 // indirect
-	github.com/google/go-cmp v0.5.7 // indirect
-	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/gopacket v1.1.19 // indirect
-	github.com/google/martian v2.1.0+incompatible // indirect
-	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/josharian/intern v1.0.0 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.14.2 // indirect
-	github.com/leodido/go-urn v1.2.1 // indirect
-	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
-	github.com/mitchellh/mapstructure v1.4.3 // indirect
-	github.com/moby/spdystream v0.2.0 // indirect
-	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
-	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
-	github.com/ohler55/ojg v1.12.12 // indirect
-	github.com/oklog/ulid v1.3.1 // indirect
-	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
-	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/russross/blackfriday v1.6.0 // indirect
-	github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
-	github.com/segmentio/kafka-go v0.4.27 // indirect
-	github.com/spf13/cobra v1.3.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/tidwall/gjson v1.14.0 // indirect
-	github.com/tidwall/match v1.1.1 // indirect
-	github.com/tidwall/pretty v1.2.0 // indirect
-	github.com/tidwall/sjson v1.2.4 // indirect
-	github.com/ugorji/go/codec v1.2.6 // indirect
-	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
-	github.com/xlab/treeprint v1.1.0 // indirect
-	go.mongodb.org/mongo-driver v1.8.3 // indirect
-	go.starlark.net v0.0.0-20220203230714-bb14e151c28f // indirect
-	golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab // indirect
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/cli-runtime v0.23.3 // indirect
-	k8s.io/component-base v0.23.3 // indirect
-	k8s.io/klog/v2 v2.40.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf // indirect
-	k8s.io/kubectl v0.23.3 // indirect
-	k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
-	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
-	sigs.k8s.io/kustomize/api v0.11.1 // indirect
-	sigs.k8s.io/kustomize/kyaml v0.13.3 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	go.mongodb.org/mongo-driver v1.7.1
+	gorm.io/driver/sqlite v1.1.4
+	gorm.io/gorm v1.21.8
+	k8s.io/api v0.21.2
+	k8s.io/apimachinery v0.21.2
+	k8s.io/client-go v0.21.2
 )
 
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
@@ -154,11 +33,3 @@ replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 replace github.com/up9inc/mizu/tap v0.0.0 => ../tap
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api
-
-replace github.com/up9inc/mizu/tap/extensions/amqp v0.0.0 => ../tap/extensions/amqp
-
-replace github.com/up9inc/mizu/tap/extensions/http v0.0.0 => ../tap/extensions/http
-
-replace github.com/up9inc/mizu/tap/extensions/kafka v0.0.0 => ../tap/extensions/kafka
-
-replace github.com/up9inc/mizu/tap/extensions/redis v0.0.0 => ../tap/extensions/redis

agent/keto/Dockerfile

@@ -1,13 +0,0 @@
-FROM oryd/keto:v0.7.0-alpha.1-sqlite
-
-USER root
-
-RUN apk add sqlite
-
-RUN mkdir -p /etc/config/keto
-
-COPY ./keto.yml /etc/config/keto/keto.yml
-COPY ./start.sh /opt/start.sh
-RUN chmod +x /opt/start.sh
-
-ENTRYPOINT ["/opt/start.sh"]

agent/keto/keto.yml

@@ -1,22 +0,0 @@
-version: v0.7.0-alpha.1
-
-dsn: sqlite:///app/data/kratos.sqlite?_fk=true
-
-log:
-  level: info
-  format: text
-  leak_sensitive_values: false
-
-serve:
-  read:
-    host: 0.0.0.0
-    port: 4466
-  write:
-    host: 0.0.0.0
-    port: 4467
-
-namespaces:
-  - id: 0
-    name: system
-  - id: 1
-    name: workspaces

agent/keto/start.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-keto migrate up -c /etc/config/keto/keto.yml --yes # this initializes the db
-keto serve -c /etc/config/keto/keto.yml # start keto

agent/kratos/Dockerfile

@@ -1,14 +0,0 @@
-FROM gcr.io/up9-docker-hub/mizu-kratos-base/simple-password-policy:latest
-
-USER root
-
-RUN apk add sqlite
-
-RUN mkdir -p /etc/config/kratos
-
-COPY ./kratos.yml /etc/config/kratos/kratos.yml
-COPY ./identity.schema.json /etc/config/kratos/identity.schema.json
-COPY ./start.sh /opt/start.sh
-RUN chmod +x /opt/start.sh
-
-ENTRYPOINT ["/opt/start.sh"]

agent/kratos/build-push-featurebranch.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-set -e
-
-GCP_PROJECT=up9-docker-hub
-REPOSITORY=gcr.io/$GCP_PROJECT
-SERVER_NAME=mizu-kratos
-GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]')
-
-DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
-VER=${VER=0.0.0}
-
-DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$VER")
-
-if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ]
-then
-  echo "Pushing to $GIT_BRANCH is allowed only via CI"
-  exit 1
-fi
-
-echo "building ${DOCKER_TAGGED_BUILDS[@]}"
-DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..."
-docker build $DOCKER_TAGS_ARGS --build-arg VER=${VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
-
-for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}"
-do
-  echo pushing "$DOCKER_TAG"
-  docker push "$DOCKER_TAG"
-done

agent/kratos/identity.schema.json

@@ -1,43 +0,0 @@
-{
-	"$id": "https://schemas.ory.sh/presets/kratos/quickstart/email-password/identity.schema.json",
-	"$schema": "http://json-schema.org/draft-07/schema#",
-	"title": "Person",
-	"type": "object",
-	"properties": {
-		"traits": {
-			"type": "object",
-			"properties": {
-				"username": {
-					"type": "string",
-					"format": "username",
-					"title": "Username",
-					"minLength": 3,
-					"ory.sh/kratos": {
-						"credentials": {
-							"password": {
-								"identifier": true
-							}
-						}
-					}
-				},
-				"name": {
-					"type": "object",
-					"properties": {
-						"first": {
-							"title": "First Name",
-							"type": "string"
-						},
-						"last": {
-							"title": "Last Name",
-							"type": "string"
-						}
-					}
-				}
-			},
-			"required": [
-				"username"
-			],
-			"additionalProperties": false
-		}
-	}
-}

agent/kratos/kratos.yml

@@ -1,84 +0,0 @@
-version: v0.8.2-alpha.1
-
-dsn: sqlite:///app/data/kratos.sqlite?_fk=true
-
-serve:
-  public:
-    base_url: http://127.0.0.1:4433/
-    cors:
-      enabled: true
-  admin:
-    base_url: http://kratos:4434/
-
-selfservice:
-  default_browser_return_url: http://127.0.0.1:4455/
-  whitelisted_return_urls:
-    - http://127.0.0.1:4455
-
-  methods:
-    password:
-      enabled: true
-
-  flows:
-    error:
-      ui_url: http://127.0.0.1:4455/error
-
-    settings:
-      ui_url: http://127.0.0.1:4455/settings
-      privileged_session_max_age: 15m
-
-    recovery:
-      enabled: true
-      ui_url: http://127.0.0.1:4455/recovery
-
-    verification:
-      enabled: false
-      ui_url: http://127.0.0.1:4455/verification
-      after:
-        default_browser_return_url: http://127.0.0.1:4455/
-
-    logout:
-      after:
-        default_browser_return_url: http://127.0.0.1:4455/login
-
-    login:
-      ui_url: http://127.0.0.1:4455/login
-      lifespan: 10m
-
-    registration:
-      lifespan: 10m
-      ui_url: http://127.0.0.1:4455/registration
-      after:
-        password:
-          hooks:
-            -
-              hook: session
-
-log:
-  level: info
-  format: text
-  leak_sensitive_values: false
-
-secrets:
-  cookie:
-    - PLEASE-CHANGE-ME-I-AM-VERY-INSECURE
-  cipher:
-    - 32-LONG-SECRET-NOT-SECURE-AT-ALL
-
-ciphers:
-  algorithm: xchacha20-poly1305
-
-hashers:
-  argon2:
-    parallelism: 1
-    memory: 128MB
-    iterations: 2
-    salt_length: 16
-    key_length: 16
-
-identity:
-  default_schema_url: file:///etc/config/kratos/identity.schema.json
-
-courier:
-  smtp:
-    connection_uri: smtps://test:test@mailslurper:1025/?skip_ssl_verify=true

agent/kratos/start.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-kratos migrate sql sqlite:///app/data/kratos.sqlite?_fk=true --yes # this initializes the db
-kratos serve -c /etc/config/kratos/kratos.yml --watch-courier # start kratos

agent/main.go

@@ -1,36 +1,35 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
+	"github.com/up9inc/mizu/shared/kubernetes"
 	"io/ioutil"
+	v1 "k8s.io/api/core/v1"
+	"mizuserver/pkg/api"
+	"mizuserver/pkg/config"
+	"mizuserver/pkg/controllers"
+	"mizuserver/pkg/database"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/routes"
+	"mizuserver/pkg/up9"
+	"mizuserver/pkg/utils"
 	"net/http"
 	"os"
 	"os/signal"
-	"strconv"
-	"strings"
+	"path"
+	"path/filepath"
+	"plugin"
+	"sort"
 	"syscall"
 	"time"
 
 	"github.com/gin-contrib/static"
 	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/agent/pkg/elastic"
-	"github.com/up9inc/mizu/agent/pkg/middlewares"
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/oas"
-	"github.com/up9inc/mizu/agent/pkg/routes"
-	"github.com/up9inc/mizu/agent/pkg/servicemap"
-	"github.com/up9inc/mizu/agent/pkg/up9"
-	"github.com/up9inc/mizu/agent/pkg/utils"
-
-	"github.com/up9inc/mizu/agent/pkg/api"
-	"github.com/up9inc/mizu/agent/pkg/app"
-	"github.com/up9inc/mizu/agent/pkg/config"
-
-	v1 "k8s.io/api/core/v1"
-
 	"github.com/gorilla/websocket"
 	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/shared"
@@ -46,12 +45,14 @@ var apiServerAddress = flag.String("api-server-address", "", "Address of mizu AP
 var namespace = flag.String("namespace", "", "Resolve IPs if they belong to resources in this namespace (default is all)")
 var harsReaderMode = flag.Bool("hars-read", false, "Run in hars-read mode")
 var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
-var startTime int64
+
+var extensions []*tapApi.Extension             // global
+var extensionsMap map[string]*tapApi.Extension // global
 
 const (
-	socketConnectionRetries    = 30
+	socketConnectionRetries = 10
 	socketConnectionRetryDelay = time.Second * 2
-	socketHandshakeTimeout     = time.Second * 2
+	socketHandshakeTimeout = time.Second * 2
 )
 
 func main() {
@@ -61,20 +62,77 @@ func main() {
 	if err := config.LoadConfig(); err != nil {
 		logger.Log.Fatalf("Error loading config file %v", err)
 	}
-	app.LoadExtensions()
+	loadExtensions()
 
 	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode {
 		panic("One of the flags --tap, --api or --standalone or --hars-read must be provided")
 	}
 
 	if *standaloneMode {
-		runInStandaloneMode()
+		api.StartResolving(*namespace)
+
+		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
+		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+
+		filteringOptions := getTrafficFilteringOptions()
+		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+		tapOpts := &tap.TapOpts{HostMode: hostMode}
+		tap.StartPassiveTapper(tapOpts, outputItemsChannel, extensions, filteringOptions)
+
+		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
+		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
+
+		hostApi(nil)
 	} else if *tapperMode {
-		runInTapperMode()
+		logger.Log.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
+		if *apiServerAddress == "" {
+			panic("API server address must be provided with --api-server-address when using --tap")
+		}
+
+		tapTargets := getTapTargets()
+		if tapTargets != nil {
+			tap.SetFilterAuthorities(tapTargets)
+			logger.Log.Infof("Filtering for the following authorities: %v", tap.GetFilterIPs())
+		}
+
+		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+
+		filteringOptions := getTrafficFilteringOptions()
+		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+		tapOpts := &tap.TapOpts{HostMode: hostMode}
+		tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions)
+		socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
+		if err != nil {
+			panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
+		}
+		logger.Log.Infof("Connected successfully to websocket %s", *apiServerAddress)
+
+		go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
 	} else if *apiServerMode {
-		utils.StartServer(runInApiServerMode(*namespace))
+		database.InitDataBase(config.Config.AgentDatabasePath)
+		api.StartResolving(*namespace)
+
+		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
+		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+
+		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
+		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
+
+		syncEntriesConfig := getSyncEntriesConfig()
+		if syncEntriesConfig != nil {
+			if err := up9.SyncEntries(syncEntriesConfig); err != nil {
+				panic(fmt.Sprintf("Error syncing entries, err: %v", err))
+			}
+		}
+
+		hostApi(outputItemsChannel)
 	} else if *harsReaderMode {
-		runInHarReaderMode()
+		outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
+		filteredHarChannel := make(chan *tapApi.OutputChannelItem)
+
+		go filterItems(outputItemsChannel, filteredHarChannel)
+		go api.StartReadingEntries(filteredHarChannel, harsDir, extensionsMap)
+		hostApi(nil)
 	}
 
 	signalChan := make(chan os.Signal, 1)
@@ -84,7 +142,50 @@ func main() {
 	logger.Log.Info("Exiting")
 }
 
-func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engine {
+func loadExtensions() {
+	dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
+	extensionsDir := path.Join(dir, "./extensions/")
+
+	files, err := ioutil.ReadDir(extensionsDir)
+	if err != nil {
+		logger.Log.Fatal(err)
+	}
+	extensions = make([]*tapApi.Extension, len(files))
+	extensionsMap = make(map[string]*tapApi.Extension)
+	for i, file := range files {
+		filename := file.Name()
+		logger.Log.Infof("Loading extension: %s\n", filename)
+		extension := &tapApi.Extension{
+			Path: path.Join(extensionsDir, filename),
+		}
+		plug, _ := plugin.Open(extension.Path)
+		extension.Plug = plug
+		symDissector, err := plug.Lookup("Dissector")
+
+		var dissector tapApi.Dissector
+		var ok bool
+		dissector, ok = symDissector.(tapApi.Dissector)
+		if err != nil || !ok {
+			panic(fmt.Sprintf("Failed to load the extension: %s\n", extension.Path))
+		}
+		dissector.Register(extension)
+		extension.Dissector = dissector
+		extensions[i] = extension
+		extensionsMap[extension.Protocol.Name] = extension
+	}
+
+	sort.Slice(extensions, func(i, j int) bool {
+		return extensions[i].Protocol.Priority < extensions[j].Protocol.Priority
+	})
+
+	for _, extension := range extensions {
+		logger.Log.Infof("Extension Properties: %+v\n", extension)
+	}
+
+	controllers.InitExtensionsMap(extensionsMap)
+}
+
+func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 	app := gin.Default()
 
 	app.GET("/echo", func(c *gin.Context) {
@@ -95,147 +196,29 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engin
 		SocketOutChannel: socketHarOutputChannel,
 	}
 
-	app.Use(disableRootStaticCache())
+	app.Use(DisableRootStaticCache())
+	app.Use(static.ServeRoot("/", "./site"))
+	app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
 
-	var staticFolder string
-	if config.Config.StandaloneMode {
-		staticFolder = "./site-standalone"
-	} else {
-		staticFolder = "./site"
-	}
-
-	indexStaticFile := staticFolder + "/index.html"
-	if err := setUIFlags(indexStaticFile); err != nil {
-		logger.Log.Errorf("Error setting ui flags, err: %v", err)
-	}
-
-	app.Use(static.ServeRoot("/", staticFolder))
-	app.NoRoute(func(c *gin.Context) {
-		c.File(indexStaticFile)
-	})
-
-	app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
-
-	api.WebSocketRoutes(app, &eventHandlers, startTime)
-
-	if config.Config.StandaloneMode {
-		routes.ConfigRoutes(app)
-		routes.UserRoutes(app)
-		routes.InstallRoutes(app)
-	}
-	if config.Config.OAS {
-		routes.OASRoutes(app)
-	}
-	if config.Config.ServiceMap {
-		routes.ServiceMapRoutes(app)
-	}
-
-	routes.QueryRoutes(app)
+	api.WebSocketRoutes(app, &eventHandlers)
 	routes.EntriesRoutes(app)
 	routes.MetadataRoutes(app)
 	routes.StatusRoutes(app)
+	routes.NotFoundRoute(app)
 
-	return app
-}
+	if config.Config.DaemonMode {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
 
-func runInApiServerMode(namespace string) *gin.Engine {
-	app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort)
-	startTime = time.Now().UnixNano() / int64(time.Millisecond)
-	api.StartResolving(namespace)
-
-	enableExpFeatureIfNeeded()
-
-	syncEntriesConfig := getSyncEntriesConfig()
-	if syncEntriesConfig != nil {
-		if err := up9.SyncEntries(syncEntriesConfig); err != nil {
-			logger.Log.Error("Error syncing entries, err: %v", err)
+		if _, err := startMizuTapperSyncer(ctx); err != nil {
+			logger.Log.Fatalf("error initializing tapper syncer: %+v", err)
 		}
 	}
 
-	return hostApi(app.GetEntryInputChannel())
+	utils.StartServer(app)
 }
 
-func runInTapperMode() {
-	logger.Log.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
-	if *apiServerAddress == "" {
-		panic("API server address must be provided with --api-server-address when using --tap")
-	}
-
-	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-	tapOpts := &tap.TapOpts{HostMode: hostMode}
-	tapTargets := getTapTargets()
-	if tapTargets != nil {
-		tapOpts.FilterAuthorities = tapTargets
-		logger.Log.Infof("Filtering for the following authorities: %v", tapOpts.FilterAuthorities)
-	}
-
-	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-
-	filteringOptions := getTrafficFilteringOptions()
-	tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, app.Extensions, filteringOptions)
-	socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
-	if err != nil {
-		panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
-	}
-	logger.Log.Infof("Connected successfully to websocket %s", *apiServerAddress)
-
-	go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
-}
-
-func runInStandaloneMode() {
-	api.StartResolving(*namespace)
-
-	outputItemsChannel := make(chan *tapApi.OutputChannelItem)
-	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-
-	filteringOptions := getTrafficFilteringOptions()
-	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-	tapOpts := &tap.TapOpts{HostMode: hostMode}
-	tap.StartPassiveTapper(tapOpts, outputItemsChannel, app.Extensions, filteringOptions)
-
-	go app.FilterItems(outputItemsChannel, filteredOutputItemsChannel)
-	go api.StartReadingEntries(filteredOutputItemsChannel, nil, app.ExtensionsMap)
-
-	ginApp := hostApi(nil)
-	utils.StartServer(ginApp)
-}
-
-func runInHarReaderMode() {
-	outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
-	filteredHarChannel := make(chan *tapApi.OutputChannelItem)
-
-	go app.FilterItems(outputItemsChannel, filteredHarChannel)
-	go api.StartReadingEntries(filteredHarChannel, harsDir, app.ExtensionsMap)
-	ginApp := hostApi(nil)
-	utils.StartServer(ginApp)
-}
-
-func enableExpFeatureIfNeeded() {
-	if config.Config.OAS {
-		oas.GetOasGeneratorInstance().Start()
-	}
-	if config.Config.ServiceMap {
-		servicemap.GetInstance().SetConfig(config.Config)
-	}
-	elastic.GetInstance().Configure(config.Config.Elastic)
-}
-
-func getSyncEntriesConfig() *shared.SyncEntriesConfig {
-	syncEntriesConfigJson := os.Getenv(shared.SyncEntriesConfigEnvVar)
-	if syncEntriesConfigJson == "" {
-		return nil
-	}
-
-	var syncEntriesConfig = &shared.SyncEntriesConfig{}
-	err := json.Unmarshal([]byte(syncEntriesConfigJson), syncEntriesConfig)
-	if err != nil {
-		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.SyncEntriesConfig struct, err: %v", shared.SyncEntriesConfigEnvVar, syncEntriesConfigJson, err))
-	}
-
-	return syncEntriesConfig
-}
-
-func disableRootStaticCache() gin.HandlerFunc {
+func DisableRootStaticCache() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		if c.Request.RequestURI == "/" {
 			// Disable cache only for the main static route
@@ -246,25 +229,24 @@ func disableRootStaticCache() gin.HandlerFunc {
 	}
 }
 
-func setUIFlags(uiIndexPath string) error {
-	read, err := ioutil.ReadFile(uiIndexPath)
-	if err != nil {
-		return err
+func CORSMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
+		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
+		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
+		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")
+
+		if c.Request.Method == "OPTIONS" {
+			c.AbortWithStatus(204)
+			return
+		}
+
+		c.Next()
 	}
-
-	replacedContent := strings.Replace(string(read), "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS), 1)
-	replacedContent = strings.Replace(replacedContent, "__IS_SERVICE_MAP_ENABLED__", strconv.FormatBool(config.Config.ServiceMap), 1)
-
-	err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
-	if err != nil {
-		return err
-	}
-
-	return nil
 }
 
-func parseEnvVar(env string) map[string][]v1.Pod {
-	var mapOfList map[string][]v1.Pod
+func parseEnvVar(env string) map[string][]string {
+	var mapOfList map[string][]string
 
 	val, present := os.LookupEnv(env)
 
@@ -274,12 +256,12 @@ func parseEnvVar(env string) map[string][]v1.Pod {
 
 	err := json.Unmarshal([]byte(val), &mapOfList)
 	if err != nil {
-		panic(fmt.Sprintf("env var %s's value of %v is invalid! must be map[string][]v1.Pod %v", env, mapOfList, err))
+		panic(fmt.Sprintf("env var %s's value of %s is invalid! must be map[string][]string %v", env, mapOfList, err))
 	}
 	return mapOfList
 }
 
-func getTapTargets() []v1.Pod {
+func getTapTargets() []string {
 	nodeName := os.Getenv(shared.NodeNameEnvVar)
 	tappedAddressesPerNodeDict := parseEnvVar(shared.TappedAddressesPerNodeDictEnvVar)
 	return tappedAddressesPerNodeDict[nodeName]
@@ -301,6 +283,16 @@ func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
 	return &filteringOptions
 }
 
+func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
+	for message := range inChannel {
+		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
+			continue
+		}
+
+		outChannel <- message
+	}
+}
+
 func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tapApi.OutputChannelItem) {
 	if connection == nil {
 		panic("Websocket connection is nil")
@@ -336,12 +328,26 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
 	}
 }
 
-func determineLogLevel() (logLevel logging.Level) {
-	logLevel, err := logging.LogLevel(os.Getenv(shared.LogLevelEnvVar))
-	if err != nil {
-		logLevel = logging.INFO
+func getSyncEntriesConfig() *shared.SyncEntriesConfig {
+	syncEntriesConfigJson := os.Getenv(shared.SyncEntriesConfigEnvVar)
+	if syncEntriesConfigJson == "" {
+		return nil
 	}
 
+	var syncEntriesConfig = &shared.SyncEntriesConfig{}
+	err := json.Unmarshal([]byte(syncEntriesConfigJson), syncEntriesConfig)
+	if err != nil {
+		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.SyncEntriesConfig struct, err: %v", shared.SyncEntriesConfigEnvVar, syncEntriesConfigJson, err))
+	}
+
+	return syncEntriesConfig
+}
+
+func determineLogLevel() (logLevel logging.Level) {
+	logLevel = logging.INFO
+	if os.Getenv(shared.DebugModeEnvVar) == "1" {
+		logLevel = logging.DEBUG
+	}
 	return
 }
 
@@ -355,42 +361,69 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time.
 		socketConnection, _, err := dialer.Dial(socketAddress, nil)
 		if err != nil {
 			if i < retryAmount {
-				logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay/time.Second)
+				logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay / time.Second)
 				time.Sleep(retryDelay)
 			}
 		} else {
-			go handleIncomingMessageAsTapper(socketConnection)
 			return socketConnection, nil
 		}
 	}
 	return nil, lastErr
 }
 
-func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
-	for {
-		if _, message, err := socketConnection.ReadMessage(); err != nil {
-			logger.Log.Errorf("error reading message from socket connection, err: %s, (%v,%+v)", err, err, err)
-			if errors.Is(err, syscall.EPIPE) {
-				// socket has disconnected, we can safely stop this goroutine
+
+func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, error){
+	provider, err := kubernetes.NewProviderInCluster()
+	if err != nil {
+		return nil, err
+	}
+
+	tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
+		TargetNamespaces:         config.Config.TargetNamespaces,
+		PodFilterRegex:           config.Config.TapTargetRegex.Regexp,
+		MizuResourcesNamespace:   config.Config.MizuResourcesNamespace,
+		AgentImage:               config.Config.AgentImage,
+		TapperResources:          config.Config.TapperResources,
+		ImagePullPolicy:          v1.PullPolicy(config.Config.PullPolicy),
+		DumpLogs:                 config.Config.DumpLogs,
+		IgnoredUserAgents:        config.Config.IgnoredUserAgents,
+		MizuApiFilteringOptions:  config.Config.MizuApiFilteringOptions,
+		MizuServiceAccountExists: true, //assume service account exists since daemon mode will not function without it anyway
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	// handle tapperSyncer events (pod changes and errors)
+	go func() {
+		for {
+			select {
+			case syncerErr, ok := <-tapperSyncer.ErrorOut:
+				if !ok {
+					logger.Log.Debug("mizuTapperSyncer err channel closed, ending listener loop")
+					return
+				}
+				logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr)
+			case _, ok := <-tapperSyncer.TapPodChangesOut:
+				if !ok {
+					logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop")
+					return
+				}
+				tapStatus := shared.TapStatus{Pods: kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods)}
+
+				serializedTapStatus, err := json.Marshal(shared.CreateWebSocketStatusMessage(tapStatus))
+				if err != nil {
+					logger.Log.Fatalf("error serializing tap status: %v", err)
+				}
+				api.BroadcastToBrowserClients(serializedTapStatus)
+				providers.TapStatus.Pods = tapStatus.Pods
+			case <-ctx.Done():
+				logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")
 				return
 			}
-		} else {
-			var socketMessageBase shared.WebSocketMessageMetadata
-			if err := json.Unmarshal(message, &socketMessageBase); err != nil {
-				logger.Log.Errorf("Could not unmarshal websocket message %v", err)
-			} else {
-				switch socketMessageBase.MessageType {
-				case shared.WebSocketMessageTypeTapConfig:
-					var tapConfigMessage *shared.WebSocketTapConfigMessage
-					if err := json.Unmarshal(message, &tapConfigMessage); err != nil {
-						logger.Log.Errorf("received unknown message from socket connection: %s, err: %s, (%v,%+v)", string(message), err, err, err)
-					} else {
-						tap.UpdateTapTargets(tapConfigMessage.TapTargets)
-					}
-				default:
-					logger.Log.Warningf("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
-				}
-			}
 		}
-	}
+	}()
+
+	return tapperSyncer, nil
 }

agent/pkg/api/main.go

@@ -5,29 +5,24 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"mizuserver/pkg/database"
+	"mizuserver/pkg/holder"
+	"mizuserver/pkg/providers"
 	"os"
 	"path"
 	"sort"
 	"strings"
 	"time"
 
-	"github.com/up9inc/mizu/agent/pkg/elastic"
-	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/agent/pkg/holder"
-	"github.com/up9inc/mizu/agent/pkg/providers"
+	"go.mongodb.org/mongo-driver/bson/primitive"
 
-	"github.com/up9inc/mizu/agent/pkg/servicemap"
-
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/oas"
-	"github.com/up9inc/mizu/agent/pkg/resolver"
-	"github.com/up9inc/mizu/agent/pkg/utils"
-
-	"github.com/up9inc/mizu/shared"
+	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 
-	basenine "github.com/up9inc/basenine/client/go"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/resolver"
+	"mizuserver/pkg/utils"
 )
 
 var k8sResolver *resolver.Resolver
@@ -43,8 +38,10 @@ func StartResolving(namespace string) {
 	res.Start(ctx)
 	go func() {
 		for {
-			err := <-errOut
-			logger.Log.Infof("name resolving error %s", err)
+			select {
+			case err := <-errOut:
+				logger.Log.Infof("name resolving error %s", err)
+			}
 		}
 	}()
 
@@ -66,7 +63,7 @@ func startReadingFiles(workingDir string) {
 		return
 	}
 
-	for {
+	for true {
 		dir, _ := os.Open(workingDir)
 		dirFiles, _ := dir.Readdir(-1)
 
@@ -79,7 +76,7 @@ func startReadingFiles(workingDir string) {
 		sort.Sort(utils.ByModTime(harFiles))
 
 		if len(harFiles) == 0 {
-			logger.Log.Infof("Waiting for new files")
+			logger.Log.Infof("Waiting for new files\n")
 			time.Sleep(3 * time.Second)
 			continue
 		}
@@ -102,59 +99,47 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		panic("Channel of captured messages is nil")
 	}
 
-	connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
-	if err != nil {
-		panic(err)
-	}
-	connection.InsertMode()
-
 	disableOASValidation := false
 	ctx := context.Background()
 	doc, contractContent, router, err := loadOAS(ctx)
 	if err != nil {
-		logger.Log.Infof("Disabled OAS validation: %s", err.Error())
+		logger.Log.Infof("Disabled OAS validation: %s\n", err.Error())
 		disableOASValidation = true
 	}
 
 	for item := range outputItems {
+		providers.EntryAdded()
+
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
-		mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
+		mizuEntry := extension.Dissector.Analyze(item, primitive.NewObjectID().Hex(), resolvedSource, resolvedDestionation)
+		baseEntry := extension.Dissector.Summarize(mizuEntry)
+		mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
 		if extension.Protocol.Name == "http" {
 			if !disableOASValidation {
 				var httpPair tapApi.HTTPRequestResponsePair
-				if err := json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair); err != nil {
-					logger.Log.Error(err)
-				}
+				json.Unmarshal([]byte(mizuEntry.Entry), &httpPair)
 
 				contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
+				baseEntry.ContractStatus = contract.Status
 				mizuEntry.ContractStatus = contract.Status
 				mizuEntry.ContractRequestReason = contract.RequestReason
 				mizuEntry.ContractResponseReason = contract.ResponseReason
 				mizuEntry.ContractContent = contract.Content
 			}
 
-			harEntry, err := har.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
+			var pair tapApi.RequestResponsePair
+			json.Unmarshal([]byte(mizuEntry.Entry), &pair)
+			harEntry, err := utils.NewEntry(&pair)
 			if err == nil {
-				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
-				mizuEntry.Rules = rules
+				rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Service)
+				baseEntry.Rules = rules
 			}
-
-			entryWSource := oas.EntryWithSource{Entry: *harEntry, Source: mizuEntry.Source.Name, Id: mizuEntry.Id}
-			oas.GetOasGeneratorInstance().PushEntry(&entryWSource)
 		}
+		database.CreateEntry(mizuEntry)
 
-		data, err := json.Marshal(mizuEntry)
-		if err != nil {
-			panic(err)
-		}
-
-		providers.EntryAdded(len(data))
-
-		connection.SendText(string(data))
-
-		servicemap.GetInstance().NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
-		elastic.GetInstance().PushEntry(mizuEntry)
+		baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(baseEntry)
+		BroadcastToBrowserClients(baseEntryBytes)
 	}
 }
 
@@ -163,7 +148,7 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re
 		unresolvedSource := connectionInfo.ClientIP
 		resolvedSource = k8sResolver.Resolve(unresolvedSource)
 		if resolvedSource == "" {
-			logger.Log.Debugf("Cannot find resolved name to source: %s", unresolvedSource)
+			logger.Log.Debugf("Cannot find resolved name to source: %s\n", unresolvedSource)
 			if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
 				return
 			}
@@ -171,7 +156,7 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re
 		unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
 		resolvedDestination = k8sResolver.Resolve(unresolvedDestination)
 		if resolvedDestination == "" {
-			logger.Log.Debugf("Cannot find resolved name to dest: %s", unresolvedDestination)
+			logger.Log.Debugf("Cannot find resolved name to dest: %s\n", unresolvedDestination)
 			if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
 				return
 			}
@@ -186,3 +171,21 @@ func CheckIsServiceIP(address string) bool {
 	}
 	return k8sResolver.CheckIsServiceIP(address)
 }
+
+// gives a rough estimate of the size this will take up in the db, good enough for maintaining db size limit accurately
+func getEstimatedEntrySizeBytes(mizuEntry *tapApi.MizuEntry) int {
+	sizeBytes := len(mizuEntry.Entry)
+	sizeBytes += len(mizuEntry.EntryId)
+	sizeBytes += len(mizuEntry.Service)
+	sizeBytes += len(mizuEntry.Url)
+	sizeBytes += len(mizuEntry.Method)
+	sizeBytes += len(mizuEntry.RequestSenderIp)
+	sizeBytes += len(mizuEntry.ResolvedDestination)
+	sizeBytes += len(mizuEntry.ResolvedSource)
+	sizeBytes += 8 // Status bytes (sqlite integer is always 8 bytes)
+	sizeBytes += 8 // Timestamp bytes
+	sizeBytes += 8 // SizeBytes bytes
+	sizeBytes += 1 // IsOutgoing bytes
+
+	return sizeBytes
+}

agent/pkg/api/socket_routes.go

@@ -1,20 +1,15 @@
 package api
 
 import (
-	"encoding/json"
-	"fmt"
+	"errors"
 	"net/http"
 	"sync"
 	"time"
 
-	"github.com/up9inc/mizu/agent/pkg/models"
-
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 type EventHandlers interface {
@@ -30,44 +25,31 @@ type SocketConnection struct {
 	isTapper      bool
 }
 
-var (
-	websocketUpgrader = websocket.Upgrader{
-		ReadBufferSize:  1024,
-		WriteBufferSize: 1024,
-	}
+var websocketUpgrader = websocket.Upgrader{
+	ReadBufferSize:  1024,
+	WriteBufferSize: 1024,
+}
 
-	websocketIdsLock            = sync.Mutex{}
-	connectedWebsockets         map[int]*SocketConnection
-	connectedWebsocketIdCounter = 0
-	SocketGetBrowserHandler     gin.HandlerFunc
-	SocketGetTapperHandler      gin.HandlerFunc
-)
+var websocketIdsLock = sync.Mutex{}
+var connectedWebsockets map[int]*SocketConnection
+var connectedWebsocketIdCounter = 0
 
 func init() {
 	websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
-	connectedWebsockets = make(map[int]*SocketConnection)
+	connectedWebsockets = make(map[int]*SocketConnection, 0)
 }
 
-func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
-	SocketGetBrowserHandler = func(c *gin.Context) {
-		websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime)
-	}
-
-	SocketGetTapperHandler = func(c *gin.Context) {
-		websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime)
-	}
-
+func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
 	app.GET("/ws", func(c *gin.Context) {
-		SocketGetBrowserHandler(c)
+		websocketHandler(c.Writer, c.Request, eventHandlers, false)
 	})
-
-	app.GET("/wsTapper", func(c *gin.Context) { // TODO: add m2m authentication to this route
-		SocketGetTapperHandler(c)
+	app.GET("/wsTapper", func(c *gin.Context) {
+		websocketHandler(c.Writer, c.Request, eventHandlers, true)
 	})
 }
 
-func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool, startTime int64) {
-	ws, err := websocketUpgrader.Upgrade(w, r, nil)
+func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) {
+	conn, err := websocketUpgrader.Upgrade(w, r, nil)
 	if err != nil {
 		logger.Log.Errorf("Failed to set websocket upgrade: %v", err)
 		return
@@ -77,123 +59,30 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 
 	connectedWebsocketIdCounter++
 	socketId := connectedWebsocketIdCounter
-	connectedWebsockets[socketId] = &SocketConnection{connection: ws, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
+	connectedWebsockets[socketId] = &SocketConnection{connection: conn, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
 
 	websocketIdsLock.Unlock()
 
-	var connection *basenine.Connection
-	var isQuerySet bool
-
-	// `!isTapper` means it's a connection from the web UI
-	if !isTapper {
-		connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	data := make(chan []byte)
-	meta := make(chan []byte)
-
 	defer func() {
 		socketCleanup(socketId, connectedWebsockets[socketId])
-		data <- []byte(basenine.CloseChannel)
-		meta <- []byte(basenine.CloseChannel)
-		connection.Close()
 	}()
 
 	eventHandlers.WebSocketConnect(socketId, isTapper)
 
-	startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime)
-
-	if err = SendToSocket(socketId, startTimeBytes); err != nil {
-		logger.Log.Error(err)
-	}
-
 	for {
-		_, msg, err := ws.ReadMessage()
+		_, msg, err := conn.ReadMessage()
 		if err != nil {
-			if _, ok := err.(*websocket.CloseError); ok {
-				logger.Log.Debugf("Received websocket close message, socket id: %d", socketId)
-			} else {
-				logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
-			}
-
+			logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
 			break
 		}
-
-		if !isTapper && !isQuerySet {
-			query := string(msg)
-			err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
-			if err != nil {
-				toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
-					Type:      "error",
-					AutoClose: 5000,
-					Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
-				})
-				if err := SendToSocket(socketId, toastBytes); err != nil {
-					logger.Log.Error(err)
-				}
-				break
-			}
-
-			isQuerySet = true
-
-			handleDataChannel := func(c *basenine.Connection, data chan []byte) {
-				for {
-					bytes := <-data
-
-					if string(bytes) == basenine.CloseChannel {
-						return
-					}
-
-					var entry *tapApi.Entry
-					err = json.Unmarshal(bytes, &entry)
-
-					base := tapApi.Summarize(entry)
-
-					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
-					if err := SendToSocket(socketId, baseEntryBytes); err != nil {
-						logger.Log.Error(err)
-					}
-				}
-			}
-
-			handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
-				for {
-					bytes := <-meta
-
-					if string(bytes) == basenine.CloseChannel {
-						return
-					}
-
-					var metadata *basenine.Metadata
-					err = json.Unmarshal(bytes, &metadata)
-					if err != nil {
-						logger.Log.Debugf("Error recieving metadata: %v", err.Error())
-					}
-
-					metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
-					if err := SendToSocket(socketId, metadataBytes); err != nil {
-						logger.Log.Error(err)
-					}
-				}
-			}
-
-			go handleDataChannel(connection, data)
-			go handleMetaChannel(connection, meta)
-
-			connection.Query(query, data, meta)
-		} else {
-			eventHandlers.WebSocketMessage(socketId, msg)
-		}
+		eventHandlers.WebSocketMessage(socketId, msg)
 	}
 }
 
 func socketCleanup(socketId int, socketConnection *SocketConnection) {
 	err := socketConnection.connection.Close()
 	if err != nil {
-		logger.Log.Errorf("Error closing socket connection for socket id %d: %v", socketId, err)
+		logger.Log.Errorf("Error closing socket connection for socket id %d: %v\n", socketId, err)
 	}
 
 	websocketIdsLock.Lock()
@@ -203,10 +92,14 @@ func socketCleanup(socketId int, socketConnection *SocketConnection) {
 	socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
 }
 
+var db = debounce.NewDebouncer(time.Second*5, func() {
+	logger.Log.Error("Successfully sent to socket")
+})
+
 func SendToSocket(socketId int, message []byte) error {
 	socketObj := connectedWebsockets[socketId]
 	if socketObj == nil {
-		return fmt.Errorf("Socket %v is disconnected", socketId)
+		return errors.New("Socket is disconnected")
 	}
 
 	var sent = false
@@ -220,10 +113,7 @@ func SendToSocket(socketId int, message []byte) error {
 	socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
 	err := socketObj.connection.WriteMessage(1, message)
 	socketObj.lock.Unlock()
-	sent = true
 
-	if err != nil {
-		return fmt.Errorf("Failed to write message to socket %v, err: %w", socketId, err)
-	}
-	return nil
+	sent = true
+	return err
 }

agent/pkg/api/socket_server_handlers.go

@@ -3,13 +3,11 @@ package api
 import (
 	"encoding/json"
 	"fmt"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/up9"
 	"sync"
 
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/providers"
-	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
-	"github.com/up9inc/mizu/agent/pkg/up9"
-
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	"github.com/up9inc/mizu/shared"
@@ -31,22 +29,19 @@ func init() {
 func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
-		tappers.Connected()
+		providers.TapperAdded()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
-
 		socketListLock.Lock()
 		browserClientSocketUUIDs = append(browserClientSocketUUIDs, socketId)
 		socketListLock.Unlock()
-
-		BroadcastTappedPodsStatus()
 	}
 }
 
 func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 	if isTapper {
 		logger.Log.Infof("Websocket event - Tapper disconnected, socket ID:  %d", socketId)
-		tappers.Disconnected()
+		providers.TapperRemoved()
 	} else {
 		logger.Log.Infof("Websocket event - Browser socket disconnected, socket ID:  %d", socketId)
 		socketListLock.Lock()
@@ -58,8 +53,9 @@ func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
 func BroadcastToBrowserClients(message []byte) {
 	for _, socketId := range browserClientSocketUUIDs {
 		go func(socketId int) {
-			if err := SendToSocket(socketId, message); err != nil {
-				logger.Log.Error(err)
+			err := SendToSocket(socketId, message)
+			if err != nil {
+				logger.Log.Errorf("error sending message to socket ID %d: %v", socketId, err)
 			}
 		}(socketId)
 	}
@@ -69,14 +65,14 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 	var socketMessageBase shared.WebSocketMessageMetadata
 	err := json.Unmarshal(message, &socketMessageBase)
 	if err != nil {
-		logger.Log.Infof("Could not unmarshal websocket message %v", err)
+		logger.Log.Infof("Could not unmarshal websocket message %v\n", err)
 	} else {
 		switch socketMessageBase.MessageType {
 		case shared.WebSocketMessageTypeTappedEntry:
 			var tappedEntryMessage models.WebSocketTappedEntryMessage
 			err := json.Unmarshal(message, &tappedEntryMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
+				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
 			} else {
 				// NOTE: This is where the message comes back from the intermediate WebSocket to code.
 				h.SocketOutChannel <- tappedEntryMessage.Data
@@ -85,15 +81,16 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
 			var statusMessage shared.WebSocketStatusMessage
 			err := json.Unmarshal(message, &statusMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
+				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
 			} else {
+				providers.TapStatus.Pods = statusMessage.TappingStatus.Pods
 				BroadcastToBrowserClients(message)
 			}
 		case shared.WebsocketMessageTypeOutboundLink:
 			var outboundLinkMessage models.WebsocketOutboundLinkMessage
 			err := json.Unmarshal(message, &outboundLinkMessage)
 			if err != nil {
-				logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
+				logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
 			} else {
 				handleTLSLink(outboundLinkMessage)
 			}

agent/pkg/api/utils.go

@@ -1,20 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-
-	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-func BroadcastTappedPodsStatus() {
-	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
-
-	message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
-	if jsonBytes, err := json.Marshal(message); err != nil {
-		logger.Log.Errorf("Could not Marshal message %v", err)
-	} else {
-		BroadcastToBrowserClients(jsonBytes)
-	}
-}

agent/pkg/app/main.go

@@ -1,116 +0,0 @@
-package app
-
-import (
-	"fmt"
-	"sort"
-	"time"
-
-	"github.com/antelman107/net-wait-go/wait"
-	"github.com/op/go-logging"
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/agent/pkg/api"
-	"github.com/up9inc/mizu/agent/pkg/config"
-	"github.com/up9inc/mizu/agent/pkg/controllers"
-	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
-	amqpExt "github.com/up9inc/mizu/tap/extensions/amqp"
-	httpExt "github.com/up9inc/mizu/tap/extensions/http"
-	kafkaExt "github.com/up9inc/mizu/tap/extensions/kafka"
-	redisExt "github.com/up9inc/mizu/tap/extensions/redis"
-)
-
-var (
-	Extensions    []*tapApi.Extension          // global
-	ExtensionsMap map[string]*tapApi.Extension // global
-)
-
-func LoadExtensions() {
-	Extensions = make([]*tapApi.Extension, 4)
-	ExtensionsMap = make(map[string]*tapApi.Extension)
-
-	extensionAmqp := &tapApi.Extension{}
-	dissectorAmqp := amqpExt.NewDissector()
-	dissectorAmqp.Register(extensionAmqp)
-	extensionAmqp.Dissector = dissectorAmqp
-	Extensions[0] = extensionAmqp
-	ExtensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
-
-	extensionHttp := &tapApi.Extension{}
-	dissectorHttp := httpExt.NewDissector()
-	dissectorHttp.Register(extensionHttp)
-	extensionHttp.Dissector = dissectorHttp
-	Extensions[1] = extensionHttp
-	ExtensionsMap[extensionHttp.Protocol.Name] = extensionHttp
-
-	extensionKafka := &tapApi.Extension{}
-	dissectorKafka := kafkaExt.NewDissector()
-	dissectorKafka.Register(extensionKafka)
-	extensionKafka.Dissector = dissectorKafka
-	Extensions[2] = extensionKafka
-	ExtensionsMap[extensionKafka.Protocol.Name] = extensionKafka
-
-	extensionRedis := &tapApi.Extension{}
-	dissectorRedis := redisExt.NewDissector()
-	dissectorRedis.Register(extensionRedis)
-	extensionRedis.Dissector = dissectorRedis
-	Extensions[3] = extensionRedis
-	ExtensionsMap[extensionRedis.Protocol.Name] = extensionRedis
-
-	sort.Slice(Extensions, func(i, j int) bool {
-		return Extensions[i].Protocol.Priority < Extensions[j].Protocol.Priority
-	})
-
-	for _, extension := range Extensions {
-		logger.Log.Infof("Extension Properties: %+v", extension)
-	}
-
-	controllers.InitExtensionsMap(ExtensionsMap)
-}
-
-func ConfigureBasenineServer(host string, port string) {
-	if !wait.New(
-		wait.WithProto("tcp"),
-		wait.WithWait(200*time.Millisecond),
-		wait.WithBreak(50*time.Millisecond),
-		wait.WithDeadline(5*time.Second),
-		wait.WithDebug(config.Config.LogLevel == logging.DEBUG),
-	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
-		logger.Log.Panicf("Basenine is not available!")
-	}
-
-	// Limit the database size to default 200MB
-	err := basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
-	if err != nil {
-		logger.Log.Panicf("Error while limiting database size: %v", err)
-	}
-
-	// Define the macros
-	for _, extension := range Extensions {
-		macros := extension.Dissector.Macros()
-		for macro, expanded := range macros {
-			err = basenine.Macro(host, port, macro, expanded)
-			if err != nil {
-				logger.Log.Panicf("Error while adding a macro: %v", err)
-			}
-		}
-	}
-}
-
-func GetEntryInputChannel() chan *tapApi.OutputChannelItem {
-	outputItemsChannel := make(chan *tapApi.OutputChannelItem)
-	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-	go FilterItems(outputItemsChannel, filteredOutputItemsChannel)
-	go api.StartReadingEntries(filteredOutputItemsChannel, nil, ExtensionsMap)
-
-	return outputItemsChannel
-}
-
-func FilterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
-	for message := range inChannel {
-		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
-			continue
-		}
-
-		outChannel <- message
-	}
-}

agent/pkg/config/config.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap/api"
 	"io/ioutil"
 	"os"
 )
@@ -11,6 +12,7 @@ import (
 // these values are used when the config.json file is not present
 const (
 	defaultMaxDatabaseSizeBytes int64  = 200 * 1000 * 1000
+	defaultRegexTarget          string = ".*"
 	DefaultDatabasePath         string = "./entries"
 )
 
@@ -46,8 +48,14 @@ func applyDefaultConfig() error {
 }
 
 func getDefaultConfig() (*shared.MizuAgentConfig, error) {
+	regex, err := api.CompileRegexToSerializableRegexp(defaultRegexTarget)
+	if err != nil {
+		return nil, err
+	}
 	return &shared.MizuAgentConfig{
+		TapTargetRegex:    *regex,
 		MaxDBSizeBytes:    defaultMaxDatabaseSizeBytes,
 		AgentDatabasePath: DefaultDatabasePath,
+		DaemonMode:        false,
 	}, nil
 }

agent/pkg/controllers/config_controller.go

@@ -1,156 +0,0 @@
-package controllers
-
-import (
-	"context"
-	"net/http"
-	"regexp"
-	"time"
-
-	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/agent/pkg/api"
-	"github.com/up9inc/mizu/agent/pkg/config"
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/providers"
-	"github.com/up9inc/mizu/agent/pkg/providers/tapConfig"
-	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
-	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/kubernetes"
-	"github.com/up9inc/mizu/shared/logger"
-	tapApi "github.com/up9inc/mizu/tap/api"
-	v1 "k8s.io/api/core/v1"
-)
-
-var cancelTapperSyncer context.CancelFunc
-
-func PostTapConfig(c *gin.Context) {
-	requestTapConfig := &models.TapConfig{}
-
-	if err := c.Bind(requestTapConfig); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-		return
-	}
-
-	if cancelTapperSyncer != nil {
-		cancelTapperSyncer()
-
-		tappedPods.Set([]*shared.PodInfo{})
-		tappers.ResetStatus()
-
-		api.BroadcastTappedPodsStatus()
-	}
-
-	var tappedNamespaces []string
-	for namespace, tapped := range requestTapConfig.TappedNamespaces {
-		if tapped {
-			tappedNamespaces = append(tappedNamespaces, namespace)
-		}
-	}
-
-	podRegex, _ := regexp.Compile(".*")
-
-	kubernetesProvider, err := providers.GetKubernetesProvider()
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, err)
-		return
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-
-	if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{}, tapApi.TrafficFilteringOptions{}, false); err != nil {
-		c.JSON(http.StatusInternalServerError, err)
-		cancel()
-		return
-	}
-
-	cancelTapperSyncer = cancel
-	tapConfig.Save(requestTapConfig)
-
-	c.JSON(http.StatusOK, "OK")
-}
-
-func GetTapConfig(c *gin.Context) {
-	kubernetesProvider, err := providers.GetKubernetesProvider()
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, err)
-		return
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	namespaces, err := kubernetesProvider.ListAllNamespaces(ctx)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, err)
-		return
-	}
-
-	savedTapConfig := tapConfig.Get()
-
-	tappedNamespaces := make(map[string]bool)
-	for _, namespace := range namespaces {
-		if namespace.Name == config.Config.MizuResourcesNamespace {
-			continue
-		}
-
-		tappedNamespaces[namespace.Name] = savedTapConfig.TappedNamespaces[namespace.Name]
-	}
-
-	tapConfigToReturn := models.TapConfig{TappedNamespaces: tappedNamespaces}
-	c.JSON(http.StatusOK, tapConfigToReturn)
-}
-
-func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, serviceMesh bool) (*kubernetes.MizuTapperSyncer, error) {
-	tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
-		TargetNamespaces:         targetNamespaces,
-		PodFilterRegex:           podFilterRegex,
-		MizuResourcesNamespace:   config.Config.MizuResourcesNamespace,
-		AgentImage:               config.Config.AgentImage,
-		TapperResources:          config.Config.TapperResources,
-		ImagePullPolicy:          v1.PullPolicy(config.Config.PullPolicy),
-		LogLevel:                 config.Config.LogLevel,
-		IgnoredUserAgents:        ignoredUserAgents,
-		MizuApiFilteringOptions:  mizuApiFilteringOptions,
-		MizuServiceAccountExists: true, //assume service account exists since install mode will not function without it anyway
-		ServiceMesh:              serviceMesh,
-	}, time.Now())
-
-	if err != nil {
-		return nil, err
-	}
-
-	// handle tapperSyncer events (pod changes and errors)
-	go func() {
-		for {
-			select {
-			case syncerErr, ok := <-tapperSyncer.ErrorOut:
-				if !ok {
-					logger.Log.Debug("mizuTapperSyncer err channel closed, ending listener loop")
-					return
-				}
-				logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr)
-			case _, ok := <-tapperSyncer.TapPodChangesOut:
-				if !ok {
-					logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop")
-					return
-				}
-
-				tappedPods.Set(kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods))
-				api.BroadcastTappedPodsStatus()
-			case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut:
-				if !ok {
-					logger.Log.Debug("mizuTapperSyncer tapper status changed channel closed, ending listener loop")
-					return
-				}
-
-				tappers.SetStatus(&tapperStatus)
-				api.BroadcastTappedPodsStatus()
-			case <-ctx.Done():
-				logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")
-				return
-			}
-		}
-	}()
-
-	return tapperSyncer, nil
-}

agent/pkg/controllers/entries_controller.go

@@ -2,20 +2,14 @@ package controllers
 
 import (
 	"encoding/json"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/validation"
-
+	"fmt"
 	"github.com/gin-gonic/gin"
-
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
+	"mizuserver/pkg/database"
+	"mizuserver/pkg/models"
+	"mizuserver/pkg/utils"
+	"mizuserver/pkg/validation"
+	"net/http"
 )
 
 var extensionsMap map[string]*tapApi.Extension // global
@@ -24,124 +18,78 @@ func InitExtensionsMap(ref map[string]*tapApi.Extension) {
 	extensionsMap = ref
 }
 
-func Error(c *gin.Context, err error) bool {
-	if err != nil {
-		logger.Log.Errorf("Error getting entry: %v", err)
-		_ = c.Error(err)
-		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
-			"error":     true,
-			"type":      "error",
-			"autoClose": "5000",
-			"msg":       err.Error(),
-		})
-		return true // signal that there was an error and the caller should return
-	}
-	return false // no error, can continue
-}
-
 func GetEntries(c *gin.Context) {
-	entriesRequest := &models.EntriesRequest{}
+	entriesFilter := &models.EntriesFilter{}
 
-	if err := c.BindQuery(entriesRequest); err != nil {
+	if err := c.BindQuery(entriesFilter); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 	}
-	validationError := validation.Validate(entriesRequest)
-	if validationError != nil {
-		c.JSON(http.StatusBadRequest, validationError)
-	}
-
-	if entriesRequest.TimeoutMs == 0 {
-		entriesRequest.TimeoutMs = 3000
-	}
-
-	data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
-		entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
-		entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
+	err := validation.Validate(entriesFilter)
 	if err != nil {
-		c.JSON(http.StatusInternalServerError, validationError)
+		c.JSON(http.StatusBadRequest, err)
 	}
 
-	response := &models.EntriesResponse{}
-	var dataSlice []interface{}
+	order := database.OperatorToOrderMapping[entriesFilter.Operator]
+	operatorSymbol := database.OperatorToSymbolMapping[entriesFilter.Operator]
+	var entries []tapApi.MizuEntry
+	database.GetEntriesTable().
+		Order(fmt.Sprintf("timestamp %s", order)).
+		Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)).
+		Limit(entriesFilter.Limit).
+		Find(&entries)
 
-	for _, row := range data {
-		var entry *tapApi.Entry
-		err = json.Unmarshal(row, &entry)
-		if err != nil {
-			c.JSON(http.StatusBadRequest, gin.H{
-				"error":     true,
-				"type":      "error",
-				"autoClose": "5000",
-				"msg":       string(row),
-			})
-			return // exit
+	if len(entries) > 0 && order == database.OrderDesc {
+		// the entries always order from oldest to newest - we should reverse
+		utils.ReverseSlice(entries)
+	}
+
+	baseEntries := make([]tapApi.BaseEntryDetails, 0)
+	for _, entry := range entries {
+		baseEntryDetails := tapApi.BaseEntryDetails{}
+		if err := models.GetEntry(&entry, &baseEntryDetails); err != nil {
+			continue
 		}
 
-		base := tapApi.Summarize(entry)
+		var pair tapApi.RequestResponsePair
+		json.Unmarshal([]byte(entry.Entry), &pair)
+		harEntry, err := utils.NewEntry(&pair)
+		if err == nil {
+			rules, _, _ := models.RunValidationRulesState(*harEntry, entry.Service)
+			baseEntryDetails.Rules = rules
+		}
 
-		dataSlice = append(dataSlice, base)
+		baseEntries = append(baseEntries, baseEntryDetails)
 	}
 
-	var metadata *basenine.Metadata
-	err = json.Unmarshal(meta, &metadata)
-	if err != nil {
-		logger.Log.Debugf("Error recieving metadata: %v", err.Error())
-	}
-
-	response.Data = dataSlice
-	response.Meta = metadata
-
-	c.JSON(http.StatusOK, response)
+	c.JSON(http.StatusOK, baseEntries)
 }
 
 func GetEntry(c *gin.Context) {
-	singleEntryRequest := &models.SingleEntryRequest{}
+	var entryData tapApi.MizuEntry
+	database.GetEntriesTable().
+		Where(map[string]string{"entryId": c.Param("entryId")}).
+		First(&entryData)
 
-	if err := c.BindQuery(singleEntryRequest); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-	}
-	validationError := validation.Validate(singleEntryRequest)
-	if validationError != nil {
-		c.JSON(http.StatusBadRequest, validationError)
-	}
-
-	id, _ := strconv.Atoi(c.Param("id"))
-	var entry *tapApi.Entry
-	bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id, singleEntryRequest.Query)
-	if Error(c, err) {
-		return // exit
-	}
-	err = json.Unmarshal(bytes, &entry)
-	if err != nil {
-		c.JSON(http.StatusNotFound, gin.H{
-			"error":     true,
-			"type":      "error",
-			"autoClose": "5000",
-			"msg":       string(bytes),
-		})
-		return // exit
-	}
-
-	extension := extensionsMap[entry.Protocol.Name]
-	representation, bodySize, _ := extension.Dissector.Represent(entry.Request, entry.Response)
+	extension := extensionsMap[entryData.ProtocolName]
+	protocol, representation, bodySize, _ := extension.Dissector.Represent(&entryData)
 
 	var rules []map[string]interface{}
 	var isRulesEnabled bool
-	if entry.Protocol.Name == "http" {
-		harEntry, _ := har.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
-		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
+	if entryData.ProtocolName == "http" {
+		var pair tapApi.RequestResponsePair
+		json.Unmarshal([]byte(entryData.Entry), &pair)
+		harEntry, _ := utils.NewEntry(&pair)
+		_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entryData.Service)
 		isRulesEnabled = _isRulesEnabled
 		inrec, _ := json.Marshal(rulesMatched)
-		if err := json.Unmarshal(inrec, &rules); err != nil {
-			logger.Log.Error(err)
-		}
+		json.Unmarshal(inrec, &rules)
 	}
 
-	c.JSON(http.StatusOK, tapApi.EntryWrapper{
-		Protocol:       entry.Protocol,
+	c.JSON(http.StatusOK, tapApi.MizuEntryWrapper{
+		Protocol:       protocol,
 		Representation: string(representation),
 		BodySize:       bodySize,
-		Data:           entry,
+		Data:           entryData,
 		Rules:          rules,
 		IsRulesEnabled: isRulesEnabled,
 	})

agent/pkg/controllers/install_controller.go

@@ -1,24 +0,0 @@
-package controllers
-
-import (
-	"net/http"
-
-	"github.com/up9inc/mizu/agent/pkg/providers"
-
-	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-func IsSetupNecessary(c *gin.Context) {
-	if IsInstallNeeded, err := providers.IsInstallNeeded(); err != nil {
-		logger.Log.Errorf("unknown internal while checking if install is needed %s", err)
-		c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while checking if install is needed"})
-	} else {
-		c.JSON(http.StatusOK, IsInstallNeeded)
-	}
-}
-
-func SetupAdminUser(c *gin.Context) {
-	token, err, formErrorMessages := providers.CreateAdminUser(c.PostForm("password"), c.Request.Context())
-	handleRegistration(token, err, formErrorMessages, c)
-}

agent/pkg/controllers/metadata_controller.go

@@ -1,14 +1,13 @@
 package controllers
 
 import (
-	"net/http"
-
 	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/agent/pkg/version"
 	"github.com/up9inc/mizu/shared"
+	"mizuserver/pkg/version"
+	"net/http"
 )
 
 func GetVersion(c *gin.Context) {
-	resp := shared.VersionResponse{Ver: version.Ver}
+	resp := shared.VersionResponse{SemVer: version.SemVer}
 	c.JSON(http.StatusOK, resp)
 }

agent/pkg/controllers/oas_controller.go

@@ -1,63 +0,0 @@
-package controllers
-
-import (
-	"net/http"
-
-	"github.com/chanced/openapi"
-	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/agent/pkg/oas"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-func GetOASServers(c *gin.Context) {
-	m := make([]string, 0)
-	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
-		m = append(m, key.(string))
-		return true
-	})
-
-	c.JSON(http.StatusOK, m)
-}
-
-func GetOASSpec(c *gin.Context) {
-	res, ok := oas.GetOasGeneratorInstance().ServiceSpecs.Load(c.Param("id"))
-	if !ok {
-		c.JSON(http.StatusNotFound, gin.H{
-			"error":     true,
-			"type":      "error",
-			"autoClose": "5000",
-			"msg":       "Service not found among specs",
-		})
-		return // exit
-	}
-
-	gen := res.(*oas.SpecGen)
-	spec, err := gen.GetSpec()
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{
-			"error":     true,
-			"type":      "error",
-			"autoClose": "5000",
-			"msg":       err,
-		})
-		return // exit
-	}
-
-	c.JSON(http.StatusOK, spec)
-}
-
-func GetOASAllSpecs(c *gin.Context) {
-	res := map[string]*openapi.OpenAPI{}
-	oas.GetOasGeneratorInstance().ServiceSpecs.Range(func(key, value interface{}) bool {
-		svc := key.(string)
-		gen := value.(*oas.SpecGen)
-		spec, err := gen.GetSpec()
-		if err != nil {
-			logger.Log.Warningf("Failed to obtain spec for service %s: %s", svc, err)
-			return true
-		}
-		res[svc] = spec
-		return true
-	})
-	c.JSON(http.StatusOK, res)
-}

agent/pkg/controllers/oas_controller_test.go

@@ -1,42 +0,0 @@
-package controllers
-
-import (
-	"net/http/httptest"
-	"testing"
-
-	"github.com/up9inc/mizu/agent/pkg/oas"
-
-	"github.com/gin-gonic/gin"
-)
-
-func TestGetOASServers(t *testing.T) {
-	recorder := httptest.NewRecorder()
-	c, _ := gin.CreateTestContext(recorder)
-	oas.GetOasGeneratorInstance().Start()
-	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
-
-	GetOASServers(c)
-	t.Logf("Written body: %s", recorder.Body.String())
-}
-
-func TestGetOASAllSpecs(t *testing.T) {
-	recorder := httptest.NewRecorder()
-	c, _ := gin.CreateTestContext(recorder)
-	oas.GetOasGeneratorInstance().Start()
-	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
-
-	GetOASAllSpecs(c)
-	t.Logf("Written body: %s", recorder.Body.String())
-}
-
-func TestGetOASSpec(t *testing.T) {
-	recorder := httptest.NewRecorder()
-	c, _ := gin.CreateTestContext(recorder)
-	oas.GetOasGeneratorInstance().Start()
-	oas.GetOasGeneratorInstance().ServiceSpecs.Store("some", oas.NewGen("some"))
-
-	c.Params = []gin.Param{{Key: "id", Value: "some"}}
-
-	GetOASSpec(c)
-	t.Logf("Written body: %s", recorder.Body.String())
-}

agent/pkg/controllers/query_controller.go

@@ -1,31 +0,0 @@
-package controllers
-
-import (
-	"net/http"
-
-	"github.com/gin-gonic/gin"
-	basenine "github.com/up9inc/basenine/client/go"
-	"github.com/up9inc/mizu/shared"
-)
-
-type ValidateResponse struct {
-	Valid   bool   `json:"valid"`
-	Message string `json:"message"`
-}
-
-func PostValidate(c *gin.Context) {
-	query := c.PostForm("query")
-	valid := true
-	message := ""
-
-	err := basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
-	if err != nil {
-		valid = false
-		message = err.Error()
-	}
-
-	c.JSON(http.StatusOK, ValidateResponse{
-		Valid:   valid,
-		Message: message,
-	})
-}

agent/pkg/controllers/service_map_controller.go

@@ -1,37 +0,0 @@
-package controllers
-
-import (
-	"net/http"
-
-	"github.com/up9inc/mizu/agent/pkg/servicemap"
-
-	"github.com/gin-gonic/gin"
-)
-
-type ServiceMapController struct {
-	service servicemap.ServiceMap
-}
-
-func NewServiceMapController() *ServiceMapController {
-	return &ServiceMapController{
-		service: servicemap.GetInstance(),
-	}
-}
-
-func (s *ServiceMapController) Status(c *gin.Context) {
-	c.JSON(http.StatusOK, s.service.GetStatus())
-}
-
-func (s *ServiceMapController) Get(c *gin.Context) {
-	response := &servicemap.ServiceMapResponse{
-		Status: s.service.GetStatus(),
-		Nodes:  s.service.GetNodes(),
-		Edges:  s.service.GetEdges(),
-	}
-	c.JSON(http.StatusOK, response)
-}
-
-func (s *ServiceMapController) Reset(c *gin.Context) {
-	s.service.Reset()
-	s.Status(c)
-}

agent/pkg/controllers/service_map_controller_test.go

@@ -1,147 +0,0 @@
-package controllers
-
-import (
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/up9inc/mizu/agent/pkg/servicemap"
-
-	"github.com/gin-gonic/gin"
-	"github.com/stretchr/testify/suite"
-	"github.com/up9inc/mizu/shared"
-	tapApi "github.com/up9inc/mizu/tap/api"
-)
-
-const (
-	a    = "aService"
-	b    = "bService"
-	Ip   = "127.0.0.1"
-	Port = "80"
-)
-
-var (
-	TCPEntryA = &tapApi.TCP{
-		Name: a,
-		Port: Port,
-		IP:   fmt.Sprintf("%s.%s", Ip, a),
-	}
-	TCPEntryB = &tapApi.TCP{
-		Name: b,
-		Port: Port,
-		IP:   fmt.Sprintf("%s.%s", Ip, b),
-	}
-)
-
-var ProtocolHttp = &tapApi.Protocol{
-	Name:            "http",
-	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
-	Abbreviation:    "HTTP",
-	Macro:           "http",
-	Version:         "1.1",
-	BackgroundColor: "#205cf5",
-	ForegroundColor: "#ffffff",
-	FontSize:        12,
-	ReferenceLink:   "https://datatracker.ietf.org/doc/html/rfc2616",
-	Ports:           []string{"80", "443", "8080"},
-	Priority:        0,
-}
-
-type ServiceMapControllerSuite struct {
-	suite.Suite
-
-	c *ServiceMapController
-	w *httptest.ResponseRecorder
-	g *gin.Context
-}
-
-func (s *ServiceMapControllerSuite) SetupTest() {
-	s.c = NewServiceMapController()
-	s.c.service.SetConfig(&shared.MizuAgentConfig{
-		ServiceMap: true,
-	})
-	s.c.service.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
-
-	s.w = httptest.NewRecorder()
-	s.g, _ = gin.CreateTestContext(s.w)
-}
-
-func (s *ServiceMapControllerSuite) TestGetStatus() {
-	assert := s.Assert()
-
-	s.c.Status(s.g)
-	assert.Equal(http.StatusOK, s.w.Code)
-
-	var status servicemap.ServiceMapStatus
-	err := json.Unmarshal(s.w.Body.Bytes(), &status)
-	assert.NoError(err)
-	assert.Equal("enabled", status.Status)
-	assert.Equal(1, status.EntriesProcessedCount)
-	assert.Equal(2, status.NodeCount)
-	assert.Equal(1, status.EdgeCount)
-}
-
-func (s *ServiceMapControllerSuite) TestGet() {
-	assert := s.Assert()
-
-	s.c.Get(s.g)
-	assert.Equal(http.StatusOK, s.w.Code)
-
-	var response servicemap.ServiceMapResponse
-	err := json.Unmarshal(s.w.Body.Bytes(), &response)
-	assert.NoError(err)
-
-	// response status
-	assert.Equal("enabled", response.Status.Status)
-	assert.Equal(1, response.Status.EntriesProcessedCount)
-	assert.Equal(2, response.Status.NodeCount)
-	assert.Equal(1, response.Status.EdgeCount)
-
-	// response nodes
-	aNode := servicemap.ServiceMapNode{
-		Id:    1,
-		Name:  TCPEntryA.IP,
-		Entry: TCPEntryA,
-		Count: 1,
-	}
-	bNode := servicemap.ServiceMapNode{
-		Id:    2,
-		Name:  TCPEntryB.IP,
-		Entry: TCPEntryB,
-		Count: 1,
-	}
-	assert.Contains(response.Nodes, aNode)
-	assert.Contains(response.Nodes, bNode)
-	assert.Len(response.Nodes, 2)
-
-	// response edges
-	assert.Equal([]servicemap.ServiceMapEdge{
-		{
-			Source:      aNode,
-			Destination: bNode,
-			Protocol:    ProtocolHttp,
-			Count:       1,
-		},
-	}, response.Edges)
-}
-
-func (s *ServiceMapControllerSuite) TestGetReset() {
-	assert := s.Assert()
-
-	s.c.Reset(s.g)
-	assert.Equal(http.StatusOK, s.w.Code)
-
-	var status servicemap.ServiceMapStatus
-	err := json.Unmarshal(s.w.Body.Bytes(), &status)
-	assert.NoError(err)
-	assert.Equal("enabled", status.Status)
-	assert.Equal(0, status.EntriesProcessedCount)
-	assert.Equal(0, status.NodeCount)
-	assert.Equal(0, status.EdgeCount)
-}
-
-func TestServiceMapControllerSuite(t *testing.T) {
-	suite.Run(t, new(ServiceMapControllerSuite))
-}

agent/pkg/controllers/status_controller.go

@@ -1,65 +1,50 @@
 package controllers
 
 import (
+	"encoding/json"
+	"mizuserver/pkg/api"
+	"mizuserver/pkg/holder"
+	"mizuserver/pkg/providers"
+	"mizuserver/pkg/up9"
+	"mizuserver/pkg/validation"
 	"net/http"
 
 	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/agent/pkg/api"
-	"github.com/up9inc/mizu/agent/pkg/holder"
-	"github.com/up9inc/mizu/agent/pkg/providers"
-	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
-	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
-	"github.com/up9inc/mizu/agent/pkg/up9"
-	"github.com/up9inc/mizu/agent/pkg/validation"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 )
 
 func HealthCheck(c *gin.Context) {
-	tappersStatus := make([]*shared.TapperStatus, 0)
-	for _, value := range tappers.GetStatus() {
-		tappersStatus = append(tappersStatus, value)
-	}
-
 	response := shared.HealthResponse{
-		TappedPods:            tappedPods.Get(),
-		ConnectedTappersCount: tappers.GetConnectedCount(),
-		TappersStatus:         tappersStatus,
+		TapStatus:    providers.TapStatus,
+		TappersCount: providers.TappersCount,
 	}
 	c.JSON(http.StatusOK, response)
 }
 
+
 func PostTappedPods(c *gin.Context) {
-	var requestTappedPods []*shared.PodInfo
-	if err := c.Bind(&requestTappedPods); err != nil {
+	tapStatus := &shared.TapStatus{}
+	if err := c.Bind(tapStatus); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
-
-	logger.Log.Infof("[Status] POST request: %d tapped pods", len(requestTappedPods))
-	tappedPods.Set(requestTappedPods)
-	api.BroadcastTappedPodsStatus()
+	if err := validation.Validate(tapStatus); err != nil {
+		c.JSON(http.StatusBadRequest, err)
+		return
+	}
+	logger.Log.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods))
+	providers.TapStatus.Pods = tapStatus.Pods
+	message := shared.CreateWebSocketStatusMessage(*tapStatus)
+	if jsonBytes, err := json.Marshal(message); err != nil {
+		logger.Log.Errorf("Could not Marshal message %v\n", err)
+	} else {
+		api.BroadcastToBrowserClients(jsonBytes)
+	}
 }
 
-func PostTapperStatus(c *gin.Context) {
-	tapperStatus := &shared.TapperStatus{}
-	if err := c.Bind(tapperStatus); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-		return
-	}
-
-	if err := validation.Validate(tapperStatus); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-		return
-	}
-
-	logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus)
-	tappers.SetStatus(tapperStatus)
-	api.BroadcastTappedPodsStatus()
-}
-
-func GetConnectedTappersCount(c *gin.Context) {
-	c.JSON(http.StatusOK, tappers.GetConnectedCount())
+func GetTappersCount(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.TappersCount)
 }
 
 func GetAuthStatus(c *gin.Context) {
@@ -73,8 +58,7 @@ func GetAuthStatus(c *gin.Context) {
 }
 
 func GetTappingStatus(c *gin.Context) {
-	tappedPodsStatus := tappedPods.GetTappedPodsStatus()
-	c.JSON(http.StatusOK, tappedPodsStatus)
+	c.JSON(http.StatusOK, providers.TapStatus)
 }
 
 func AnalyzeInformation(c *gin.Context) {

agent/pkg/controllers/user_controller.go

@@ -1,46 +0,0 @@
-package controllers
-
-import (
-	"github.com/up9inc/mizu/agent/pkg/providers"
-
-	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/shared/logger"
-
-	ory "github.com/ory/kratos-client-go"
-)
-
-func Login(c *gin.Context) {
-	if token, err := providers.PerformLogin(c.PostForm("username"), c.PostForm("password"), c.Request.Context()); err != nil {
-		c.AbortWithStatusJSON(401, gin.H{"error": "bad login"})
-	} else {
-		c.JSON(200, gin.H{"token": token})
-	}
-}
-
-func Logout(c *gin.Context) {
-	token := c.GetHeader("x-session-token")
-	if err := providers.Logout(token, c.Request.Context()); err != nil {
-		c.AbortWithStatusJSON(500, gin.H{"error": "error occured while logging out, the session might still be valid"})
-	} else {
-		c.JSON(200, "")
-	}
-}
-
-func Register(c *gin.Context) {
-	token, _, err, formErrorMessages := providers.RegisterUser(c.PostForm("username"), c.PostForm("password"), c.Request.Context())
-	handleRegistration(token, err, formErrorMessages, c)
-}
-
-func handleRegistration(token *string, err error, formErrorMessages map[string][]ory.UiText, c *gin.Context) {
-	if err != nil {
-		if formErrorMessages != nil {
-			logger.Log.Infof("user attempted to register but had form errors %v %v", formErrorMessages, err)
-			c.AbortWithStatusJSON(400, formErrorMessages)
-		} else {
-			logger.Log.Errorf("unknown internal error registering user %s", err)
-			c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while registering"})
-		}
-	} else {
-		c.JSON(201, gin.H{"token": token})
-	}
-}

agent/pkg/database/main.go

@@ -0,0 +1,78 @@
+package database
+
+import (
+	"fmt"
+	"mizuserver/pkg/utils"
+	"time"
+
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+const (
+	OrderDesc  = "desc"
+	OrderAsc   = "asc"
+	LT         = "lt"
+	GT         = "gt"
+	TimeFormat = "2006-01-02 15:04:05.000000000"
+)
+
+var (
+	DB                      *gorm.DB
+	IsDBLocked              = false
+	OperatorToSymbolMapping = map[string]string{
+		LT: "<",
+		GT: ">",
+	}
+	OperatorToOrderMapping = map[string]string{
+		LT: OrderDesc,
+		GT: OrderAsc,
+	}
+)
+
+var DBPath string
+
+func GetEntriesTable() *gorm.DB {
+	return DB.Table("mizu_entries")
+}
+
+func CreateEntry(entry *tapApi.MizuEntry) {
+	if IsDBLocked {
+		return
+	}
+	GetEntriesTable().Create(entry)
+}
+
+func InitDataBase(databasePath string) *gorm.DB {
+	DBPath = databasePath
+	DB, _ = gorm.Open(sqlite.Open(databasePath), &gorm.Config{
+		Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond},
+	})
+	_ = DB.AutoMigrate(&tapApi.MizuEntry{}) // this will ensure table is created
+	go StartEnforcingDatabaseSize()
+	return DB
+}
+
+func GetEntriesFromDb(timeFrom time.Time, timeTo time.Time, protocolName *string) []tapApi.MizuEntry {
+	order := OrderDesc
+	protocolNameCondition := "1 = 1"
+	if protocolName != nil {
+		protocolNameCondition = fmt.Sprintf("protocolName = '%s'", *protocolName)
+	}
+
+	var entries []tapApi.MizuEntry
+	GetEntriesTable().
+		Where(protocolNameCondition).
+		Where(fmt.Sprintf("created_at BETWEEN '%s' AND '%s'", timeFrom.Format(TimeFormat), timeTo.Format(TimeFormat))).
+		Order(fmt.Sprintf("timestamp %s", order)).
+		Find(&entries)
+
+	if len(entries) > 0 {
+		// the entries always order from oldest to newest so we should revers
+		utils.ReverseSlice(entries)
+	}
+	return entries
+}

agent/pkg/database/size_enforcer.go

@@ -0,0 +1,102 @@
+package database
+
+import (
+	"mizuserver/pkg/config"
+	"os"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/up9inc/mizu/shared/debounce"
+	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/shared/units"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+const percentageOfMaxSizeBytesToPrune = 15
+
+func StartEnforcingDatabaseSize() {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		logger.Log.Fatalf("Error creating filesystem watcher for db size enforcement: %v\n", err)
+		return
+	}
+
+	checkFileSizeDebouncer := debounce.NewDebouncer(5*time.Second, func() {
+		checkFileSize(config.Config.MaxDBSizeBytes)
+	})
+
+	go func() {
+		for {
+			select {
+			case event, ok := <-watcher.Events:
+				if !ok {
+					return // closed channel
+				}
+				if event.Op == fsnotify.Write {
+					checkFileSizeDebouncer.SetOn()
+				}
+			case err, ok := <-watcher.Errors:
+				if !ok {
+					return // closed channel
+				}
+				logger.Log.Errorf("filesystem watcher encountered error:%v", err)
+			}
+		}
+	}()
+
+	err = watcher.Add(DBPath)
+	if err != nil {
+		logger.Log.Fatalf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err)
+	}
+}
+
+func checkFileSize(maxSizeBytes int64) {
+	fileStat, err := os.Stat(DBPath)
+	if err != nil {
+		logger.Log.Errorf("Error checking %s file size: %v", DBPath, err)
+	} else {
+		if fileStat.Size() > maxSizeBytes {
+			pruneOldEntries(fileStat.Size())
+		}
+	}
+}
+
+func pruneOldEntries(currentFileSize int64) {
+	// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
+	IsDBLocked = true
+	defer func() { IsDBLocked = false }()
+
+	amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
+
+	rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()
+	if err != nil {
+		logger.Log.Errorf("Error getting 10000 first db rows: %v", err)
+		return
+	}
+
+	entryIdsToRemove := make([]uint, 0)
+	bytesToBeRemoved := int64(0)
+	for rows.Next() {
+		if bytesToBeRemoved >= amountOfBytesToTrim {
+			break
+		}
+		var entry tapApi.MizuEntry
+		err = DB.ScanRows(rows, &entry)
+		if err != nil {
+			logger.Log.Errorf("Error scanning db row: %v", err)
+			continue
+		}
+
+		entryIdsToRemove = append(entryIdsToRemove, entry.ID)
+		bytesToBeRemoved += int64(entry.EstimatedSizeBytes)
+	}
+
+	if len(entryIdsToRemove) > 0 {
+		GetEntriesTable().Where(entryIdsToRemove).Delete(tapApi.MizuEntry{})
+		// VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this
+		DB.Exec("VACUUM")
+		logger.Log.Errorf("Removed %d rows and cleared %s", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
+	} else {
+		logger.Log.Error("Found no rows to remove when pruning")
+	}
+}

agent/pkg/elastic/esClient.go

@@ -1,120 +0,0 @@
-package elastic
-
-import (
-	"bytes"
-	"crypto/tls"
-	"encoding/json"
-	"github.com/elastic/go-elasticsearch/v7"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/up9inc/mizu/tap/api"
-	"net/http"
-	"sync"
-	"time"
-)
-
-type client struct {
-	es            *elasticsearch.Client
-	index         string
-	insertedCount int
-}
-
-var instance *client
-var once sync.Once
-
-func GetInstance() *client {
-	once.Do(func() {
-		instance = newClient()
-	})
-	return instance
-}
-
-func (client *client) Configure(config shared.ElasticConfig) {
-	if config.Url == "" || config.User == "" || config.Password == "" {
-		logger.Log.Infof("No elastic configuration was supplied, elastic exporter disabled")
-		return
-	}
-	transport := http.DefaultTransport
-	tlsClientConfig := &tls.Config{InsecureSkipVerify: true}
-	transport.(*http.Transport).TLSClientConfig = tlsClientConfig
-	cfg := elasticsearch.Config{
-		Addresses: []string{config.Url},
-		Username:  config.User,
-		Password:  config.Password,
-		Transport: transport,
-	}
-
-	es, err := elasticsearch.NewClient(cfg)
-	if err != nil {
-		logger.Log.Fatalf("Failed to initialize elastic client %v", err)
-	}
-
-	// Have the client instance return a response
-	res, err := es.Info()
-	if err != nil {
-		logger.Log.Fatalf("Elastic client.Info() ERROR: %v", err)
-	} else {
-		client.es = es
-		client.index = "mizu_traffic_http_" + time.Now().Format("2006_01_02_15_04")
-		client.insertedCount = 0
-		logger.Log.Infof("Elastic client configured, index: %s, cluster info: %v", client.index, res)
-	}
-	defer res.Body.Close()
-}
-
-func newClient() *client {
-	return &client{
-		es:    nil,
-		index: "",
-	}
-}
-
-type httpEntry struct {
-	Source      *api.TCP               `json:"src"`
-	Destination *api.TCP               `json:"dst"`
-	Outgoing    bool                   `json:"outgoing"`
-	CreatedAt   time.Time              `json:"createdAt"`
-	Request     map[string]interface{} `json:"request"`
-	Response    map[string]interface{} `json:"response"`
-	Summary     string                 `json:"summary"`
-	Method      string                 `json:"method"`
-	Status      int                    `json:"status"`
-	ElapsedTime int64                  `json:"elapsedTime"`
-	Path        string                 `json:"path"`
-}
-
-func (client *client) PushEntry(entry *api.Entry) {
-	if client.es == nil {
-		return
-	}
-
-	if entry.Protocol.Name != "http" {
-		return
-	}
-
-	entryToPush := httpEntry{
-		Source:      entry.Source,
-		Destination: entry.Destination,
-		Outgoing:    entry.Outgoing,
-		CreatedAt:   entry.StartTime,
-		Request:     entry.Request,
-		Response:    entry.Response,
-		Summary:     entry.Summary,
-		Method:      entry.Method,
-		Status:      entry.Status,
-		ElapsedTime: entry.ElapsedTime,
-		Path:        entry.Path,
-	}
-
-	entryJson, err := json.Marshal(entryToPush)
-	if err != nil {
-		logger.Log.Errorf("json.Marshal ERROR: %v", err)
-		return
-	}
-	var buffer bytes.Buffer
-	buffer.WriteString(string(entryJson))
-	res, _ := client.es.Index(client.index, &buffer)
-	if res.StatusCode == 201 {
-		client.insertedCount += 1
-	}
-}

agent/pkg/har/types.go

@@ -1,375 +0,0 @@
-package har
-
-import (
-	"encoding/base64"
-	"github.com/up9inc/mizu/shared/logger"
-	"time"
-	"unicode/utf8"
-)
-
-/*
-HTTP Archive (HAR) format
-https://w3c.github.io/web-performance/specs/HAR/Overview.html
-*/
-
-// HAR is a container type for deserialization
-type HAR struct {
-	Log Log `json:"log"`
-}
-
-// Log represents the root of the exported data. This object MUST be present and its name MUST be "log".
-type Log struct {
-	// The object contains the following name/value pairs:
-
-	// Required. Version number of the format.
-	Version string `json:"version"`
-	// Required. An object of type creator that contains the name and version
-	// information of the log creator application.
-	Creator Creator `json:"creator"`
-	// Optional. An object of type browser that contains the name and version
-	// information of the user agent.
-	Browser Browser `json:"browser"`
-	// Optional. An array of objects of type page, each representing one exported
-	// (tracked) page. Leave out this field if the application does not support
-	// grouping by pages.
-	Pages []Page `json:"pages,omitempty"`
-	// Required. An array of objects of type entry, each representing one
-	// exported (tracked) HTTP request.
-	Entries []Entry `json:"entries"`
-	// Optional. A comment provided by the user or the application. Sorting
-	// entries by startedDateTime (starting from the oldest) is preferred way how
-	// to export data since it can make importing faster. However the reader
-	// application should always make sure the array is sorted (if required for
-	// the import).
-	Comment string `json:"comment"`
-}
-
-// Creator contains information about the log creator application
-type Creator struct {
-	// Required. The name of the application that created the log.
-	Name string `json:"name"`
-	// Required. The version number of the application that created the log.
-	Version string `json:"version"`
-	// Optional. A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// Browser that created the log
-type Browser struct {
-	// Required. The name of the browser that created the log.
-	Name string `json:"name"`
-	// Required. The version number of the browser that created the log.
-	Version string `json:"version"`
-	// Optional. A comment provided by the user or the browser.
-	Comment string `json:"comment"`
-}
-
-// Page object for every exported web page and one <entry> object for every HTTP request.
-// In case when an HTTP trace tool isn't able to group requests by a page,
-// the <pages> object is empty and individual requests doesn't have a parent page.
-type Page struct {
-	/* There is one <page> object for every exported web page and one <entry>
-	object for every HTTP request. In case when an HTTP trace tool isn't able to
-	group requests by a page, the <pages> object is empty and individual
-	requests doesn't have a parent page.
-	*/
-
-	// Date and time stamp for the beginning of the page load
-	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00).
-	StartedDateTime string `json:"startedDateTime"`
-	// Unique identifier of a page within the . Entries use it to refer the parent page.
-	ID string `json:"id"`
-	// Page title.
-	Title string `json:"title"`
-	// Detailed timing info about page load.
-	PageTiming PageTiming `json:"pageTiming"`
-	// (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// PageTiming describes timings for various events (states) fired during the page load.
-// All times are specified in milliseconds. If a time info is not available appropriate field is set to -1.
-type PageTiming struct {
-	// Content of the page loaded. Number of milliseconds since page load started
-	// (page.startedDateTime). Use -1 if the timing does not apply to the current
-	// request.
-	// Depeding on the browser, onContentLoad property represents DOMContentLoad
-	// event or document.readyState == interactive.
-	OnContentLoad int `json:"onContentLoad"`
-	// Page is loaded (onLoad event fired). Number of milliseconds since page
-	// load started (page.startedDateTime). Use -1 if the timing does not apply
-	// to the current request.
-	OnLoad int `json:"onLoad"`
-	// (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment"`
-}
-
-// Entry is a unique, optional Reference to the parent page.
-// Leave out this field if the application does not support grouping by pages.
-type Entry struct {
-	Pageref string `json:"pageref,omitempty"`
-	// Date and time stamp of the request start
-	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD).
-	StartedDateTime string `json:"startedDateTime"`
-	// Total elapsed time of the request in milliseconds. This is the sum of all
-	// timings available in the timings object (i.e. not including -1 values) .
-	Time int `json:"time"`
-	// Detailed info about the request.
-	Request Request `json:"request"`
-	// Detailed info about the response.
-	Response Response `json:"response"`
-	// Info about cache usage.
-	Cache Cache `json:"cache"`
-	// Detailed timing info about request/response round trip.
-	PageTimings PageTimings `json:"pageTimings"`
-	// optional (new in 1.2) IP address of the server that was connected
-	// (result of DNS resolution).
-	ServerIPAddress string `json:"serverIPAddress,omitempty"`
-	// optional (new in 1.2) Unique ID of the parent TCP/IP connection, can be
-	// the client port number. Note that a port number doesn't have to be unique
-	// identifier in cases where the port is shared for more connections. If the
-	// port isn't available for the application, any other unique connection ID
-	// can be used instead (e.g. connection index). Leave out this field if the
-	// application doesn't support this info.
-	Connection string `json:"connection,omitempty"`
-	// (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// Request contains detailed info about performed request.
-type Request struct {
-	// Request method (GET, POST, ...).
-	Method string `json:"method"`
-	// Absolute URL of the request (fragments are not included).
-	URL string `json:"url"`
-	// Request HTTP Version.
-	HTTPVersion string `json:"httpVersion"`
-	// List of cookie objects.
-	Cookies []Cookie `json:"cookies"`
-	// List of header objects.
-	Headers []NVP `json:"headers"`
-	// List of query parameter objects.
-	QueryString []NVP `json:"queryString"`
-	// Posted data.
-	PostData PostData `json:"postData"`
-	// Total number of bytes from the start of the HTTP request message until
-	// (and including) the double CRLF before the body. Set to -1 if the info
-	// is not available.
-	HeaderSize int `json:"headerSize"`
-	// Size of the request body (POST data payload) in bytes. Set to -1 if the
-	// info is not available.
-	BodySize int `json:"bodySize"`
-	// (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment"`
-}
-
-// Response contains detailed info about the response.
-type Response struct {
-	// Response status.
-	Status int `json:"status"`
-	// Response status description.
-	StatusText string `json:"statusText"`
-	// Response HTTP Version.
-	HTTPVersion string `json:"httpVersion"`
-	// List of cookie objects.
-	Cookies []Cookie `json:"cookies"`
-	// List of header objects.
-	Headers []NVP `json:"headers"`
-	// Details about the response body.
-	Content Content `json:"content"`
-	// Redirection target URL from the Location response header.
-	RedirectURL string `json:"redirectURL"`
-	// Total number of bytes from the start of the HTTP response message until
-	// (and including) the double CRLF before the body. Set to -1 if the info is
-	// not available.
-	// The size of received response-headers is computed only from headers that
-	// are really received from the server. Additional headers appended by the
-	// browser are not included in this number, but they appear in the list of
-	// header objects.
-	HeadersSize int `json:"headersSize"`
-	// Size of the received response body in bytes. Set to zero in case of
-	// responses coming from the cache (304). Set to -1 if the info is not
-	// available.
-	BodySize int `json:"bodySize"`
-	// optional (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// Cookie contains list of all cookies (used in <request> and <response> objects).
-type Cookie struct {
-	// The name of the cookie.
-	Name string `json:"name"`
-	// The cookie value.
-	Value string `json:"value"`
-	// optional The path pertaining to the cookie.
-	Path string `json:"path,omitempty"`
-	// optional The host of the cookie.
-	Domain string `json:"domain,omitempty"`
-	// optional Cookie expiration time.
-	// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.123+02:00).
-	Expires string `json:"expires,omitempty"`
-	// optional Set to true if the cookie is HTTP only, false otherwise.
-	HTTPOnly bool `json:"httpOnly,omitempty"`
-	// optional (new in 1.2) True if the cookie was transmitted over ssl, false
-	// otherwise.
-	Secure bool `json:"secure,omitempty"`
-	// optional (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// NVP is simply a name/value pair with a comment
-type NVP struct {
-	Name    string `json:"name"`
-	Value   string `json:"value"`
-	Comment string `json:"comment,omitempty"`
-}
-
-// PostData describes posted data, if any (embedded in <request> object).
-type PostData struct {
-	//  Mime type of posted data.
-	MimeType string `json:"mimeType"`
-	//  List of posted parameters (in case of URL encoded parameters).
-	Params []PostParam `json:"params"`
-	//  Plain text posted data
-	Text string `json:"text"`
-	// optional (new in 1.2) A comment provided by the user or the
-	// application.
-	Comment string `json:"comment,omitempty"`
-}
-
-func (d PostData) B64Decoded() (bool, []byte, string) {
-	// there is a weird gap in HAR spec 1.2, that does not define encoding for binary POST bodies
-	// we have own convention of putting `base64` into comment field to handle it similar to response `Content`
-	return b64Decoded(d.Comment, d.Text)
-}
-
-// PostParam is a list of posted parameters, if any (embedded in <postData> object).
-type PostParam struct {
-	// name of a posted parameter.
-	Name string `json:"name"`
-	// optional value of a posted parameter or content of a posted file.
-	Value string `json:"value,omitempty"`
-	// optional name of a posted file.
-	FileName string `json:"fileName,omitempty"`
-	// optional content type of a posted file.
-	ContentType string `json:"contentType,omitempty"`
-	// optional (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// Content describes details about response content (embedded in <response> object).
-type Content struct {
-	// Length of the returned content in bytes. Should be equal to
-	// response.bodySize if there is no compression and bigger when the content
-	// has been compressed.
-	Size int `json:"size"`
-	// optional Number of bytes saved. Leave out this field if the information
-	// is not available.
-	Compression int `json:"compression,omitempty"`
-	// MIME type of the response text (value of the Content-Type response
-	// header). The charset attribute of the MIME type is included (if
-	// available).
-	MimeType string `json:"mimeType"`
-	// optional Response body sent from the server or loaded from the browser
-	// cache. This field is populated with textual content only. The text field
-	// is either HTTP decoded text or a encoded (e.g. "base64") representation of
-	// the response body. Leave out this field if the information is not
-	// available.
-	Text string `json:"text,omitempty"`
-	// optional (new in 1.2) Encoding used for response text field e.g
-	// "base64". Leave out this field if the text field is HTTP decoded
-	// (decompressed & unchunked), than trans-coded from its original character
-	// set into UTF-8.
-	Encoding string `json:"encoding,omitempty"`
-	// optional (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-func (c Content) B64Decoded() (bool, []byte, string) {
-	return b64Decoded(c.Encoding, c.Text)
-}
-
-func b64Decoded(enc string, text string) (isBinary bool, asBytes []byte, asString string) {
-	if enc == "base64" {
-		decoded, err := base64.StdEncoding.DecodeString(text)
-		if err != nil {
-			logger.Log.Warningf("Failed to decode content as base64: %s", text)
-			return false, []byte(text), text
-		}
-		valid := utf8.Valid(decoded)
-		return !valid, decoded, string(decoded)
-	} else {
-		return false, nil, text
-	}
-}
-
-// Cache contains info about a request coming from browser cache.
-type Cache struct {
-	// optional State of a cache entry before the request. Leave out this field
-	// if the information is not available.
-	BeforeRequest CacheObject `json:"beforeRequest,omitempty"`
-	// optional State of a cache entry after the request. Leave out this field if
-	// the information is not available.
-	AfterRequest CacheObject `json:"afterRequest,omitempty"`
-	// optional (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// CacheObject is used by both beforeRequest and afterRequest
-type CacheObject struct {
-	// optional - Expiration time of the cache entry.
-	Expires string `json:"expires,omitempty"`
-	// The last time the cache entry was opened.
-	LastAccess string `json:"lastAccess"`
-	// Etag
-	ETag string `json:"eTag"`
-	// The number of times the cache entry has been opened.
-	HitCount int `json:"hitCount"`
-	// optional (new in 1.2) A comment provided by the user or the application.
-	Comment string `json:"comment,omitempty"`
-}
-
-// PageTimings describes various phases within request-response round trip.
-// All times are specified in milliseconds.
-type PageTimings struct {
-	Blocked int `json:"blocked,omitempty"`
-	// optional - Time spent in a queue waiting for a network connection. Use -1
-	// if the timing does not apply to the current request.
-	DNS int `json:"dns,omitempty"`
-	// optional - DNS resolution time. The time required to resolve a host name.
-	// Use -1 if the timing does not apply to the current request.
-	Connect int `json:"connect,omitempty"`
-	// optional - Time required to create TCP connection. Use -1 if the timing
-	// does not apply to the current request.
-	Send int `json:"send"`
-	// Time required to send HTTP request to the server.
-	Wait int `json:"wait"`
-	// Waiting for a response from the server.
-	Receive int `json:"receive"`
-	// Time required to read entire response from the server (or cache).
-	Ssl int `json:"ssl,omitempty"`
-	// optional (new in 1.2) - Time required for SSL/TLS negotiation. If this
-	// field is defined then the time is also included in the connect field (to
-	// ensure backward compatibility with HAR 1.1). Use -1 if the timing does not
-	// apply to the current request.
-	Comment string `json:"comment,omitempty"`
-	// optional (new in 1.2) - A comment provided by the user or the application.
-}
-
-// TestResult contains results for an individual HTTP request
-type TestResult struct {
-	URL       string    `json:"url"`
-	Status    int       `json:"status"` // 200, 500, etc.
-	StartTime time.Time `json:"startTime"`
-	EndTime   time.Time `json:"endTime"`
-	Latency   int       `json:"latency"` // milliseconds
-	Method    string    `json:"method"`
-	HarFile   string    `json:"harfile"`
-}
-
-// aliases for martian lib compatibility
-
-type Header = NVP
-type QueryString = NVP
-type Param = PostParam
-type Timings = PageTimings

agent/pkg/har/types_test.go

@@ -1,38 +0,0 @@
-package har
-
-import "testing"
-
-func TestContentEncoded(t *testing.T) {
-	testCases := []struct {
-		text        string
-		isBinary    bool
-		expectedStr string
-		binaryLen   int
-	}{
-		{"not-base64", false, "not-base64", 10},
-		{"dGVzdA==", false, "test", 4},
-		{"test", true, "\f@A", 3}, // valid UTF-8 with some non-printable chars
-		{"IsDggPCAgPiAgID8gICAgN/vv/e/v/u/v7/9v7+/vyIKIu+3kO+3ke+3ku+3k++3lO+3le+3lu+3l++3mO+3me+3mu+3m++3nO+3ne+3nu+3n++3oO+3oe+3ou+3o++3pO+3pe+3pu+3p++3qO+3qe+3qu+3q++3rO+3re+3ru+3ryIK", true, "test", 132}, // invalid UTF-8 (thus binary), taken from https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
-	}
-
-	for _, tc := range testCases {
-		c := Content{
-			Encoding: "base64",
-			Text:     tc.text,
-		}
-		isBinary, asBytes, asString := c.B64Decoded()
-		_ = asBytes
-
-		if tc.isBinary != isBinary {
-			t.Errorf("Binary flag mismatch: %t != %t", tc.isBinary, isBinary)
-		}
-
-		if !isBinary && tc.expectedStr != asString {
-			t.Errorf("Decode value mismatch: %s != %s", tc.expectedStr, asString)
-		}
-
-		if tc.binaryLen != len(asBytes) {
-			t.Errorf("Binary len mismatch: %d != %d", tc.binaryLen, len(asBytes))
-		}
-	}
-}

agent/pkg/holder/main.go

@@ -1,6 +1,6 @@
 package holder
 
-import "github.com/up9inc/mizu/agent/pkg/resolver"
+import "mizuserver/pkg/resolver"
 
 var k8sResolver *resolver.Resolver
 
@@ -11,3 +11,4 @@ func SetResolver(param *resolver.Resolver) {
 func GetResolver() *resolver.Resolver {
 	return k8sResolver
 }
+

agent/pkg/middlewares/auth.go

@@ -1,73 +0,0 @@
-package middlewares
-
-import (
-	"github.com/up9inc/mizu/agent/pkg/config"
-	"github.com/up9inc/mizu/agent/pkg/providers"
-
-	"github.com/gin-gonic/gin"
-	ory "github.com/ory/kratos-client-go"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-func RequiresAuth() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		// auth is irrelevant for ephermeral mizu
-		if !config.Config.StandaloneMode {
-			c.Next()
-			return
-		}
-
-		verifyKratosSessionForRequest(c)
-		if !c.IsAborted() {
-			c.Next()
-		}
-	}
-}
-
-func RequiresAdmin() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		// auth is irrelevant for ephermeral mizu
-		if !config.Config.StandaloneMode {
-			c.Next()
-			return
-		}
-
-		session := verifyKratosSessionForRequest(c)
-		if c.IsAborted() {
-			return
-		}
-
-		traits := session.Identity.Traits.(map[string]interface{})
-		username := traits["username"].(string)
-
-		isAdmin, err := providers.CheckIfUserHasSystemRole(username, providers.AdminRole)
-		if err != nil {
-			logger.Log.Errorf("error checking user role %v", err)
-			c.AbortWithStatusJSON(403, gin.H{"error": "unknown auth error occured"})
-		} else if !isAdmin {
-			logger.Log.Warningf("user %s attempted to call an admin only endpoint with insufficient privileges", username)
-			c.AbortWithStatusJSON(403, gin.H{"error": "unauthorized"})
-		} else {
-			c.Next()
-		}
-	}
-}
-
-func verifyKratosSessionForRequest(c *gin.Context) *ory.Session {
-	token := c.GetHeader("x-session-token")
-	if token == "" {
-		c.AbortWithStatusJSON(401, gin.H{"error": "token header is empty"})
-		return nil
-	}
-
-	if session, err := providers.VerifyToken(token, c.Request.Context()); err != nil {
-		logger.Log.Errorf("error verifying token %v", err)
-		c.AbortWithStatusJSON(401, gin.H{"error": "unknown auth error occured"})
-		return nil
-	} else if session == nil {
-		c.AbortWithStatusJSON(401, gin.H{"error": "invalid token"})
-		return nil
-	} else {
-		return session
-	}
-}

agent/pkg/middlewares/cors.go

@@ -1,19 +0,0 @@
-package middlewares
-
-import "github.com/gin-gonic/gin"
-
-func CORSMiddleware() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
-		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
-		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With, x-session-token")
-		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")
-
-		if c.Request.Method == "OPTIONS" {
-			c.AbortWithStatus(204)
-			return
-		}
-
-		c.Next()
-	}
-}

agent/pkg/models/models.go

@@ -3,43 +3,28 @@ package models
 import (
 	"encoding/json"
 
-	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/agent/pkg/rules"
 	tapApi "github.com/up9inc/mizu/tap/api"
 
-	basenine "github.com/up9inc/basenine/client/go"
+	"mizuserver/pkg/rules"
+
+	"github.com/google/martian/har"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
 )
 
-func GetEntry(r *tapApi.Entry, v tapApi.DataUnmarshaler) error {
+func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error {
 	return v.UnmarshalData(r)
 }
 
-type TapConfig struct {
-	TappedNamespaces map[string]bool `json:"tappedNamespaces"`
-}
-
-type EntriesRequest struct {
-	LeftOff   int    `form:"leftOff" validate:"required,min=-1"`
-	Direction int    `form:"direction" validate:"required,oneof='1' '-1'"`
-	Query     string `form:"query"`
-	Limit     int    `form:"limit" validate:"required,min=1"`
-	TimeoutMs int    `form:"timeoutMs" validate:"min=1"`
-}
-
-type SingleEntryRequest struct {
-	Query string `form:"query"`
-}
-
-type EntriesResponse struct {
-	Data []interface{}      `json:"data"`
-	Meta *basenine.Metadata `json:"meta"`
+type EntriesFilter struct {
+	Limit     int    `form:"limit" validate:"required,min=1,max=200"`
+	Operator  string `form:"operator" validate:"required,oneof='lt' 'gt'"`
+	Timestamp int64  `form:"timestamp" validate:"required,min=1"`
 }
 
 type WebSocketEntryMessage struct {
 	*shared.WebSocketMessageMetadata
-	Data *tapApi.BaseEntry `json:"data,omitempty"`
+	Data *tapApi.BaseEntryDetails `json:"data,omitempty"`
 }
 
 type WebSocketTappedEntryMessage struct {
@@ -57,28 +42,7 @@ type AuthStatus struct {
 	Model string `json:"model"`
 }
 
-type ToastMessage struct {
-	Type      string `json:"type"`
-	AutoClose uint   `json:"autoClose"`
-	Text      string `json:"text"`
-}
-
-type WebSocketToastMessage struct {
-	*shared.WebSocketMessageMetadata
-	Data *ToastMessage `json:"data,omitempty"`
-}
-
-type WebSocketQueryMetadataMessage struct {
-	*shared.WebSocketMessageMetadata
-	Data *basenine.Metadata `json:"data,omitempty"`
-}
-
-type WebSocketStartTimeMessage struct {
-	*shared.WebSocketMessageMetadata
-	Data int64 `json:"data"`
-}
-
-func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) {
+func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntryDetails) ([]byte, error) {
 	message := &WebSocketEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
 			MessageType: shared.WebSocketMessageTypeEntry,
@@ -108,36 +72,6 @@ func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error)
 	return json.Marshal(message)
 }
 
-func CreateWebsocketToastMessage(base *ToastMessage) ([]byte, error) {
-	message := &WebSocketToastMessage{
-		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
-			MessageType: shared.WebSocketMessageTypeToast,
-		},
-		Data: base,
-	}
-	return json.Marshal(message)
-}
-
-func CreateWebsocketQueryMetadataMessage(base *basenine.Metadata) ([]byte, error) {
-	message := &WebSocketQueryMetadataMessage{
-		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
-			MessageType: shared.WebSocketMessageTypeQueryMetadata,
-		},
-		Data: base,
-	}
-	return json.Marshal(message)
-}
-
-func CreateWebsocketStartTimeMessage(base int64) ([]byte, error) {
-	message := &WebSocketStartTimeMessage{
-		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
-			MessageType: shared.WebSocketMessageTypeStartTime,
-		},
-		Data: base,
-	}
-	return json.Marshal(message)
-}
-
 // ExtendedHAR is the top level object of a HAR log.
 type ExtendedHAR struct {
 	Log *ExtendedLog `json:"log"`
@@ -163,7 +97,3 @@ func RunValidationRulesState(harEntry har.Entry, service string) (tapApi.Applica
 	statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend)
 	return tapApi.ApplicableRules{Status: statusPolicyToSend, Latency: latency, NumberOfRules: numberOfRules}, resultPolicyToSend, isEnabled
 }
-
-type InstallState struct {
-	Completed bool `json:"completed"`
-}

agent/pkg/oas/counters.go

@@ -1,119 +0,0 @@
-package oas
-
-import (
-	"fmt"
-	"github.com/chanced/openapi"
-	"math"
-	"strings"
-)
-
-type Counter struct {
-	Entries     int     `json:"entries"`
-	Failures    int     `json:"failures"`
-	FirstSeen   float64 `json:"firstSeen"`
-	LastSeen    float64 `json:"lastSeen"`
-	SumRT       float64 `json:"sumRT"`
-	SumDuration float64 `json:"sumDuration"`
-}
-
-func (c *Counter) addEntry(ts float64, rt float64, succ bool, dur float64) {
-	if dur < 0 {
-		panic("Duration cannot be negative")
-	}
-
-	c.Entries += 1
-	c.SumRT += rt
-	c.SumDuration += dur
-	if !succ {
-		c.Failures += 1
-	}
-
-	if c.FirstSeen == 0 {
-		c.FirstSeen = ts
-	} else {
-		c.FirstSeen = math.Min(c.FirstSeen, ts)
-	}
-
-	c.LastSeen = math.Max(c.LastSeen, ts)
-}
-
-func (c *Counter) addOther(other *Counter) {
-	c.Entries += other.Entries
-	c.SumRT += other.SumRT
-	c.Failures += other.Failures
-	c.SumDuration += other.SumDuration
-
-	if c.FirstSeen == 0 {
-		c.FirstSeen = other.FirstSeen
-	} else {
-		c.FirstSeen = math.Min(c.FirstSeen, other.FirstSeen)
-	}
-
-	c.LastSeen = math.Max(c.LastSeen, other.LastSeen)
-}
-
-type CounterMap map[string]*Counter
-
-func (m *CounterMap) addOther(other *CounterMap) {
-	for src, cmap := range *other {
-		if existing, ok := (*m)[src]; ok {
-			existing.addOther(cmap)
-		} else {
-			copied := *cmap
-			(*m)[src] = &copied
-		}
-	}
-}
-
-func setCounterMsgIfOk(oldStr string, cnt *Counter) string {
-	tpl := "Mizu observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds"
-	if oldStr == "" || (strings.HasPrefix(oldStr, "Mizu ") && strings.HasSuffix(oldStr, " seconds")) {
-		return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries))
-	}
-	return oldStr
-}
-
-type CounterMaps struct {
-	counterTotal    Counter
-	counterMapTotal CounterMap
-}
-
-func (m *CounterMaps) processOp(opObj *openapi.Operation) error {
-	if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
-		counter := new(Counter)
-		err := opObj.Extensions.DecodeExtension(CountersTotal, counter)
-		if err != nil {
-			return err
-		}
-		m.counterTotal.addOther(counter)
-
-		opObj.Description = setCounterMsgIfOk(opObj.Description, counter)
-	}
-
-	if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
-		counterMap := new(CounterMap)
-		err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap)
-		if err != nil {
-			return err
-		}
-		m.counterMapTotal.addOther(counterMap)
-	}
-	return nil
-}
-
-func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error {
-	if oas.Extensions == nil {
-		oas.Extensions = openapi.Extensions{}
-	}
-
-	err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal)
-	if err != nil {
-		return err
-	}
-
-	err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal)
-	if err != nil {
-		return nil
-	}
-	return nil
-}

agent/pkg/oas/feeder_test.go

@@ -1,209 +0,0 @@
-package oas
-
-import (
-	"bufio"
-	"encoding/json"
-	"errors"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-	"testing"
-
-	"github.com/up9inc/mizu/agent/pkg/har"
-
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-func getFiles(baseDir string) (result []string, err error) {
-	result = make([]string, 0)
-	logger.Log.Infof("Reading files from tree: %s", baseDir)
-
-	inputs := []string{baseDir}
-
-	// https://yourbasic.org/golang/list-files-in-directory/
-	visitor := func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if info.Mode()&os.ModeSymlink != 0 {
-			path, _ = os.Readlink(path)
-			inputs = append(inputs, path)
-			return nil
-		}
-
-		ext := strings.ToLower(filepath.Ext(path))
-		if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
-			result = append(result, path)
-		}
-
-		return nil
-	}
-
-	for len(inputs) > 0 {
-		path := inputs[0]
-		inputs = inputs[1:]
-		err = filepath.Walk(path, visitor)
-	}
-
-	sort.SliceStable(result, func(i, j int) bool {
-		return fileSize(result[i]) < fileSize(result[j])
-	})
-
-	logger.Log.Infof("Got files: %d", len(result))
-	return result, err
-}
-
-func fileSize(fname string) int64 {
-	fi, err := os.Stat(fname)
-	if err != nil {
-		panic(err)
-	}
-
-	return fi.Size()
-}
-
-func feedEntries(fromFiles []string, isSync bool) (count int, err error) {
-	badFiles := make([]string, 0)
-	cnt := 0
-	for _, file := range fromFiles {
-		logger.Log.Info("Processing file: " + file)
-		ext := strings.ToLower(filepath.Ext(file))
-		eCnt := 0
-		switch ext {
-		case ".har":
-			eCnt, err = feedFromHAR(file, isSync)
-			if err != nil {
-				logger.Log.Warning("Failed processing file: " + err.Error())
-				badFiles = append(badFiles, file)
-				continue
-			}
-		case ".ldjson":
-			eCnt, err = feedFromLDJSON(file, isSync)
-			if err != nil {
-				logger.Log.Warning("Failed processing file: " + err.Error())
-				badFiles = append(badFiles, file)
-				continue
-			}
-		default:
-			return 0, errors.New("Unsupported file extension: " + ext)
-		}
-		cnt += eCnt
-	}
-
-	for _, f := range badFiles {
-		logger.Log.Infof("Bad file: %s", f)
-	}
-
-	return cnt, nil
-}
-
-func feedFromHAR(file string, isSync bool) (int, error) {
-	fd, err := os.Open(file)
-	if err != nil {
-		panic(err)
-	}
-
-	defer fd.Close()
-
-	data, err := ioutil.ReadAll(fd)
-	if err != nil {
-		return 0, err
-	}
-
-	var harDoc har.HAR
-	err = json.Unmarshal(data, &harDoc)
-	if err != nil {
-		return 0, err
-	}
-
-	cnt := 0
-	for _, entry := range harDoc.Log.Entries {
-		cnt += 1
-		feedEntry(&entry, "", isSync, file)
-	}
-
-	return cnt, nil
-}
-
-func feedEntry(entry *har.Entry, source string, isSync bool, file string) {
-	entry.Comment = file
-	if entry.Response.Status == 302 {
-		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
-	}
-
-	if strings.Contains(entry.Request.URL, "some") { // for debugging
-		logger.Log.Debugf("Interesting: %s", entry.Request.URL)
-	}
-
-	ews := EntryWithSource{Entry: *entry, Source: source, Id: uint(0)}
-	if isSync {
-		GetOasGeneratorInstance().entriesChan <- ews // blocking variant, right?
-	} else {
-		GetOasGeneratorInstance().PushEntry(&ews)
-	}
-}
-
-func feedFromLDJSON(file string, isSync bool) (int, error) {
-	fd, err := os.Open(file)
-	if err != nil {
-		panic(err)
-	}
-
-	defer fd.Close()
-
-	reader := bufio.NewReader(fd)
-
-	var meta map[string]interface{}
-	buf := strings.Builder{}
-	cnt := 0
-	source := ""
-	for {
-		substr, isPrefix, err := reader.ReadLine()
-		if err == io.EOF {
-			break
-		}
-
-		buf.WriteString(string(substr))
-		if isPrefix {
-			continue
-		}
-
-		line := buf.String()
-		buf.Reset()
-
-		if meta == nil {
-			err := json.Unmarshal([]byte(line), &meta)
-			if err != nil {
-				return 0, err
-			}
-			if s, ok := meta["_source"]; ok && s != nil {
-				source = s.(string)
-			}
-		} else {
-			var entry har.Entry
-			err := json.Unmarshal([]byte(line), &entry)
-			if err != nil {
-				logger.Log.Warningf("Failed decoding entry: %s", line)
-			} else {
-				cnt += 1
-				feedEntry(&entry, source, isSync, file)
-			}
-		}
-	}
-
-	return cnt, nil
-}
-
-func TestFilesList(t *testing.T) {
-	res, err := getFiles("./test_artifacts/")
-	t.Log(len(res))
-	t.Log(res)
-	if err != nil || len(res) != 3 {
-		t.Logf("Should return 2 files but returned %d", len(res))
-		t.FailNow()
-	}
-}

agent/pkg/oas/gibberish.go

@@ -1,185 +0,0 @@
-package oas
-
-import (
-	"math"
-	"regexp"
-	"strings"
-	"unicode"
-)
-
-var (
-	patUuid4    = regexp.MustCompile(`(?i)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
-	patEmail    = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
-	patLongNum  = regexp.MustCompile(`^\d{3,}$`)
-	patLongNumB = regexp.MustCompile(`[^\d]\d{3,}`)
-	patLongNumA = regexp.MustCompile(`\d{3,}[^\d]`)
-)
-
-func IsGibberish(str string) bool {
-	if IsVersionString(str) {
-		return false
-	}
-
-	if patEmail.MatchString(str) {
-		return true
-	}
-
-	if patUuid4.MatchString(str) {
-		return true
-	}
-
-	if patLongNum.MatchString(str) || patLongNumB.MatchString(str) || patLongNumA.MatchString(str) {
-		return true
-	}
-
-	//alNum := cleanStr(str, isAlNumRune)
-	//alpha := cleanStr(str, isAlphaRune)
-	// noiseAll := isNoisy(alNum)
-	//triAll := isTrigramBad(strings.ToLower(alpha))
-	// _ = noiseAll
-
-	isNotAlNum := func(r rune) bool { return !isAlNumRune(r) }
-	chunks := strings.FieldsFunc(str, isNotAlNum)
-	noisyLen := 0
-	alnumLen := 0
-	for _, chunk := range chunks {
-		alnumLen += len(chunk)
-		noise := isNoisy(chunk)
-		tri := isTrigramBad(strings.ToLower(chunk))
-		if noise || tri {
-			noisyLen += len(chunk)
-		}
-	}
-
-	return float64(noisyLen) > 0
-
-	//if float64(noisyLen) > 0 {
-	//	return true
-	//}
-
-	//if len(chunks) > 0 && float64(noisyLen) >= float64(alnumLen)/3.0 {
-	//	return true
-	//}
-
-	//if triAll {
-	//return true
-	//}
-
-	// return false
-}
-
-func noiseLevel(str string) (score float64) {
-	// opinionated algo of certain char pairs marking the non-human strings
-	prev := *new(rune)
-	cnt := 0.0
-	for _, char := range str {
-		cnt += 1
-		if prev > 0 {
-			switch {
-			// continued class of upper/lower/digit adds no noise
-			case unicode.IsUpper(prev) && unicode.IsUpper(char):
-			case unicode.IsLower(prev) && unicode.IsLower(char):
-			case unicode.IsDigit(prev) && unicode.IsDigit(char):
-
-			// upper =>
-			case unicode.IsUpper(prev) && unicode.IsLower(char):
-				score += 0.10
-			case unicode.IsUpper(prev) && unicode.IsDigit(char):
-				score += 0.5
-
-			// lower =>
-			case unicode.IsLower(prev) && unicode.IsUpper(char):
-				score += 0.75
-			case unicode.IsLower(prev) && unicode.IsDigit(char):
-				score += 0.5
-
-			// digit =>
-			case unicode.IsDigit(prev) && unicode.IsUpper(char):
-				score += 0.75
-			case unicode.IsDigit(prev) && unicode.IsLower(char):
-				score += 1.0
-
-			// the rest is 100% noise
-			default:
-				score += 1
-			}
-		}
-		prev = char
-	}
-
-	return score
-}
-
-func IsVersionString(component string) bool {
-	if component == "" {
-		return false
-	}
-
-	hasV := false
-	if strings.HasPrefix(component, "v") {
-		component = component[1:]
-		hasV = true
-	}
-
-	for _, c := range component {
-		if string(c) != "." && !unicode.IsDigit(c) {
-			return false
-		}
-	}
-
-	if !hasV && !strings.Contains(component, ".") {
-		return false
-	}
-
-	return true
-}
-
-func trigramScore(str string) (float64, int) {
-	tgScore := 0.0
-	trigrams := ngrams(str, 3)
-	if len(trigrams) > 0 {
-		for _, trigram := range trigrams {
-			score, found := corpus_trigrams[trigram]
-			if found {
-				tgScore += score
-			}
-		}
-	}
-
-	return tgScore, len(trigrams)
-}
-
-func isTrigramBad(s string) bool {
-	tgScore, cnt := trigramScore(s)
-
-	if cnt > 0 {
-		val := math.Sqrt(tgScore) / float64(cnt)
-		val2 := tgScore / float64(cnt)
-		threshold := 0.005
-		bad := val < threshold
-		threshold2 := math.Log(float64(cnt)-2) * 0.1
-		bad2 := val2 < threshold2
-		return bad && bad2 // TODO: improve this logic to be clearer
-	}
-	return false
-}
-
-func isNoisy(s string) bool {
-	noise := noiseLevel(s)
-
-	if len(s) > 0 {
-		val := (noise * noise) / float64(len(s))
-		threshold := 0.6
-		bad := val > threshold
-		return bad
-	}
-	return false
-}
-
-func ngrams(s string, n int) []string {
-	result := make([]string, 0)
-	for i := 0; i < len(s)-n+1; i++ {
-		result = append(result, s[i:i+n])
-	}
-	return result
-}

agent/pkg/oas/gibberish_test.go

@@ -1,212 +0,0 @@
-package oas
-
-import (
-	"testing"
-)
-
-func TestNegative(t *testing.T) {
-	cases := []string{
-		"",
-		"{}",
-		"0.0.29",
-		"0.1",
-		"1.0",
-		"1.0.0",
-		"2.1.73",
-		"abTestV2",
-		"actionText,setName,setAttribute,save,ignore,onEnd,getContext,end,get",
-		"AddUserGroupLink",
-		"advert-management.adBlockerMessage.html",
-		"agents.author.1.json",
-		"animated-gif",
-		"b", // can be valid hexadecimal
-		"big-danger-coronavirus-panic-greater-crisis",
-		"breakout-box",
-		"callback",
-		"core.algorithm_execution.view",
-		"core.devices.view",
-		"data.json",
-		"dialog.overlay.infinity.json",
-		"domain-input",
-		"embeddable", // lul, it's a valid HEX!
-		"embeddable_blip",
-		"E PLURIBUS UNUM",
-		"etc",
-		"eu-central-1a",
-		"fcgi-bin",
-		"footer.include.html",
-		"fullHashes:find",
-		"generate-feed",
-		"GetAds",
-		"GetCart",
-		"GetUniversalVariableUser",
-		"github-audit-exports",
-		"g.js",
-		"g.pixel",
-		".html",
-		"Hugo Michiels",
-		"image.sbix",
-		"index.html",
-		"iPad",
-		"Joanna Mazewski",
-		"LibGit2Sharp",
-		"Michael_Vaughan1.png",
-		"New RSS feed has been generated",
-		"nick-clegg",
-		"opt-out",
-		"pixel_details.html",
-		"post.json",
-		"profile-method-info",
-		"project-id",
-		"publisha.1.json",
-		"publish_and_moderate",
-		"Ronna McDaniel",
-		"rtb-h",
-		"runs",
-		"sign-up",
-		"some-uuid-maybe",
-		"stable-4.0-version.json",
-		"StartUpCheckout",
-		"Steve Flunk",
-		"sync_a9",
-		"Ted Cruz",
-		"test.png",
-		"token",
-		"ToList",
-		"v2.1.3",
-		"VersionCheck.php",
-		"v Rusiji",
-		"Walnut St",
-		"web_widget",
-		"zoom_in.cur",
-		"xray",
-		"web",
-		"vipbets1",
-		"trcc",
-		"fbpixel",
-
-		// TODO below
-		// "tcfv2",
-		// "Matt-cartoon-255x206px-small.png",
-		// "TheTelegraph_portal_white-320-small.png",
-		// "testdata-10kB.js",
-	}
-
-	for _, str := range cases {
-		if IsGibberish(str) {
-			t.Errorf("Mistakenly true: %s", str)
-		}
-	}
-}
-
-func TestPositive(t *testing.T) {
-	cases := []string{
-		"0a0d0174-b338-4520-a1c3-24f7e3d5ec50.html",
-		"1024807212418223",
-		"11ca096cbc224a67360493d44a9903",
-		"1553183382779",
-		"1554507871",
-		"19180481",
-		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
-		"456795af-b48f-4a8d-9b37-3e932622c2f0",
-		"601a2bdcc5b69137248ddbbf",
-		"60fe9aaeaefe2400012df94f",
-		"610bc3fd5a77a7fa25033fb0",
-		"610bd0315a77a7fa25034368",
-		"610bd0315a77a7fa25034368zh",
-		"6120c057c7a97b03f6986f1b",
-		"710a462e",
-		"730970532670-compute@developer.gserviceaccount.com",
-		"819db2242a648b305395537022523d65",
-		"952bea17-3776-11ea-9341-42010a84012a",
-		"a3226860758.html",
-		"AAAA028295945",
-		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4",
-		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
-		"bnjksfd897345nl098asd53412kl98",
-		"c738338322370b47a79251f7510dd",                        // prefixed hex
-		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
-		"css/login.0f48c49a34eb53ea4623.min.css",
-		"d_fLLxlhzDilixeBEimaZ5",
-		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
-		"e8782afc112720300c049ff124434b79",
-		"fb6cjraf9cejut2a",
-		"i-0236530c66ed02200",
-		"JEHJW4BKVFDRTMTUQLHKK5WVAU",
-		"john.dow.1981@protonmail.com",
-		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
-		"MNUTGVFMGLEMFTBH0XSE5E02F6J2DS",
-		"n63nd45qsj",
-		"n9z9QGNiz",
-		"NC4WTmcy",
-		"proxy.3d2100fd7107262ecb55ce6847f01fa5.html",
-		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
-		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
-		"qwerqwerasdfqwer@protonmai.com",
-		"r-ext-5579e00a95c90",
-		"r-ext-5579e8b12f11e",
-		"r-v4-5c92513c9e04a",
-		"r-v4-5c92513c9e04a-0-c.monitoring.json",
-		"segments-1563566437171.639994",
-		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
-		"sp_dxJTfx11_576742227280287872",
-		"sp_NnUPB5wj_601a2bdcc5b69137248ddbbf",
-		"sp_NxITuoE4_premiumchron-article-14302157_c_ryGQBs_r_yIWvwP",
-		"t_52d94268-8810-4a7e-ba87-ffd657a6752f",
-		"timeouts-1563566437171.639994",
-		"u_YPF3GsGKMo02",
-
-		"0000000000 65535 f",
-		"0000000178 00000 n",
-		"0-10000",
-		"01526123,",
-		"0,18168,183955,3,4,1151616,5663,731,223,5104,207,3204,10,1051,175,364,1435,4,60,576,241,383,246,5,1102",
-		"05/10/2020",
-		"14336456724940333",
-		"fb6cjraf9cejut2a",
-		"JEHJW4BKVFDRTMTUQLHKK5WVAU",
-
-		// TODO
-		/*
-			"0,20",
-			"0.001",
-			"YISAtiX1",
-			"Fxvd1timk", // questionable
-			"B4GCSkORAJs",
-			"D_4EDAqenHQ",
-			"EICJp29EGOk",
-			"Fxvd1timk",
-			"GTqMZELYfQQ",
-			"GZPTpLPEGmwHGWPC",
-			"_HChnE9NDPY",
-			"NwhjgIWHgGg",
-			"production/tsbqksph4xswqjexfbec",
-			"p/u/bguhrxupr23mw3nwxcrw",
-			"nRSNapbJZnc",
-			"zgfpbtolciznub5egzxk",
-			"zufnu7aimadua9wrgwwo",
-			"zznto1jzch9yjsbtbrul",
-		*/
-	}
-
-	for _, str := range cases {
-		if !IsGibberish(str) {
-			t.Errorf("Mistakenly false: %s", str)
-		}
-	}
-}
-
-func TestVersionStrings(t *testing.T) {
-	cases := []string{
-		"1.0",
-		"1.0.0",
-		"v2.1.3",
-		"2.1.73",
-	}
-
-	for _, str := range cases {
-		if !IsVersionString(str) {
-			t.Errorf("Mistakenly false: %s", str)
-		}
-	}
-}

agent/pkg/oas/ignores.go

@@ -1,77 +0,0 @@
-package oas
-
-import "strings"
-
-var ignoredExtensions = []string{"gif", "svg", "css", "png", "ico", "js", "woff2", "woff", "jpg", "jpeg", "swf", "ttf", "map", "webp", "otf", "mp3"}
-
-var ignoredCtypePrefixes = []string{"image/", "font/", "video/", "audio/", "text/javascript"}
-var ignoredCtypes = []string{"application/javascript", "application/x-javascript", "text/css", "application/font-woff2", "application/font-woff", "application/x-font-woff"}
-
-var ignoredHeaders = []string{
-	"a-im", "accept",
-	"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-type", "cookie",
-	"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
-	"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
-	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests", "x-download-options",
-	"server", "user-agent", "via", "warning", "strict-transport-security", "x-permitted-cross-domain-policies",
-	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data", "x-dns-prefetch-control",
-	"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
-	"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
-	"x-wap-profile", "x-scheme", "status", "x-cache", "x-application-context", "retry-after",
-	"newrelic", "x-cloud-trace-context", "sentry-trace", "x-cache-hits", "x-served-by", "x-span-name",
-	"expires", "set-cookie", "p3p", "content-security-policy", "content-security-policy-report-only",
-	"last-modified", "content-language", "x-varnish", "true-client-ip", "akamai-origin-hop",
-	"keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by",
-	"age", "allow", "www-authenticate", "expect-ct", "timing-allow-origin", "referrer-policy",
-	"x-aspnet-version", "x-aspnetmvc-version", "x-timer", "x-abuse-info", "x-mod-pagespeed",
-	"duration_ms", // UP9 custom
-}
-
-var ignoredHeaderPrefixes = []string{
-	":", "accept-", "access-control-", "if-", "sec-", "grpc-",
-	"x-forwarded-", "x-original-", "cf-",
-	"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
-	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-", "x-goog-",
-}
-
-func isCtypeIgnored(ctype string) bool {
-	for _, prefix := range ignoredCtypePrefixes {
-		if strings.HasPrefix(ctype, prefix) {
-			return true
-		}
-	}
-
-	for _, toIgnore := range ignoredCtypes {
-		if ctype == toIgnore {
-			return true
-		}
-	}
-	return false
-}
-
-func isExtIgnored(path string) bool {
-	for _, extIgn := range ignoredExtensions {
-		if strings.HasSuffix(path, "."+extIgn) {
-			return true
-		}
-	}
-	return false
-}
-
-func isHeaderIgnored(name string) bool {
-	name = strings.ToLower(name)
-
-	for _, ignore := range ignoredHeaders {
-		if name == ignore {
-			return true
-		}
-	}
-
-	for _, prefix := range ignoredHeaderPrefixes {
-		if strings.HasPrefix(name, prefix) {
-			return true
-		}
-	}
-
-	return false
-}

agent/pkg/oas/oas_generator.go

@@ -1,119 +0,0 @@
-package oas
-
-import (
-	"context"
-	"encoding/json"
-	"net/url"
-	"sync"
-
-	"github.com/up9inc/mizu/agent/pkg/har"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-var (
-	syncOnce sync.Once
-	instance *oasGenerator
-)
-
-func GetOasGeneratorInstance() *oasGenerator {
-	syncOnce.Do(func() {
-		instance = newOasGenerator()
-		logger.Log.Debug("OAS Generator Initialized")
-	})
-	return instance
-}
-
-func (g *oasGenerator) Start() {
-	if g.started {
-		return
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	g.cancel = cancel
-	g.ctx = ctx
-	g.entriesChan = make(chan EntryWithSource, 100) // buffer up to 100 entries for OAS processing
-	g.ServiceSpecs = &sync.Map{}
-	g.started = true
-	go instance.runGeneretor()
-}
-
-func (g *oasGenerator) runGeneretor() {
-	for {
-		select {
-		case <-g.ctx.Done():
-			logger.Log.Infof("OAS Generator was canceled")
-			return
-
-		case entryWithSource, ok := <-g.entriesChan:
-			if !ok {
-				logger.Log.Infof("OAS Generator - entries channel closed")
-				break
-			}
-			entry := entryWithSource.Entry
-			u, err := url.Parse(entry.Request.URL)
-			if err != nil {
-				logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
-			}
-
-			val, found := g.ServiceSpecs.Load(u.Host)
-			var gen *SpecGen
-			if !found {
-				gen = NewGen(u.Scheme + "://" + u.Host)
-				g.ServiceSpecs.Store(u.Host, gen)
-			} else {
-				gen = val.(*SpecGen)
-			}
-
-			opId, err := gen.feedEntry(entryWithSource)
-			if err != nil {
-				txt, suberr := json.Marshal(entry)
-				if suberr == nil {
-					logger.Log.Debugf("Problematic entry: %s", txt)
-				}
-
-				logger.Log.Warningf("Failed processing entry: %s", err)
-				continue
-			}
-
-			logger.Log.Debugf("Handled entry %s as opId: %s", entry.Request.URL, opId) // TODO: set opId back to entry?
-		}
-	}
-}
-
-func (g *oasGenerator) Reset() {
-	g.ServiceSpecs = &sync.Map{}
-}
-
-func (g *oasGenerator) PushEntry(entryWithSource *EntryWithSource) {
-	if !g.started {
-		return
-	}
-	select {
-	case g.entriesChan <- *entryWithSource:
-	default:
-		logger.Log.Warningf("OAS Generator - entry wasn't sent to channel because the channel has no buffer or there is no receiver")
-	}
-}
-
-func newOasGenerator() *oasGenerator {
-	return &oasGenerator{
-		started:      false,
-		ctx:          nil,
-		cancel:       nil,
-		ServiceSpecs: nil,
-		entriesChan:  nil,
-	}
-}
-
-type EntryWithSource struct {
-	Source string
-	Entry  har.Entry
-	Id     uint
-}
-
-type oasGenerator struct {
-	started      bool
-	ctx          context.Context
-	cancel       context.CancelFunc
-	ServiceSpecs *sync.Map
-	entriesChan  chan EntryWithSource
-}

agent/pkg/oas/specgen.go

@@ -1,729 +0,0 @@
-package oas
-
-import (
-	"encoding/json"
-	"errors"
-	"github.com/chanced/openapi"
-	"github.com/google/uuid"
-	"github.com/nav-inc/datetime"
-	"github.com/up9inc/mizu/shared/logger"
-	"io"
-	"io/ioutil"
-	"mime"
-	"mime/multipart"
-	"net/textproto"
-	"net/url"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-
-	"github.com/up9inc/mizu/agent/pkg/har"
-
-	"time"
-)
-
-const LastSeenTS = "x-last-seen-ts"
-const CountersTotal = "x-counters-total"
-const CountersPerSource = "x-counters-per-source"
-const SampleId = "x-sample-entry"
-
-type reqResp struct { // hello, generics in Go
-	Req  *har.Request
-	Resp *har.Response
-}
-
-type SpecGen struct {
-	oas  *openapi.OpenAPI
-	tree *Node
-	lock sync.Mutex
-}
-
-func NewGen(server string) *SpecGen {
-	spec := new(openapi.OpenAPI)
-	spec.Version = "3.1.0"
-
-	info := openapi.Info{Title: server}
-	info.Version = "1.0"
-	spec.Info = &info
-	spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
-
-	spec.Servers = make([]*openapi.Server, 0)
-	spec.Servers = append(spec.Servers, &openapi.Server{URL: server})
-
-	gen := SpecGen{oas: spec, tree: new(Node)}
-	return &gen
-}
-
-func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
-	g.oas = oas
-	g.tree = new(Node)
-	for pathStr, pathObj := range oas.Paths.Items {
-		pathSplit := strings.Split(string(pathStr), "/")
-		g.tree.getOrSet(pathSplit, pathObj)
-
-		// clean "last entry timestamp" markers from the past
-		for _, pathAndOp := range g.tree.listOps() {
-			delete(pathAndOp.op.Extensions, LastSeenTS)
-		}
-	}
-}
-
-func (g *SpecGen) feedEntry(entryWithSource EntryWithSource) (string, error) {
-	g.lock.Lock()
-	defer g.lock.Unlock()
-
-	opId, err := g.handlePathObj(&entryWithSource)
-	if err != nil {
-		return "", err
-	}
-
-	// NOTE: opId can be empty for some failed entries
-	return opId, err
-}
-
-func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
-	g.lock.Lock()
-	defer g.lock.Unlock()
-
-	g.tree.compact()
-
-	counters := CounterMaps{counterTotal: Counter{}, counterMapTotal: CounterMap{}}
-
-	for _, pathAndOp := range g.tree.listOps() {
-		opObj := pathAndOp.op
-		if opObj.Summary == "" {
-			opObj.Summary = pathAndOp.path
-		}
-
-		err := counters.processOp(opObj)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	err := counters.processOas(g.oas)
-	if err != nil {
-		return nil, err
-	}
-
-	// put paths back from tree into OAS
-	g.oas.Paths = g.tree.listPaths()
-
-	suggestTags(g.oas)
-
-	g.oas.Info.Description = setCounterMsgIfOk(g.oas.Info.Description, &counters.counterTotal)
-
-	// to make a deep copy, no better idea than marshal+unmarshal
-	specText, err := json.MarshalIndent(g.oas, "", "\t")
-	if err != nil {
-		return nil, err
-	}
-
-	spec := new(openapi.OpenAPI)
-	err = json.Unmarshal(specText, spec)
-	if err != nil {
-		return nil, err
-	}
-
-	return spec, err
-}
-
-func suggestTags(oas *openapi.OpenAPI) {
-	paths := getPathsKeys(oas.Paths.Items)
-	sort.Strings(paths) // make it stable in case of multiple candidates
-	for len(paths) > 0 {
-		group := make([]string, 0)
-		group = append(group, paths[0])
-		paths = paths[1:]
-
-		pathsClone := append(paths[:0:0], paths...)
-		for _, path := range pathsClone {
-			if getSimilarPrefix([]string{group[0], path}) != "" {
-				group = append(group, path)
-				paths = deleteFromSlice(paths, path)
-			}
-		}
-
-		common := getSimilarPrefix(group)
-
-		if len(group) > 1 {
-			for _, path := range group {
-				pathObj := oas.Paths.Items[openapi.PathValue(path)]
-				for _, op := range getOps(pathObj) {
-					if op.Tags == nil {
-						op.Tags = make([]string, 0)
-					}
-					// only add tags if not present
-					if len(op.Tags) == 0 {
-						op.Tags = append(op.Tags, common)
-					}
-				}
-			}
-		}
-	}
-}
-
-func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
-	keys := make([]string, len(mymap))
-
-	i := 0
-	for k := range mymap {
-		keys[i] = string(k)
-		i++
-	}
-	return keys
-}
-
-func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error) {
-	entry := entryWithSource.Entry
-	urlParsed, err := url.Parse(entry.Request.URL)
-	if err != nil {
-		return "", err
-	}
-
-	if isExtIgnored(urlParsed.Path) {
-		logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
-		return "", nil
-	}
-
-	if entry.Request.Method == "OPTIONS" {
-		logger.Log.Debugf("Dropped traffic entry due to its method: %s", urlParsed.Path)
-		return "", nil
-	}
-
-	ctype := getRespCtype(&entry.Response)
-	if isCtypeIgnored(ctype) {
-		logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
-		return "", nil
-	}
-
-	if entry.Response.Status < 100 {
-		logger.Log.Debugf("Dropped traffic entry due to status<100: %s", entry.StartedDateTime)
-		return "", nil
-	}
-
-	if entry.Response.Status == 301 || entry.Response.Status == 308 {
-		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
-		return "", nil
-	}
-
-	if entry.Response.Status == 502 || entry.Response.Status == 503 || entry.Response.Status == 504 {
-		logger.Log.Debugf("Dropped traffic entry due to temporary server error: %s", entry.StartedDateTime)
-		return "", nil
-	}
-
-	var split []string
-	if urlParsed.RawPath != "" {
-		split = strings.Split(urlParsed.RawPath, "/")
-	} else {
-		split = strings.Split(urlParsed.Path, "/")
-	}
-	node := g.tree.getOrSet(split, new(openapi.PathObj))
-	opObj, err := handleOpObj(entryWithSource, node.pathObj)
-
-	if opObj != nil {
-		return opObj.OperationID, err
-	}
-
-	return "", err
-}
-
-func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj) (*openapi.Operation, error) {
-	entry := entryWithSource.Entry
-	isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
-	opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
-	if err != nil {
-		return nil, err
-	}
-
-	if !isSuccess && wasMissing {
-		logger.Log.Debugf("Dropped traffic entry due to failed status and no known endpoint at: %s", entry.StartedDateTime)
-		return nil, nil
-	}
-
-	err = handleRequest(&entry.Request, opObj, isSuccess)
-	if err != nil {
-		return nil, err
-	}
-
-	err = handleResponse(&entry.Response, opObj, isSuccess)
-	if err != nil {
-		return nil, err
-	}
-
-	err = handleCounters(opObj, isSuccess, entryWithSource)
-	if err != nil {
-		return nil, err
-	}
-
-	return opObj, nil
-}
-
-func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *EntryWithSource) error {
-	// TODO: if performance around DecodeExtension+SetExtension is bad, store counters as separate maps
-	counter := Counter{}
-	counterMap := CounterMap{}
-	prevTs := 0.0
-	if opObj.Extensions == nil {
-		opObj.Extensions = openapi.Extensions{}
-	} else {
-		if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
-			err := opObj.Extensions.DecodeExtension(CountersTotal, &counter)
-			if err != nil {
-				return err
-			}
-		}
-
-		if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
-			err := opObj.Extensions.DecodeExtension(CountersPerSource, &counterMap)
-			if err != nil {
-				return err
-			}
-		}
-
-		if _, ok := opObj.Extensions.Extension(LastSeenTS); ok {
-			err := opObj.Extensions.DecodeExtension(LastSeenTS, &prevTs)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	var counterPerSource *Counter
-	if existing, ok := counterMap[entryWithSource.Source]; ok {
-		counterPerSource = existing
-	} else {
-		counterPerSource = new(Counter)
-		counterMap[entryWithSource.Source] = counterPerSource
-	}
-
-	started, err := datetime.Parse(entryWithSource.Entry.StartedDateTime, time.UTC)
-	if err != nil {
-		return err
-	}
-
-	ts := float64(started.UnixNano()) / float64(time.Millisecond) / 1000
-	rt := float64(entryWithSource.Entry.Time) / 1000
-
-	dur := 0.0
-	if prevTs != 0 && ts >= prevTs {
-		dur = ts - prevTs
-	}
-
-	counter.addEntry(ts, rt, success, dur)
-	counterPerSource.addEntry(ts, rt, success, dur)
-
-	err = opObj.Extensions.SetExtension(LastSeenTS, ts)
-	if err != nil {
-		return err
-	}
-
-	err = opObj.Extensions.SetExtension(CountersTotal, counter)
-	if err != nil {
-		return err
-	}
-
-	err = opObj.Extensions.SetExtension(CountersPerSource, counterMap)
-	if err != nil {
-		return err
-	}
-
-	err = opObj.Extensions.SetExtension(SampleId, entryWithSource.Id)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error {
-	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
-	urlParsed, err := url.Parse(req.URL)
-	if err != nil {
-		return err
-	}
-
-	qs := make([]har.NVP, 0)
-	for name, vals := range urlParsed.Query() {
-		for _, val := range vals {
-			qs = append(qs, har.NVP{Name: name, Value: val})
-		}
-	}
-
-	if len(qs) != len(req.QueryString) {
-		logger.Log.Warningf("QStr params in HAR do not match URL: %s", req.URL)
-	}
-
-	qstrGW := nvParams{
-		In:             openapi.InQuery,
-		Pairs:          qs,
-		IsIgnored:      func(name string) bool { return false },
-		GeneralizeName: func(name string) string { return name },
-	}
-	handleNameVals(qstrGW, &opObj.Parameters, false)
-
-	hdrGW := nvParams{
-		In:             openapi.InHeader,
-		Pairs:          req.Headers,
-		IsIgnored:      isHeaderIgnored,
-		GeneralizeName: strings.ToLower,
-	}
-	handleNameVals(hdrGW, &opObj.Parameters, true)
-
-	if isSuccess {
-		reqBody, err := getRequestBody(req, opObj)
-		if err != nil {
-			return err
-		}
-
-		if reqBody != nil {
-			if req.PostData.Text == "" {
-				reqBody.Required = false
-			} else {
-
-				reqCtype, _ := getReqCtype(req)
-				reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype)
-				if err != nil {
-					return err
-				}
-
-				_ = reqMedia
-			}
-		}
-	}
-	return nil
-}
-
-func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool) error {
-	// TODO: we don't support "default" response
-	respObj, err := getResponseObj(resp, opObj, isSuccess)
-	if err != nil {
-		return err
-	}
-
-	handleRespHeaders(resp.Headers, respObj)
-
-	respCtype := getRespCtype(resp)
-	respContent := respObj.Content
-	respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype)
-	if err != nil {
-		return err
-	}
-	_ = respMedia
-	return nil
-}
-
-func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj) {
-	visited := map[string]*openapi.HeaderObj{}
-	for _, pair := range reqHeaders {
-		if isHeaderIgnored(pair.Name) {
-			continue
-		}
-
-		nameGeneral := strings.ToLower(pair.Name)
-
-		initHeaders(respObj)
-		objHeaders := respObj.Headers
-		param := findHeaderByName(&respObj.Headers, pair.Name)
-		if param == nil {
-			param = createHeader(openapi.TypeString)
-			objHeaders[nameGeneral] = param
-		}
-		exmp := &param.Examples
-		err := fillParamExample(&exmp, pair.Value)
-		if err != nil {
-			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
-		}
-		visited[nameGeneral] = param
-	}
-
-	// maintain "required" flag
-	if respObj.Headers != nil {
-		for name, param := range respObj.Headers {
-			paramObj, err := param.ResolveHeader(headerResolver)
-			if err != nil {
-				logger.Log.Warningf("Failed to resolve param: %s", err)
-				continue
-			}
-
-			_, ok := visited[strings.ToLower(name)]
-			if !ok {
-				flag := false
-				paramObj.Required = &flag
-			}
-		}
-	}
-}
-
-func fillContent(reqResp reqResp, respContent openapi.Content, ctype string) (*openapi.MediaType, error) {
-	content, found := respContent[ctype]
-	if !found {
-		respContent[ctype] = &openapi.MediaType{}
-		content = respContent[ctype]
-	}
-
-	var text string
-	var isBinary bool
-	if reqResp.Req != nil {
-		isBinary, _, text = reqResp.Req.PostData.B64Decoded()
-	} else {
-		isBinary, _, text = reqResp.Resp.Content.B64Decoded()
-	}
-
-	if !isBinary && text != "" {
-		var exampleMsg []byte
-		// try treating it as json
-		any, isJSON := anyJSON(text)
-		if isJSON {
-			// re-marshal with forced indent
-			if msg, err := json.MarshalIndent(any, "", "\t"); err != nil {
-				panic("Failed to re-marshal value, super-strange")
-			} else {
-				exampleMsg = msg
-			}
-		} else {
-			if msg, err := json.Marshal(text); err != nil {
-				return nil, err
-			} else {
-				exampleMsg = msg
-			}
-		}
-
-		if ctype == "application/x-www-form-urlencoded" && reqResp.Req != nil {
-			handleFormDataUrlencoded(text, content)
-		} else if strings.HasPrefix(ctype, "multipart/form-data") && reqResp.Req != nil {
-			_, params := getReqCtype(reqResp.Req)
-			handleFormDataMultipart(text, content, params)
-		}
-
-		if content.Example == nil && len(exampleMsg) > len(content.Example) {
-			content.Example = exampleMsg
-		}
-	}
-
-	return respContent[ctype], nil
-}
-
-func handleFormDataUrlencoded(text string, content *openapi.MediaType) {
-	formData, err := url.ParseQuery(text)
-	if err != nil {
-		logger.Log.Warningf("Could not decode urlencoded: %s", err)
-		return
-	}
-
-	parts := make([]PartWithBody, 0)
-	for name, vals := range formData {
-		for _, val := range vals {
-			part := new(multipart.Part)
-			part.Header = textproto.MIMEHeader{}
-			part.Header.Add("Content-Disposition", "form-data; name=\""+name+"\";")
-			parts = append(parts, PartWithBody{part: part, body: []byte(val)})
-		}
-	}
-	handleFormData(content, parts)
-}
-
-func handleFormData(content *openapi.MediaType, parts []PartWithBody) {
-	hadSchema := true
-	if content.Schema == nil {
-		hadSchema = false // will use it for required flags
-		content.Schema = new(openapi.SchemaObj)
-		content.Schema.Type = openapi.Types{openapi.TypeObject}
-		content.Schema.Properties = openapi.Schemas{}
-	}
-
-	props := &content.Schema.Properties
-	seenNames := map[string]struct{}{} // set equivalent in Go, yikes
-	for _, pwb := range parts {
-		name := pwb.part.FormName()
-		seenNames[name] = struct{}{}
-		existing, found := (*props)[name]
-		if !found {
-			existing = new(openapi.SchemaObj)
-			existing.Type = openapi.Types{openapi.TypeString}
-			(*props)[name] = existing
-
-			ctype := pwb.part.Header.Get("content-type")
-			if ctype != "" {
-				if existing.Keywords == nil {
-					existing.Keywords = map[string]json.RawMessage{}
-				}
-				existing.Keywords["contentMediaType"], _ = json.Marshal(ctype)
-			}
-		}
-
-		addSchemaExample(existing, string(pwb.body))
-	}
-
-	// handle required flag
-	if content.Schema.Required == nil {
-		if !hadSchema {
-			content.Schema.Required = make([]string, 0)
-			for name := range seenNames {
-				content.Schema.Required = append(content.Schema.Required, name)
-			}
-			sort.Strings(content.Schema.Required)
-		} // else it's a known schema with no required fields
-	} else {
-		content.Schema.Required = intersectSliceWithMap(content.Schema.Required, seenNames)
-		sort.Strings(content.Schema.Required)
-	}
-}
-
-type PartWithBody struct {
-	part *multipart.Part
-	body []byte
-}
-
-func handleFormDataMultipart(text string, content *openapi.MediaType, ctypeParams map[string]string) {
-	boundary, ok := ctypeParams["boundary"]
-	if !ok {
-		logger.Log.Errorf("Multipart header has no boundary")
-		return
-	}
-	mpr := multipart.NewReader(strings.NewReader(text), boundary)
-
-	parts := make([]PartWithBody, 0)
-	for {
-		part, err := mpr.NextPart()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			logger.Log.Errorf("Cannot parse multipart body: %v", err)
-			break
-		}
-		defer part.Close()
-
-		body, err := ioutil.ReadAll(part)
-		if err != nil {
-			logger.Log.Errorf("Error reading multipart Part %s: %v", part.Header, err)
-		}
-
-		parts = append(parts, PartWithBody{part: part, body: body})
-	}
-
-	handleFormData(content, parts)
-}
-
-func getRespCtype(resp *har.Response) string {
-	var ctype string
-	ctype = resp.Content.MimeType
-	for _, hdr := range resp.Headers {
-		if strings.ToLower(hdr.Name) == "content-type" {
-			ctype = hdr.Value
-		}
-	}
-
-	mediaType, _, err := mime.ParseMediaType(ctype)
-	if err != nil {
-		return ""
-	}
-	return mediaType
-}
-
-func getReqCtype(req *har.Request) (ctype string, params map[string]string) {
-	ctype = req.PostData.MimeType
-	for _, hdr := range req.Headers {
-		if strings.ToLower(hdr.Name) == "content-type" {
-			ctype = hdr.Value
-		}
-	}
-
-	if ctype == "" {
-		return "", map[string]string{}
-	}
-
-	mediaType, params, err := mime.ParseMediaType(ctype)
-	if err != nil {
-		logger.Log.Errorf("Cannot parse Content-Type header %q: %v", ctype, err)
-		return "", map[string]string{}
-	}
-	return mediaType, params
-}
-
-func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool) (*openapi.ResponseObj, error) {
-	statusStr := strconv.Itoa(resp.Status)
-
-	var response openapi.Response
-	response, found := opObj.Responses[statusStr]
-	if !found {
-		if opObj.Responses == nil {
-			opObj.Responses = map[string]openapi.Response{}
-		}
-
-		opObj.Responses[statusStr] = &openapi.ResponseObj{Content: map[string]*openapi.MediaType{}}
-		response = opObj.Responses[statusStr]
-	}
-
-	resResponse, err := response.ResolveResponse(responseResolver)
-	if err != nil {
-		return nil, err
-	}
-
-	if isSuccess {
-		resResponse.Description = "Successful call with status " + statusStr
-	} else {
-		resResponse.Description = "Failed call with status " + statusStr
-	}
-	return resResponse, nil
-}
-
-func getRequestBody(req *har.Request, opObj *openapi.Operation) (*openapi.RequestBodyObj, error) {
-	if opObj.RequestBody == nil {
-		// create if there is body in request
-		if req.PostData.Text != "" {
-			opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
-		} else {
-			return nil, nil
-		}
-	}
-
-	reqBody, err := opObj.RequestBody.ResolveRequestBody(reqBodyResolver)
-	if err != nil {
-		return nil, err
-	}
-
-	return reqBody, nil
-}
-
-func getOpObj(pathObj *openapi.PathObj, method string, createIfNone bool) (*openapi.Operation, bool, error) {
-	method = strings.ToLower(method)
-	var op **openapi.Operation
-
-	switch method {
-	case "get":
-		op = &pathObj.Get
-	case "put":
-		op = &pathObj.Put
-	case "post":
-		op = &pathObj.Post
-	case "delete":
-		op = &pathObj.Delete
-	case "options":
-		op = &pathObj.Options
-	case "head":
-		op = &pathObj.Head
-	case "patch":
-		op = &pathObj.Patch
-	case "trace":
-		op = &pathObj.Trace
-	default:
-		return nil, false, errors.New("unsupported HTTP method: " + method)
-	}
-
-	isMissing := false
-	if *op == nil {
-		isMissing = true
-		if createIfNone {
-			*op = &openapi.Operation{Responses: map[string]openapi.Response{}}
-			newUUID := uuid.New().String()
-			(**op).OperationID = newUUID
-		} else {
-			return nil, isMissing, nil
-		}
-	}
-
-	return *op, isMissing, nil
-}

agent/pkg/oas/specgen_test.go

@@ -1,253 +0,0 @@
-package oas
-
-import (
-	"encoding/json"
-	"github.com/chanced/openapi"
-	"github.com/op/go-logging"
-	"github.com/up9inc/mizu/shared/logger"
-	"github.com/wI2L/jsondiff"
-	"io/ioutil"
-	"os"
-	"regexp"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/up9inc/mizu/agent/pkg/har"
-)
-
-// if started via env, write file into subdir
-func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string {
-	content, err := json.MarshalIndent(spec, "", "\t")
-	if err != nil {
-		panic(err)
-	}
-
-	if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
-		path := "./oas-samples"
-		err := os.MkdirAll(path, 0o755)
-		if err != nil {
-			panic(err)
-		}
-		err = ioutil.WriteFile(path+"/"+label+".json", content, 0644)
-		if err != nil {
-			panic(err)
-		}
-		t.Logf("Written: %s", label)
-	} else {
-		t.Logf("%s", string(content))
-	}
-	return string(content)
-}
-
-func TestEntries(t *testing.T) {
-	logger.InitLoggerStderrOnly(logging.INFO)
-	files, err := getFiles("./test_artifacts/")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-	GetOasGeneratorInstance().Start()
-	loadStartingOAS()
-
-	cnt, err := feedEntries(files, true)
-	if err != nil {
-		t.Log(err)
-		t.Fail()
-	}
-
-	waitQueueProcessed()
-
-	svcs := strings.Builder{}
-	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
-		gen := val.(*SpecGen)
-		svc := key.(string)
-		svcs.WriteString(svc + ",")
-		spec, err := gen.GetSpec()
-		if err != nil {
-			t.Log(err)
-			t.FailNow()
-			return false
-		}
-
-		err = spec.Validate()
-		if err != nil {
-			specText, _ := json.MarshalIndent(spec, "", "\t")
-			t.Log(string(specText))
-			t.Log(err)
-			t.FailNow()
-		}
-
-		return true
-	})
-
-	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
-		svc := key.(string)
-		gen := val.(*SpecGen)
-		spec, err := gen.GetSpec()
-		if err != nil {
-			t.Log(err)
-			t.FailNow()
-		}
-
-		outputSpec(svc, spec, t)
-
-		err = spec.Validate()
-		if err != nil {
-			t.Log(err)
-			t.FailNow()
-		}
-
-		return true
-	})
-
-	logger.Log.Infof("Total entries: %d", cnt)
-}
-
-func TestFileSingle(t *testing.T) {
-	GetOasGeneratorInstance().Start()
-	GetOasGeneratorInstance().Reset()
-	// loadStartingOAS()
-	file := "test_artifacts/params.har"
-	files := []string{file}
-	cnt, err := feedEntries(files, true)
-	if err != nil {
-		logger.Log.Warning("Failed processing file: " + err.Error())
-		t.Fail()
-	}
-
-	waitQueueProcessed()
-
-	GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
-		svc := key.(string)
-		gen := val.(*SpecGen)
-		spec, err := gen.GetSpec()
-		if err != nil {
-			t.Log(err)
-			t.FailNow()
-		}
-
-		specText := outputSpec(svc, spec, t)
-
-		err = spec.Validate()
-		if err != nil {
-			t.Log(err)
-			t.FailNow()
-		}
-
-		expected, err := ioutil.ReadFile(file + ".spec.json")
-		if err != nil {
-			t.Errorf(err.Error())
-			t.FailNow()
-		}
-
-		patFloatPrecision := regexp.MustCompile(`(\d+\.\d{1,2})(\d*)`)
-
-		expected = []byte(patUuid4.ReplaceAllString(string(expected), "<UUID4>"))
-		specText = patUuid4.ReplaceAllString(specText, "<UUID4>")
-		expected = []byte(patFloatPrecision.ReplaceAllString(string(expected), "$1"))
-		specText = patFloatPrecision.ReplaceAllString(specText, "$1")
-
-		diff, err := jsondiff.CompareJSON(expected, []byte(specText))
-		if err != nil {
-			t.Errorf(err.Error())
-			t.FailNow()
-		}
-
-		if len(diff) > 0 {
-			t.Errorf("Generated spec does not match expected:\n%s", diff.String())
-		}
-
-		return true
-	})
-
-	logger.Log.Infof("Processed entries: %d", cnt)
-}
-
-func waitQueueProcessed() {
-	for {
-		time.Sleep(100 * time.Millisecond)
-		queue := len(GetOasGeneratorInstance().entriesChan)
-		logger.Log.Infof("Queue: %d", queue)
-		if queue < 1 {
-			break
-		}
-	}
-}
-
-func loadStartingOAS() {
-	file := "test_artifacts/catalogue.json"
-	fd, err := os.Open(file)
-	if err != nil {
-		panic(err)
-	}
-
-	defer fd.Close()
-
-	data, err := ioutil.ReadAll(fd)
-	if err != nil {
-		panic(err)
-	}
-
-	var doc *openapi.OpenAPI
-	err = json.Unmarshal(data, &doc)
-	if err != nil {
-		panic(err)
-	}
-
-	gen := NewGen("catalogue")
-	gen.StartFromSpec(doc)
-
-	GetOasGeneratorInstance().ServiceSpecs.Store("catalogue", gen)
-}
-
-func TestEntriesNegative(t *testing.T) {
-	files := []string{"invalid"}
-	_, err := feedEntries(files, false)
-	if err == nil {
-		t.Logf("Should have failed")
-		t.Fail()
-	}
-}
-
-func TestEntriesPositive(t *testing.T) {
-	files := []string{"test_artifacts/params.har"}
-	_, err := feedEntries(files, false)
-	if err != nil {
-		t.Logf("Failed")
-		t.Fail()
-	}
-}
-
-func TestLoadValidHAR(t *testing.T) {
-	inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}`
-	var entry *har.Entry
-	var err = json.Unmarshal([]byte(inp), &entry)
-	if err != nil {
-		t.Logf("Failed to decode entry: %s", err)
-		t.FailNow() // demonstrates the problem of `martian` HAR library
-	}
-}
-
-func TestLoadValid3_1(t *testing.T) {
-	fd, err := os.Open("test_artifacts/catalogue.json")
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	defer fd.Close()
-
-	data, err := ioutil.ReadAll(fd)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-
-	var oas openapi.OpenAPI
-	err = json.Unmarshal(data, &oas)
-	if err != nil {
-		t.Log(err)
-		t.FailNow()
-	}
-}

agent/pkg/oas/test_artifacts/catalogue.json

@@ -1,51 +0,0 @@
-{
-  "openapi": "3.1.0",
-  "info": {
-    "title": "Preloaded",
-    "version": "0.1",
-    "description": "Test file for loading pre-existing OAS"
-  },
-  "paths": {
-    "/catalogue/{id}": {
-      "parameters": [
-        {
-          "name": "id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "example": "some-uuid-maybe"
-        }
-      ],
-      "get": {
-        "parameters": [        {
-          "name": "non-required-header",
-          "in": "header",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "example": "some-uuid-maybe"
-        }
-        ]
-      }
-    },
-    "/catalogue/{id}/details": {
-      "parameters": [
-        {
-          "name": "id",
-          "in": "path",
-          "style": "simple",
-          "required": true,
-          "schema": {
-            "type": "string"
-          },
-          "example": "some-uuid-maybe"
-        }
-      ]
-    }
-  }
-}

agent/pkg/oas/test_artifacts/example.ldjson

@@ -1,13 +0,0 @@
-{"messageType": "http", "_source": "some-source", ",firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
-{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78415179Z", "time": 13, "request": {"method": "GET", "url": "http://catalogue/catalogue/size?tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "x-some", "value": "demo val"},{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [{"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "22"}], "content": {"size": 22, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInNpemUiOjl9", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 22}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 13}}
-{"_id": "", "startedDateTime": "2021-07-26T11:14:17.784918698Z", "time": 19, "request": {"method": "GET", "url": "http://catalogue/catalogue?page=1&size=6&tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "page", "value": "1"}, {"name": "size", "value": "6"}, {"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "1927"}], "content": {"size": 1927, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19LHsiaWQiOiI4MzdhYjE0MS0zOTllLTRjMWYtOWFiYy1iYWNlNDAyOTZiYWMiLCJuYW1lIjoiQ2F0IHNvY2tzIiwiZGVzY3JpcHRpb24iOiJjb25zZXF1YXQgYW1ldCBjdXBpZGF0YXQgbWluaW0gbGFib3J1bSB0ZW1wb3IgZWxpdCBleCBjb25zZXF1YXQgaW4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2NrczIuanBnIl0sInByaWNlIjoxNSwiY291bnQiOjE3NSwidGFnIjpbImJyb3duIiwiZm9ybWFsIiwiZ3JlZW4iXX1dCg==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1927}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 19}}
-{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78418182Z", "time": 7, "request": {"method": "GET", "url": "http://catalogue/tags", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "107"}], "content": {"size": 107, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInRhZ3MiOlsiYnJvd24iLCJnZWVrIiwiZm9ybWFsIiwiYmx1ZSIsInNraW4iLCJyZWQiLCJhY3Rpb24iLCJzcG9ydCIsImJsYWNrIiwibWFnaWMiLCJncmVlbiJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 107}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 7}}
-{"_id": "", "startedDateTime": "2021-07-26T11:14:18.131501482Z", "time": 5, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}, {"name": "x-some", "value": "demoval"}], ",queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 5}}
-{"_id": "", "startedDateTime": "2021-07-26T11:14:18.379836908Z", "time": 14, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 14}}
-{"_id": "", "startedDateTime": "2021-07-26T11:14:22.920540124Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/808a2de1-1aaa-4c25-a9b9-6612e8f29a38", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "275"}], "content": {"size": 275, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NzM4LCJkZXNjcmlwdGlvbiI6IkEgbWF0dXJlIHNvY2ssIGNyb3NzZWQsIHdpdGggYW4gYWlyIG9mIG5vbmNoYWxhbmNlLiIsImlkIjoiODA4YTJkZTEtMWFhYS00YzI1LWE5YjktNjYxMmU4ZjI5YTM4IiwiaW1hZ2VVcmwiOlsiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMS5qcGVnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMi5qcGVnIl0sIm5hbWUiOiJDcm9zc2VkIiwicHJpY2UiOjE3LjMyLCJ0YWciOlsiYmx1ZSIsInJlZCIsImFjdGlvbiIsImZvcm1hbCJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 275}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
-{"_id": "", "startedDateTime": "2021-07-26T11:14:22.921609501Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?sort=id&size=3&tags=blue", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "3"}, {"name": "tags", "value": "blue"}, {"name": "sort", "value": "id"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "789"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}], "content": {"size": 789, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJuYW1lIjoiQ29sb3VyZnVsIiwiZGVzY3JpcHRpb24iOiJwcm9pZGVudCBvY2NhZWNhdCBpcnVyZSBldCBleGNlcHRldXIgbGFib3JlIG1pbmltIG5pc2kgYW1ldCBpcnVyZSIsImltYWdlVXJsIjpbIi9jYXRhbG9ndWUvaW1hZ2VzL2NvbG91cmZ1bF9zb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIl0sInByaWNlIjoxOCwiY291bnQiOjQzOCwidGFnIjpbImJsdWUiXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiYmx1ZSJdfV0K", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 789}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
-{"_id": "", "startedDateTime": "2021-07-26T11:14:22.923197848Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
-{"_id": "", "startedDateTime": "2021-07-26T11:14:23.175549218Z", "time": 26, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 26}}
-{"_id": "", "startedDateTime": "2021-07-26T11:14:25.239777333Z", "time": 10, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Transfer-Encoding", "value": "chunked"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "X-Application-Context", "value": "carts:80"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 10}}
-{"_id": "", "startedDateTime": "2021-07-26T11:14:25.725866772Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?size=5", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "5"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "1643"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}], "content": {"size": 1643, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19XQo=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1643}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
-{"_id": "", "startedDateTime": "2021-07-26T11:14:25.729303217Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}

agent/pkg/oas/test_artifacts/params.har

@@ -1,544 +0,0 @@
-{
-  "log": {
-    "version": "1.2",
-    "creator": {
-      "name": "mitmproxy har_dump",
-      "version": "0.1",
-      "comment": "mitmproxy version mitmproxy 4.0.4"
-    },
-    "entries": [
-      {
-        "startedDateTime": "2019-09-06T06:14:43.864529+00:00",
-        "time": 111,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/e21f7112-3d3b-4632-9da3-a4af2e0e9166/sub1",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "headersSize": 1542,
-          "bodySize": 0,
-          "queryString": []
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "mimeType": "text/html",
-            "text": "",
-            "size": 0
-          },
-          "redirectURL": "",
-          "headersSize": 245,
-          "bodySize": 39
-        },
-        "cache": {},
-        "timings": {
-          "send": 22,
-          "receive": 2,
-          "wait": 87,
-          "connect": -1,
-          "ssl": -1
-        },
-        "serverIPAddress": "54.210.29.33"
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:18.747122+00:00",
-        "time": 630,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a/sub2",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "queryString": [],
-          "headersSize": 1542,
-          "bodySize": 0
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "content": {
-            "size": 39,
-            "compression": -20,
-            "mimeType": "application/json",
-            "text": "null"
-          },
-          "redirectURL": "",
-          "headersSize": 248,
-          "bodySize": 39
-        },
-        "cache": {},
-        "timings": {
-          "send": 14,
-          "receive": 4,
-          "wait": 350,
-          "connect": 262,
-          "ssl": -1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:19.747122+00:00",
-        "time": 630,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a;mparam=matrixparam",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "queryString": [],
-          "headersSize": 1542,
-          "bodySize": 0
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "content": {
-            "size": 39,
-            "compression": -20,
-            "mimeType": "application/json",
-            "text": "null"
-          },
-          "redirectURL": "",
-          "headersSize": 248,
-          "bodySize": 39
-        },
-        "cache": {},
-        "timings": {
-          "send": 14,
-          "receive": 4,
-          "wait": 350,
-          "connect": 262,
-          "ssl": -1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:20.047122+00:00",
-        "time": 630,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/appears-once",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "queryString": [],
-          "headersSize": 1542,
-          "bodySize": 0
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "content": {
-            "size": 39,
-            "compression": -20,
-            "mimeType": "application/json",
-            "text": "null"
-          },
-          "redirectURL": "",
-          "headersSize": 248,
-          "bodySize": 39
-        },
-        "cache": {},
-        "timings": {
-          "send": 14,
-          "receive": 4,
-          "wait": 350,
-          "connect": 262,
-          "ssl": -1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:20.747122+00:00",
-        "time": 630,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/appears-twice",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "queryString": [],
-          "headersSize": 1542,
-          "bodySize": 0
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "content": {
-            "size": 39,
-            "compression": -20,
-            "mimeType": "application/json",
-            "text": "null"
-          },
-          "redirectURL": "",
-          "headersSize": 248,
-          "bodySize": 39
-        },
-        "cache": {},
-        "timings": {
-          "send": 14,
-          "receive": 4,
-          "wait": 350,
-          "connect": 262,
-          "ssl": -1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
-        "time": 630,
-        "request": {
-          "method": "GET",
-          "url": "https://httpbin.org/appears-twice",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "queryString": [],
-          "headersSize": 1542,
-          "bodySize": 0
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "HTTP/1.1",
-          "cookies": [],
-          "headers": [],
-          "content": {
-            "size": 39,
-            "compression": -20,
-            "mimeType": "application/json",
-            "text": "null"
-          },
-          "redirectURL": "",
-          "headersSize": 248,
-          "bodySize": 39
-        },
-        "cache": {},
-        "timings": {
-          "send": 14,
-          "receive": 4,
-          "wait": 350,
-          "connect": 262,
-          "ssl": -1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:20.747122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/form-urlencoded",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-            {
-              "name": "Content-Type",
-              "value": "application/x-www-form-urlencoded"
-            }
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": "agent-id=ade&callback-url=&token=sometoken"
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/form-urlencoded",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-            {
-              "name": "Content-Type",
-              "value": "application/x-www-form-urlencoded"
-            }
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": "agent-id=ade&callback-url=&token=sometoken-second-val&optional=another"
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:22.747122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/form-multipart",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-            {
-              "name": "Content-Type",
-              "value": "multipart/form-data; boundary=BOUNDARY"
-            }
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n"
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 62,
-            "mimeType": "",
-            "text": "{}"
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 62
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/body-optional",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": ""
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/body-optional",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-            {
-              "name": "Content-Type",
-              "value": "application/json"
-            }
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": "{\"key\", \"val\"}"
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/body-optional",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": ""
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      },
-      {
-        "startedDateTime": "2019-09-06T06:16:21.757122+00:00",
-        "time": 1,
-        "request": {
-          "method": "POST",
-          "url": "https://httpbin.org/body-required",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "queryString": [],
-          "headersSize": -1,
-          "bodySize": -1,
-          "postData": {
-            "mimeType": "",
-            "text": "body exists"
-          }
-        },
-        "response": {
-          "status": 200,
-          "statusText": "OK",
-          "httpVersion": "",
-          "cookies": [],
-          "headers": [
-          ],
-          "content": {
-            "size": 0,
-            "mimeType": "",
-            "text": ""
-          },
-          "redirectURL": "",
-          "headersSize": -1,
-          "bodySize": 0
-        },
-        "cache": {},
-        "timings": {
-          "send": -1,
-          "wait": -1,
-          "receive": 1
-        }
-      }
-    ]
-  }
-}

agent/pkg/oas/test_artifacts/params.har.spec.json

@@ -1,507 +0,0 @@
-{
-  "openapi": "3.1.0",
-  "info": {
-    "title": "https://httpbin.org",
-    "description": "Mizu observed 13 entries (0 failed), at 0.155 hits/s, average response time is 0.251 seconds",
-    "version": "1.0"
-  },
-  "servers": [
-    {
-      "url": "https://httpbin.org"
-    }
-  ],
-  "paths": {
-    "/appears-once": {
-      "get": {
-        "summary": "/appears-once",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
-        "operationId": "89aa39f6-78d0-411b-b701-a33bd77868b0",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": null
-              }
-            }
-          }
-        },
-        "x-last-seen-ts": 1567750580.0471218,
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750580.0471218,
-          "lastSeen": 1567750580.0471218,
-          "sumRT": 0.63,
-          "sumDuration": 0
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750580.0471218,
-            "lastSeen": 1567750580.0471218,
-            "sumRT": 0.63,
-            "sumDuration": 0
-          }
-        },
-        "x-sample-entry": 0
-      }
-    },
-    "/appears-twice": {
-      "get": {
-        "summary": "/appears-twice",
-        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds",
-        "operationId": "f5e2b5a2-e01a-45f4-bde1-15a7e7a06d3c",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": null
-              }
-            }
-          }
-        },
-        "x-counters-total": {
-          "entries": 2,
-          "failures": 0,
-          "firstSeen": 1567750580.7471218,
-          "lastSeen": 1567750581.7471218,
-          "sumRT": 1.26,
-          "sumDuration": 1
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 2,
-            "failures": 0,
-            "firstSeen": 1567750580.7471218,
-            "lastSeen": 1567750581.7471218,
-            "sumRT": 1.26,
-            "sumDuration": 1
-          }
-        },
-        "x-sample-entry": 0,
-        "x-last-seen-ts": 1567750581.7471218
-      }
-    },
-    "/body-optional": {
-      "post": {
-        "summary": "/body-optional",
-        "description": "Mizu observed 3 entries (0 failed), at 0.003 hits/s, average response time is 0.001 seconds",
-        "operationId": "14d5b1c2-dc03-4ee5-baaa-5c7992acc82e",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {}
-            }
-          }
-        },
-        "x-counters-total": {
-          "entries": 3,
-          "failures": 0,
-          "firstSeen": 1567750581.7471218,
-          "lastSeen": 1567750581.757122,
-          "sumRT": 0.003,
-          "sumDuration": 0.010000228881835938
-        },
-        "x-sample-entry": 0,
-        "x-counters-per-source": {
-          "": {
-            "entries": 3,
-            "failures": 0,
-            "firstSeen": 1567750581.7471218,
-            "lastSeen": 1567750581.757122,
-            "sumRT": 0.003,
-            "sumDuration": 0.010000228881835938
-          }
-        },
-        "x-last-seen-ts": 1567750581.757122,
-        "requestBody": {
-          "description": "Generic request body",
-          "content": {
-            "application/json": {
-              "example": "{\"key\", \"val\"}"
-            }
-          }
-        }
-      }
-    },
-    "/body-required": {
-      "post": {
-        "summary": "/body-required",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
-        "operationId": "d0958c5a-dce6-4616-99f4-201dbc51457a",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {}
-            }
-          }
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750581.757122,
-            "lastSeen": 1567750581.757122,
-            "sumRT": 0.001,
-            "sumDuration": 0
-          }
-        },
-        "x-sample-entry": 0,
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750581.757122,
-          "lastSeen": 1567750581.757122,
-          "sumRT": 0.001,
-          "sumDuration": 0
-        },
-        "x-last-seen-ts": 1567750581.757122,
-        "requestBody": {
-          "description": "Generic request body",
-          "content": {
-            "": {
-              "example": "body exists"
-            }
-          },
-          "required": true
-        }
-      }
-    },
-    "/form-multipart": {
-      "post": {
-        "summary": "/form-multipart",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
-        "operationId": "cab5a2f3-c18a-4d5a-8f92-e40da4fd6603",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {
-                "example": {}
-              }
-            }
-          }
-        },
-        "x-sample-entry": 0,
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750582.7471218,
-            "lastSeen": 1567750582.7471218,
-            "sumRT": 0.001,
-            "sumDuration": 0
-          }
-        },
-        "x-last-seen-ts": 1567750582.7471218,
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750582.7471218,
-          "lastSeen": 1567750582.7471218,
-          "sumRT": 0.001,
-          "sumDuration": 0
-        },
-        "requestBody": {
-          "description": "Generic request body",
-          "content": {
-            "multipart/form-data": {
-              "schema": {
-                "type": "object",
-                "required": [
-                  "file",
-                  "path"
-                ],
-                "properties": {
-                  "file": {
-                    "type": "string",
-                    "contentMediaType": "application/json",
-                    "examples": [
-                      "{\"functions\": 123}"
-                    ]
-                  },
-                  "path": {
-                    "type": "string",
-                    "examples": [
-                      "/content/components"
-                    ]
-                  }
-                }
-              },
-              "example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n"
-            }
-          },
-          "required": true
-        }
-      }
-    },
-    "/form-urlencoded": {
-      "post": {
-        "summary": "/form-urlencoded",
-        "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds",
-        "operationId": "7c373ad7-6ab5-422e-971b-1cf56b18a7a2",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "": {}
-            }
-          }
-        },
-        "x-last-seen-ts": 1567750581.7471218,
-        "x-counters-total": {
-          "entries": 2,
-          "failures": 0,
-          "firstSeen": 1567750580.7471218,
-          "lastSeen": 1567750581.7471218,
-          "sumRT": 0.002,
-          "sumDuration": 1
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 2,
-            "failures": 0,
-            "firstSeen": 1567750580.7471218,
-            "lastSeen": 1567750581.7471218,
-            "sumRT": 0.002,
-            "sumDuration": 1
-          }
-        },
-        "x-sample-entry": 0,
-        "requestBody": {
-          "description": "Generic request body",
-          "content": {
-            "application/x-www-form-urlencoded": {
-              "schema": {
-                "type": "object",
-                "required": [
-                  "agent-id",
-                  "callback-url",
-                  "token"
-                ],
-                "properties": {
-                  "agent-id": {
-                    "type": "string",
-                    "examples": [
-                      "ade"
-                    ]
-                  },
-                  "callback-url": {
-                    "type": "string",
-                    "examples": [
-                      ""
-                    ]
-                  },
-                  "optional": {
-                    "type": "string",
-                    "examples": [
-                      "another"
-                    ]
-                  },
-                  "token": {
-                    "type": "string",
-                    "examples": [
-                      "sometoken",
-                      "sometoken-second-val"
-                    ]
-                  }
-                }
-              },
-              "example": "agent-id=ade\u0026callback-url=\u0026token=sometoken"
-            }
-          },
-          "required": true
-        }
-      }
-    },
-    "/{Id}": {
-      "get": {
-        "summary": "/{Id}",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
-        "operationId": "99f1d11f-29c0-48f9-8bf0-9f4b407c7c3f",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": null
-              }
-            }
-          }
-        },
-        "x-sample-entry": 0,
-        "x-last-seen-ts": 1567750579.7471218,
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750579.7471218,
-          "lastSeen": 1567750579.7471218,
-          "sumRT": 0.63,
-          "sumDuration": 0
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750579.7471218,
-            "lastSeen": 1567750579.7471218,
-            "sumRT": 0.63,
-            "sumDuration": 0
-          }
-        }
-      },
-      "parameters": [
-        {
-          "name": "Id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
-            },
-            "example #1": {
-              "value": "952bea17-3776-11ea-9341-42010a84012a"
-            }
-          }
-        }
-      ]
-    },
-    "/{Id}/sub1": {
-      "get": {
-        "summary": "/{Id}/sub1",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds",
-        "operationId": "f7e299d2-253c-4eef-975c-9a5659a7fc50",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "text/html": {}
-            }
-          }
-        },
-        "x-last-seen-ts": 1567750483.864529,
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750483.864529,
-          "lastSeen": 1567750483.864529,
-          "sumRT": 0.111,
-          "sumDuration": 0
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750483.864529,
-            "lastSeen": 1567750483.864529,
-            "sumRT": 0.111,
-            "sumDuration": 0
-          }
-        },
-        "x-sample-entry": 0
-      },
-      "parameters": [
-        {
-          "name": "Id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
-            },
-            "example #1": {
-              "value": "952bea17-3776-11ea-9341-42010a84012a"
-            }
-          }
-        }
-      ]
-    },
-    "/{Id}/sub2": {
-      "get": {
-        "summary": "/{Id}/sub2",
-        "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
-        "operationId": "23a54e06-4298-4ea5-b1f0-09b0354a0598",
-        "responses": {
-          "200": {
-            "description": "Successful call with status 200",
-            "content": {
-              "application/json": {
-                "example": null
-              }
-            }
-          }
-        },
-        "x-last-seen-ts": 1567750578.7471218,
-        "x-counters-total": {
-          "entries": 1,
-          "failures": 0,
-          "firstSeen": 1567750578.7471218,
-          "lastSeen": 1567750578.7471218,
-          "sumRT": 0.63,
-          "sumDuration": 0
-        },
-        "x-counters-per-source": {
-          "": {
-            "entries": 1,
-            "failures": 0,
-            "firstSeen": 1567750578.7471218,
-            "lastSeen": 1567750578.7471218,
-            "sumRT": 0.63,
-            "sumDuration": 0
-          }
-        },
-        "x-sample-entry": 0
-      },
-      "parameters": [
-        {
-          "name": "Id",
-          "in": "path",
-          "required": true,
-          "style": "simple",
-          "schema": {
-            "type": "string"
-          },
-          "examples": {
-            "example #0": {
-              "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
-            },
-            "example #1": {
-              "value": "952bea17-3776-11ea-9341-42010a84012a"
-            }
-          }
-        }
-      ]
-    }
-  },
-  "x-counters-total": {
-    "entries": 13,
-    "failures": 0,
-    "firstSeen": 1567750483.864529,
-    "lastSeen": 1567750582.7471218,
-    "sumRT": 3.268,
-    "sumDuration": 2.010000228881836
-  },
-  "x-counters-per-source": {
-    "": {
-      "entries": 13,
-      "failures": 0,
-      "firstSeen": 1567750483.864529,
-      "lastSeen": 1567750582.7471218,
-      "sumRT": 3.268,
-      "sumDuration": 2.010000228881836
-    }
-  }
-}

agent/pkg/oas/tree.go

@@ -1,245 +0,0 @@
-package oas
-
-import (
-	"net/url"
-	"strconv"
-	"strings"
-
-	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-type NodePath = []string
-
-type Node struct {
-	constant  *string
-	pathParam *openapi.ParameterObj
-	pathObj   *openapi.PathObj
-	parent    *Node
-	children  []*Node
-}
-
-func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node *Node) {
-	if existingPathObj == nil {
-		panic("Invalid function call")
-	}
-
-	pathChunk := path[0]
-	potentialMatrix := strings.SplitN(pathChunk, ";", 2)
-	if len(potentialMatrix) > 1 {
-		pathChunk = potentialMatrix[0]
-		logger.Log.Warningf("URI matrix params are not supported: %s", potentialMatrix[1])
-	}
-
-	chunkIsParam := strings.HasPrefix(pathChunk, "{") && strings.HasSuffix(pathChunk, "}")
-	pathChunk, err := url.PathUnescape(pathChunk)
-	if err != nil {
-		logger.Log.Warningf("URI segment is not correctly encoded: %s", pathChunk)
-		// any side effects on continuing?
-	}
-
-	chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
-
-	var paramObj *openapi.ParameterObj
-	if chunkIsParam && existingPathObj != nil && existingPathObj.Parameters != nil {
-		_, paramObj = findParamByName(existingPathObj.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
-	}
-
-	if paramObj == nil {
-		node = n.searchInConstants(pathChunk)
-	}
-
-	if node == nil {
-		node = n.searchInParams(paramObj, chunkIsGibberish)
-	}
-
-	// still no node found, should create it
-	if node == nil {
-		node = new(Node)
-		node.parent = n
-		n.children = append(n.children, node)
-
-		if paramObj != nil {
-			node.pathParam = paramObj
-		} else if chunkIsGibberish {
-			newParam := n.createParam()
-			node.pathParam = newParam
-		} else {
-			node.constant = &pathChunk
-		}
-	}
-
-	// add example if it's a gibberish chunk
-	if node.pathParam != nil && !chunkIsParam {
-		exmp := &node.pathParam.Examples
-		err := fillParamExample(&exmp, pathChunk)
-		if err != nil {
-			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
-		}
-	}
-
-	// TODO: eat up trailing slash, in a smart way: node.pathObj!=nil && path[1]==""
-	if len(path) > 1 {
-		return node.getOrSet(path[1:], existingPathObj)
-	} else if node.pathObj == nil {
-		node.pathObj = existingPathObj
-	}
-
-	return node
-}
-
-func (n *Node) createParam() *openapi.ParameterObj {
-	name := "param"
-
-	if n.constant != nil { // the node is already a param
-		// REST assumption, not always correct
-		if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
-			name = *n.constant
-			name = name[:len(name)-2] + "Id"
-		} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
-			name = *n.constant
-			name = name[:len(name)-1] + "Id"
-		} else {
-			name = *n.constant + "Id"
-		}
-
-		name = cleanStr(name, isAlNumRune)
-		if !isAlphaRune(rune(name[0])) {
-			name = "_" + name
-		}
-	}
-
-	newParam := createSimpleParam(name, "path", "string")
-	x := n.countParentParams()
-	if x > 0 {
-		newParam.Name = newParam.Name + strconv.Itoa(x)
-	}
-
-	return newParam
-}
-
-func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunkIsGibberish bool) *Node {
-	// look among params
-	if paramObj != nil || chunkIsGibberish {
-		for _, subnode := range n.children {
-			if subnode.constant != nil {
-				continue
-			}
-
-			// TODO: check the regex pattern of param? for exceptions etc
-
-			if paramObj != nil {
-				// TODO: mergeParam(subnode.pathParam, paramObj)
-				return subnode
-			} else {
-				return subnode
-			}
-		}
-	}
-	return nil
-}
-
-func (n *Node) searchInConstants(pathChunk string) *Node {
-	// look among constants
-	for _, subnode := range n.children {
-		if subnode.constant == nil {
-			continue
-		}
-
-		if *subnode.constant == pathChunk {
-			return subnode
-		}
-	}
-	return nil
-}
-
-func (n *Node) compact() {
-	// TODO
-}
-
-func (n *Node) listPaths() *openapi.Paths {
-	paths := &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
-
-	var strChunk string
-	if n.constant != nil {
-		strChunk = *n.constant
-	} else if n.pathParam != nil {
-		strChunk = "{" + n.pathParam.Name + "}"
-	} // else -> this is the root node
-
-	// add self
-	if n.pathObj != nil {
-		fillPathParams(n, n.pathObj)
-		paths.Items[openapi.PathValue(strChunk)] = n.pathObj
-	}
-
-	// recurse into children
-	for _, child := range n.children {
-		subPaths := child.listPaths()
-		for path, pathObj := range subPaths.Items {
-			var concat string
-			if n.parent == nil {
-				concat = string(path)
-			} else {
-				concat = strChunk + "/" + string(path)
-			}
-			paths.Items[openapi.PathValue(concat)] = pathObj
-		}
-	}
-
-	return paths
-}
-
-func fillPathParams(n *Node, pathObj *openapi.PathObj) {
-	// collect all path parameters from parent hierarchy
-	node := n
-	for {
-		if node.pathParam != nil {
-			initParams(&pathObj.Parameters)
-
-			idx, paramObj := findParamByName(pathObj.Parameters, openapi.InPath, node.pathParam.Name)
-			if paramObj == nil {
-				appended := append(*pathObj.Parameters, node.pathParam)
-				pathObj.Parameters = &appended
-			} else {
-				(*pathObj.Parameters)[idx] = paramObj
-			}
-		}
-
-		node = node.parent
-		if node == nil {
-			break
-		}
-	}
-}
-
-type PathAndOp struct {
-	path string
-	op   *openapi.Operation
-}
-
-func (n *Node) listOps() []PathAndOp {
-	res := make([]PathAndOp, 0)
-	for path, pathObj := range n.listPaths().Items {
-		for _, op := range getOps(pathObj) {
-			res = append(res, PathAndOp{path: string(path), op: op})
-		}
-	}
-	return res
-}
-
-func (n *Node) countParentParams() int {
-	res := 0
-	node := n
-	for {
-		if node.pathParam != nil {
-			res++
-		}
-
-		if node.parent == nil {
-			break
-		}
-		node = node.parent
-	}
-	return res
-}

agent/pkg/oas/tree_test.go

@@ -1,37 +0,0 @@
-package oas
-
-import (
-	"github.com/chanced/openapi"
-	"strings"
-	"testing"
-)
-
-func TestTree(t *testing.T) {
-	testCases := []struct {
-		inp       string
-		numParams int
-		label     string
-	}{
-		{"/", 0, ""},
-		{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation", 1, "vendor,init,conversation"},
-		{"/v1.0.0/config/launcher/sp_nKNHCzsN/{f34efcae-6583-11eb-908a-00b0fcb9d4f6}/vendor,init,conversation", 0, "vendor,init,conversation"},
-		{"/getSvgs/size/small/brand/SFLY/layoutId/170943/layoutVersion/1/sizeId/742/surface/0/isLandscape/true/childSkus/%7B%7D", 1, "{}"},
-	}
-
-	tree := new(Node)
-	for _, tc := range testCases {
-		split := strings.Split(tc.inp, "/")
-		pathObj := new(openapi.PathObj)
-		node := tree.getOrSet(split, pathObj)
-
-		fillPathParams(node, pathObj)
-
-		if node.constant != nil && *node.constant != tc.label {
-			t.Errorf("Constant does not match: %s != %s", *node.constant, tc.label)
-		}
-
-		if tc.numParams > 0 && (pathObj.Parameters == nil || len(*pathObj.Parameters) < tc.numParams) {
-			t.Errorf("Wrong num of params, expected: %d", tc.numParams)
-		}
-	}
-}

agent/pkg/oas/utils.go

@@ -1,429 +0,0 @@
-package oas
-
-import (
-	"encoding/json"
-	"errors"
-	"strconv"
-	"strings"
-
-	"github.com/up9inc/mizu/agent/pkg/har"
-
-	"github.com/chanced/openapi"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-func exampleResolver(ref string) (*openapi.ExampleObj, error) {
-	return nil, errors.New("JSON references are not supported at the moment: " + ref)
-}
-
-func responseResolver(ref string) (*openapi.ResponseObj, error) {
-	return nil, errors.New("JSON references are not supported at the moment: " + ref)
-}
-
-func reqBodyResolver(ref string) (*openapi.RequestBodyObj, error) {
-	return nil, errors.New("JSON references are not supported at the moment: " + ref)
-}
-
-func paramResolver(ref string) (*openapi.ParameterObj, error) {
-	return nil, errors.New("JSON references are not supported at the moment: " + ref)
-}
-
-func headerResolver(ref string) (*openapi.HeaderObj, error) {
-	return nil, errors.New("JSON references are not supported at the moment: " + ref)
-}
-
-func initParams(obj **openapi.ParameterList) {
-	if *obj == nil {
-		var params openapi.ParameterList = make([]openapi.Parameter, 0)
-		*obj = &params
-	}
-}
-
-func initHeaders(respObj *openapi.ResponseObj) {
-	if respObj.Headers == nil {
-		var created openapi.Headers = map[string]openapi.Header{}
-		respObj.Headers = created
-	}
-}
-
-func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *openapi.ParameterObj {
-	if name == "" {
-		panic("Cannot create parameter with empty name")
-	}
-	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
-	schema := new(openapi.SchemaObj)
-	schema.Type = openapi.Types{ptype}
-
-	style := openapi.StyleSimple
-	if in == openapi.InQuery {
-		style = openapi.StyleForm
-	}
-
-	newParam := openapi.ParameterObj{
-		Name:     name,
-		In:       in,
-		Style:    string(style),
-		Examples: map[string]openapi.Example{},
-		Schema:   schema,
-		Required: &required,
-	}
-	return &newParam
-}
-
-func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (idx int, pathParam *openapi.ParameterObj) {
-	caseInsensitive := in == openapi.InHeader
-	for i, param := range *params {
-		idx = i
-		paramObj, err := param.ResolveParameter(paramResolver)
-		if err != nil {
-			logger.Log.Warningf("Failed to resolve reference: %s", err)
-			continue
-		}
-
-		if paramObj.In != in {
-			continue
-		}
-
-		if paramObj.Name == name || (caseInsensitive && strings.EqualFold(paramObj.Name, name)) {
-			pathParam = paramObj
-			break
-		}
-	}
-
-	return idx, pathParam
-}
-
-func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj {
-	for hname, param := range *headers {
-		hdrObj, err := param.ResolveHeader(headerResolver)
-		if err != nil {
-			logger.Log.Warningf("Failed to resolve reference: %s", err)
-			continue
-		}
-
-		if strings.EqualFold(hname, name) {
-			return hdrObj
-		}
-	}
-	return nil
-}
-
-type nvParams struct {
-	In             openapi.In
-	Pairs          []har.NVP
-	IsIgnored      func(name string) bool
-	GeneralizeName func(name string) string
-}
-
-func handleNameVals(gw nvParams, params **openapi.ParameterList, checkIgnore bool) {
-	visited := map[string]*openapi.ParameterObj{}
-	for _, pair := range gw.Pairs {
-		if (checkIgnore && gw.IsIgnored(pair.Name)) || pair.Name == "" {
-			continue
-		}
-
-		nameGeneral := gw.GeneralizeName(pair.Name)
-
-		initParams(params)
-		_, param := findParamByName(*params, gw.In, pair.Name)
-		if param == nil {
-			param = createSimpleParam(nameGeneral, gw.In, openapi.TypeString)
-			appended := append(**params, param)
-			*params = &appended
-		}
-		exmp := &param.Examples
-		err := fillParamExample(&exmp, pair.Value)
-		if err != nil {
-			logger.Log.Warningf("Failed to add example to a parameter: %s", err)
-		}
-		visited[nameGeneral] = param
-	}
-
-	// maintain "required" flag
-	if *params != nil {
-		for _, param := range **params {
-			paramObj, err := param.ResolveParameter(paramResolver)
-			if err != nil {
-				logger.Log.Warningf("Failed to resolve param: %s", err)
-				continue
-			}
-			if paramObj.In != gw.In {
-				continue
-			}
-
-			_, ok := visited[strings.ToLower(paramObj.Name)]
-			if !ok {
-				flag := false
-				paramObj.Required = &flag
-			}
-		}
-	}
-}
-
-func createHeader(ptype openapi.SchemaType) *openapi.HeaderObj {
-	required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
-	schema := new(openapi.SchemaObj)
-	schema.Type = make(openapi.Types, 0)
-	schema.Type = append(schema.Type, ptype)
-
-	style := openapi.StyleSimple
-	newParam := openapi.HeaderObj{
-		Style:    string(style),
-		Examples: map[string]openapi.Example{},
-		Schema:   schema,
-		Required: &required,
-	}
-	return &newParam
-}
-
-func fillParamExample(param **openapi.Examples, exampleValue string) error {
-	if **param == nil {
-		**param = map[string]openapi.Example{}
-	}
-
-	cnt := 0
-	for _, example := range **param {
-		cnt++
-		exampleObj, err := example.ResolveExample(exampleResolver)
-		if err != nil {
-			continue
-		}
-
-		var value string
-		err = json.Unmarshal(exampleObj.Value, &value)
-		if err != nil {
-			logger.Log.Warningf("Failed decoding parameter example into string: %s", err)
-			continue
-		}
-
-		if value == exampleValue || cnt >= 5 { // 5 examples is enough
-			return nil
-		}
-	}
-
-	valMsg, err := json.Marshal(exampleValue)
-	if err != nil {
-		return err
-	}
-
-	themap := **param
-	themap["example #"+strconv.Itoa(cnt)] = &openapi.ExampleObj{Value: valMsg}
-
-	return nil
-}
-
-// TODO: somehow generalize the two example setting functions, plus add body example handling
-
-func addSchemaExample(existing *openapi.SchemaObj, bodyStr string) {
-	if len(existing.Examples) < 5 {
-		found := false
-		for _, eVal := range existing.Examples {
-			existingExample := ""
-			err := json.Unmarshal(eVal, &existingExample)
-			if err != nil {
-				logger.Log.Debugf("Failed to unmarshal example: %v", eVal)
-				continue
-			}
-
-			if existingExample == bodyStr {
-				found = true
-				break
-			}
-		}
-
-		if !found {
-			example, err := json.Marshal(bodyStr)
-			if err != nil {
-				logger.Log.Debugf("Failed to marshal example: %v", bodyStr)
-				return
-			}
-			existing.Examples = append(existing.Examples, example)
-		}
-	}
-}
-
-func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.com/jpillora/longestcommon
-	empty := make([]string, 0)
-	//short-circuit empty list
-	if len(strs) == 0 {
-		return empty
-	}
-	xfix := strs[0]
-	//short-circuit single-element list
-	if len(strs) == 1 {
-		return xfix
-	}
-	//compare first to rest
-	for _, str := range strs[1:] {
-		xfixl := len(xfix)
-		strl := len(str)
-		//short-circuit empty strings
-		if xfixl == 0 || strl == 0 {
-			return empty
-		}
-		//maximum possible length
-		maxl := xfixl
-		if strl < maxl {
-			maxl = strl
-		}
-		//compare letters
-		if pre {
-			//prefix, iterate left to right
-			for i := 0; i < maxl; i++ {
-				if xfix[i] != str[i] {
-					xfix = xfix[:i]
-					break
-				}
-			}
-		} else {
-			//suffix, iternate right to left
-			for i := 0; i < maxl; i++ {
-				xi := xfixl - i - 1
-				si := strl - i - 1
-				if xfix[xi] != str[si] {
-					xfix = xfix[xi+1:]
-					break
-				}
-			}
-		}
-	}
-	return xfix
-}
-
-func getSimilarPrefix(strs []string) string {
-	chunked := make([][]string, 0)
-	for _, item := range strs {
-		chunked = append(chunked, strings.Split(item, "/"))
-	}
-
-	cmn := longestCommonXfix(chunked, true)
-	res := make([]string, 0)
-	for _, chunk := range cmn {
-		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
-			res = append(res, chunk)
-		}
-	}
-	return strings.Join(res[1:], ".")
-}
-
-// returns all non-nil ops in PathObj
-func getOps(pathObj *openapi.PathObj) []*openapi.Operation {
-	ops := []**openapi.Operation{&pathObj.Get, &pathObj.Patch, &pathObj.Put, &pathObj.Options, &pathObj.Post, &pathObj.Trace, &pathObj.Head, &pathObj.Delete}
-	res := make([]*openapi.Operation, 0)
-	for _, opp := range ops {
-		if *opp == nil {
-			continue
-		}
-		res = append(res, *opp)
-	}
-	return res
-}
-
-// parses JSON into any possible value
-func anyJSON(text string) (anyVal interface{}, isJSON bool) {
-	isJSON = true
-	asMap := map[string]interface{}{}
-	err := json.Unmarshal([]byte(text), &asMap)
-	if err == nil && asMap != nil {
-		return asMap, isJSON
-	}
-
-	asArray := make([]interface{}, 0)
-	err = json.Unmarshal([]byte(text), &asArray)
-	if err == nil && asArray != nil {
-		return asArray, isJSON
-	}
-
-	asString := ""
-	sPtr := &asString
-	err = json.Unmarshal([]byte(text), &sPtr)
-	if err == nil && sPtr != nil {
-		return asString, isJSON
-	}
-
-	asInt := 0
-	intPtr := &asInt
-	err = json.Unmarshal([]byte(text), &intPtr)
-	if err == nil && intPtr != nil {
-		return asInt, isJSON
-	}
-
-	asFloat := 0.0
-	floatPtr := &asFloat
-	err = json.Unmarshal([]byte(text), &floatPtr)
-	if err == nil && floatPtr != nil {
-		return asFloat, isJSON
-	}
-
-	asBool := false
-	boolPtr := &asBool
-	err = json.Unmarshal([]byte(text), &boolPtr)
-	if err == nil && boolPtr != nil {
-		return asBool, isJSON
-	}
-
-	if text == "null" {
-		return nil, isJSON
-	}
-
-	return nil, false
-}
-
-func cleanStr(str string, criterion func(r rune) bool) string {
-	s := []byte(str)
-	j := 0
-	for _, b := range s {
-		if criterion(rune(b)) {
-			s[j] = b
-			j++
-		}
-	}
-	return string(s[:j])
-}
-
-/*
-func isAlpha(s string) bool {
-	for _, r := range s {
-		if isAlphaRune(r) {
-			return false
-		}
-	}
-	return true
-}
-*/
-
-func isAlphaRune(r rune) bool {
-	return !((r < 'a' || r > 'z') && (r < 'A' || r > 'Z'))
-}
-
-func isAlNumRune(b rune) bool {
-	return isAlphaRune(b) || ('0' <= b && b <= '9')
-}
-
-func deleteFromSlice(s []string, val string) []string {
-	temp := s[:0]
-	for _, x := range s {
-		if x != val {
-			temp = append(temp, x)
-		}
-	}
-	return temp
-}
-
-func sliceContains(s []string, e string) bool {
-	for _, a := range s {
-		if a == e {
-			return true
-		}
-	}
-	return false
-}
-
-func intersectSliceWithMap(required []string, names map[string]struct{}) []string {
-	for name := range names {
-		if !sliceContains(required, name) {
-			required = deleteFromSlice(required, name)
-		}
-	}
-	return required
-}

agent/pkg/oas/utils_test.go

@@ -1,59 +0,0 @@
-package oas
-
-import (
-	"testing"
-)
-
-func TestAnyJSON(t *testing.T) {
-	testCases := []struct {
-		inp    string
-		isJSON bool
-		out    interface{}
-	}{
-		{`{"key": 1, "keyNull": null}`, true, nil},
-		{`[{"key": "val"}, ["subarray"], "string", 1, 2.2, true, null]`, true, nil},
-		{`"somestring"`, true, "somestring"},
-		{"0", true, 0},
-		{"0.5", true, 0.5},
-		{"true", true, true},
-		{"null", true, nil},
-		{"sabbra cadabra", false, nil},
-		{"0.1.2.3", false, nil},
-	}
-	for _, tc := range testCases {
-		any, isJSON := anyJSON(tc.inp)
-		if isJSON != tc.isJSON {
-			t.Errorf("Parse flag mismatch: %t != %t", tc.isJSON, isJSON)
-		} else if isJSON && tc.out != nil && tc.out != any {
-			t.Errorf("%s != %s", any, tc.out)
-		} else if tc.inp == "null" && any != nil {
-			t.Errorf("null has to parse as nil (but got %s)", any)
-		} else {
-			t.Logf("%s => %s", tc.inp, any)
-		}
-	}
-}
-
-func TestStrRunes(t *testing.T) {
-	if isAlphaRune('5') {
-		t.Logf("Failed")
-	}
-	if !isAlphaRune('a') {
-		t.Logf("Failed")
-	}
-
-	if !isAlNumRune('5') {
-		t.Logf("Failed")
-	}
-	if isAlNumRune(' ') {
-		t.Logf("Failed")
-	}
-
-	if cleanStr("-abc_567", isAlphaRune) != "abc" {
-		t.Logf("Failed")
-	}
-
-	if cleanStr("-abc_567", isAlNumRune) != "abc567" {
-		t.Logf("Failed")
-	}
-}

agent/pkg/providers/install_provider.go

@@ -1,52 +0,0 @@
-package providers
-
-import (
-	"context"
-	"errors"
-
-	"github.com/up9inc/mizu/agent/pkg/config"
-
-	"github.com/up9inc/mizu/shared/logger"
-
-	ory "github.com/ory/kratos-client-go"
-)
-
-const AdminUsername = "admin"
-
-func IsInstallNeeded() (bool, error) {
-	if !config.Config.StandaloneMode { // install not needed in ephermeral mizu
-		return false, nil
-	}
-
-	if anyUserExists, err := AnyUserExists(context.Background()); err != nil {
-		return false, err
-	} else {
-		return !anyUserExists, nil
-	}
-}
-
-func CreateAdminUser(password string, ctx context.Context) (token *string, err error, formErrorMessages map[string][]ory.UiText) {
-	if isInstallNeeded, err := IsInstallNeeded(); err != nil {
-		return nil, err, nil
-	} else if !isInstallNeeded {
-		return nil, errors.New("The admin user has already been created"), nil
-	}
-
-	token, identityId, err, formErrors := RegisterUser(AdminUsername, password, ctx)
-	if err != nil {
-		return nil, err, formErrors
-	}
-
-	err = SetUserSystemRole(AdminUsername, AdminRole)
-
-	if err != nil {
-		//Delete the user to prevent a half-setup situation where admin user is created without admin privileges
-		if err := DeleteUser(identityId, ctx); err != nil {
-			logger.Log.Error(err)
-		}
-
-		return nil, err, nil
-	}
-
-	return token, nil, nil
-}

agent/pkg/providers/kubernetes_provider.go

@@ -1,27 +0,0 @@
-package providers
-
-import (
-	"github.com/up9inc/mizu/shared/kubernetes"
-	"sync"
-)
-
-var lock = &sync.Mutex{}
-
-var kubernetesProvider *kubernetes.Provider
-
-func GetKubernetesProvider() (*kubernetes.Provider, error) {
-	if kubernetesProvider == nil {
-		lock.Lock()
-		defer lock.Unlock()
-
-		if kubernetesProvider == nil {
-			var err error
-			kubernetesProvider, err = kubernetes.NewProviderInCluster()
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-
-	return kubernetesProvider, nil
-}

agent/pkg/providers/stats_provider.go

@@ -7,7 +7,6 @@ import (
 
 type GeneralStats struct {
 	EntriesCount        int
-	EntriesVolumeInGB   float64
 	FirstEntryTimestamp int
 	LastEntryTimestamp  int
 }
@@ -22,9 +21,8 @@ func GetGeneralStats() GeneralStats {
 	return generalStats
 }
 
-func EntryAdded(size int) {
+func EntryAdded() {
 	generalStats.EntriesCount++
-	generalStats.EntriesVolumeInGB += float64(size) / (1 << 30)
 
 	currentTimestamp := int(time.Now().Unix())
 
@@ -34,3 +32,5 @@ func EntryAdded(size int) {
 
 	generalStats.LastEntryTimestamp = currentTimestamp
 }
+
+

agent/pkg/providers/stats_provider_test.go

@@ -2,21 +2,22 @@ package providers_test
 
 import (
 	"fmt"
+	"mizuserver/pkg/config"
+	"mizuserver/pkg/database"
+	"mizuserver/pkg/providers"
 	"testing"
-
-	"github.com/up9inc/mizu/agent/pkg/providers"
 )
 
+func init() {
+	database.InitDataBase(config.DefaultDatabasePath)
+}
+
 func TestNoEntryAddedCount(t *testing.T) {
 	entriesStats := providers.GetGeneralStats()
 
 	if entriesStats.EntriesCount != 0 {
 		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesCount)
 	}
-
-	if entriesStats.EntriesVolumeInGB != 0 {
-		t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesVolumeInGB)
-	}
 }
 
 func TestEntryAddedCount(t *testing.T) {
@@ -25,7 +26,7 @@ func TestEntryAddedCount(t *testing.T) {
 	for _, entriesCount := range tests {
 		t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
 			for i := 0; i < entriesCount; i++ {
-				providers.EntryAdded(0)
+				providers.EntryAdded()
 			}
 
 			entriesStats := providers.GetGeneralStats()
@@ -34,38 +35,7 @@ func TestEntryAddedCount(t *testing.T) {
 				t.Errorf("unexpected result - expected: %v, actual: %v", entriesCount, entriesStats.EntriesCount)
 			}
 
-			if entriesStats.EntriesVolumeInGB != 0 {
-				t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesVolumeInGB)
-			}
-
 			t.Cleanup(providers.ResetGeneralStats)
 		})
 	}
 }
-
-func TestEntryAddedVolume(t *testing.T) {
-	// 6 bytes + 4 bytes
-	tests := [][]byte{[]byte("volume"), []byte("test")}
-	var expectedEntriesCount int
-	var expectedVolumeInGB float64
-
-	for _, data := range tests {
-		t.Run(fmt.Sprintf("%d", len(data)), func(t *testing.T) {
-			expectedEntriesCount++
-			expectedVolumeInGB += float64(len(data)) / (1 << 30)
-
-			providers.EntryAdded(len(data))
-
-			entriesStats := providers.GetGeneralStats()
-
-			if entriesStats.EntriesCount != expectedEntriesCount {
-				t.Errorf("unexpected result - expected: %v, actual: %v", expectedEntriesCount, entriesStats.EntriesCount)
-			}
-
-			if entriesStats.EntriesVolumeInGB != expectedVolumeInGB {
-				t.Errorf("unexpected result - expected: %v, actual: %v", expectedVolumeInGB, entriesStats.EntriesVolumeInGB)
-			}
-		})
-	}
-
-}

agent/pkg/providers/status_provider.go

@@ -3,20 +3,24 @@ package providers
 import (
 	"encoding/json"
 	"fmt"
-	"os"
-	"time"
-
 	"github.com/patrickmn/go-cache"
-	"github.com/up9inc/mizu/agent/pkg/models"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/tap"
+	"mizuserver/pkg/models"
+	"os"
+	"sync"
+	"time"
 )
 
 const tlsLinkRetainmentTime = time.Minute * 15
 
 var (
+	TappersCount   int
+	TapStatus      shared.TapStatus
 	authStatus     *models.AuthStatus
 	RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
+
+	tappersCountLock = sync.Mutex{}
 )
 
 func GetAuthStatus() (*models.AuthStatus, error) {
@@ -64,3 +68,15 @@ func GetAllRecentTLSAddresses() []string {
 
 	return recentTLSLinks
 }
+
+func TapperAdded() {
+	tappersCountLock.Lock()
+	TappersCount++
+	tappersCountLock.Unlock()
+}
+
+func TapperRemoved() {
+	tappersCountLock.Lock()
+	TappersCount--
+	tappersCountLock.Unlock()
+}

agent/pkg/providers/tapConfig/tap_config_provider.go

@@ -1,43 +0,0 @@
-package tapConfig
-
-import (
-	"os"
-	"sync"
-
-	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/utils"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-const FilePath = shared.DataDirPath + "tap-config.json"
-
-var (
-	lock     = &sync.Mutex{}
-	syncOnce sync.Once
-	config   *models.TapConfig
-)
-
-func Get() *models.TapConfig {
-	syncOnce.Do(func() {
-		if err := utils.ReadJsonFile(FilePath, &config); err != nil {
-			config = &models.TapConfig{TappedNamespaces: make(map[string]bool)}
-
-			if !os.IsNotExist(err) {
-				logger.Log.Errorf("Error reading tap config from file, err: %v", err)
-			}
-		}
-	})
-
-	return config
-}
-
-func Save(tapConfigToSave *models.TapConfig) {
-	lock.Lock()
-	defer lock.Unlock()
-
-	config = tapConfigToSave
-	if err := utils.SaveJsonFile(FilePath, config); err != nil {
-		logger.Log.Errorf("Error saving tap config, err: %v", err)
-	}
-}

agent/pkg/providers/tappedPods/tapped_pods_provider.go

@@ -1,57 +0,0 @@
-package tappedPods
-
-import (
-	"os"
-	"strings"
-	"sync"
-
-	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
-	"github.com/up9inc/mizu/agent/pkg/utils"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-const FilePath = shared.DataDirPath + "tapped-pods.json"
-
-var (
-	lock       = &sync.Mutex{}
-	syncOnce   sync.Once
-	tappedPods []*shared.PodInfo
-)
-
-func Get() []*shared.PodInfo {
-	syncOnce.Do(func() {
-		if err := utils.ReadJsonFile(FilePath, &tappedPods); err != nil {
-			if !os.IsNotExist(err) {
-				logger.Log.Errorf("Error reading tapped pods from file, err: %v", err)
-			}
-		}
-	})
-
-	return tappedPods
-}
-
-func Set(tappedPodsToSet []*shared.PodInfo) {
-	lock.Lock()
-	defer lock.Unlock()
-
-	tappedPods = tappedPodsToSet
-	if err := utils.SaveJsonFile(FilePath, tappedPods); err != nil {
-		logger.Log.Errorf("Error saving tapped pods, err: %v", err)
-	}
-}
-
-func GetTappedPodsStatus() []shared.TappedPodStatus {
-	tappedPodsStatus := make([]shared.TappedPodStatus, 0)
-	for _, pod := range Get() {
-		var status string
-		if tapperStatus, ok := tappers.GetStatus()[pod.NodeName]; ok {
-			status = strings.ToLower(tapperStatus.Status)
-		}
-
-		isTapped := status == "running"
-		tappedPodsStatus = append(tappedPodsStatus, shared.TappedPodStatus{Name: pod.Name, Namespace: pod.Namespace, IsTapped: isTapped})
-	}
-
-	return tappedPodsStatus
-}

agent/pkg/providers/tappers/tappers_provider.go

@@ -1,83 +0,0 @@
-package tappers
-
-import (
-	"os"
-	"sync"
-
-	"github.com/up9inc/mizu/agent/pkg/utils"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-const FilePath = shared.DataDirPath + "tappers-status.json"
-
-var (
-	lockStatus = &sync.Mutex{}
-	syncOnce   sync.Once
-	status     map[string]*shared.TapperStatus
-
-	lockConnectedCount = &sync.Mutex{}
-	connectedCount     int
-)
-
-func GetStatus() map[string]*shared.TapperStatus {
-	initStatus()
-
-	return status
-}
-
-func SetStatus(tapperStatus *shared.TapperStatus) {
-	initStatus()
-
-	lockStatus.Lock()
-	defer lockStatus.Unlock()
-
-	status[tapperStatus.NodeName] = tapperStatus
-
-	saveStatus()
-}
-
-func ResetStatus() {
-	lockStatus.Lock()
-	defer lockStatus.Unlock()
-
-	status = make(map[string]*shared.TapperStatus)
-
-	saveStatus()
-}
-
-func GetConnectedCount() int {
-	return connectedCount
-}
-
-func Connected() {
-	lockConnectedCount.Lock()
-	defer lockConnectedCount.Unlock()
-
-	connectedCount++
-}
-
-func Disconnected() {
-	lockConnectedCount.Lock()
-	defer lockConnectedCount.Unlock()
-
-	connectedCount--
-}
-
-func initStatus() {
-	syncOnce.Do(func() {
-		if err := utils.ReadJsonFile(FilePath, &status); err != nil {
-			status = make(map[string]*shared.TapperStatus)
-
-			if !os.IsNotExist(err) {
-				logger.Log.Errorf("Error reading tappers status from file, err: %v", err)
-			}
-		}
-	})
-}
-
-func saveStatus() {
-	if err := utils.SaveJsonFile(FilePath, status); err != nil {
-		logger.Log.Errorf("Error saving tappers status, err: %v", err)
-	}
-}

agent/pkg/providers/user_provider.go

@@ -1,162 +0,0 @@
-package providers
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net/http"
-	"net/http/cookiejar"
-
-	ory "github.com/ory/kratos-client-go"
-	"github.com/up9inc/mizu/shared/logger"
-)
-
-var client = getKratosClient("http://127.0.0.1:4433", "http://127.0.0.1:4434")
-
-// returns session token if successful
-func RegisterUser(username string, password string, ctx context.Context) (token *string, identityId string, err error, formErrorMessages map[string][]ory.UiText) {
-	flow, _, err := client.V0alpha2Api.InitializeSelfServiceRegistrationFlowWithoutBrowser(ctx).Execute()
-	if err != nil {
-		return nil, "", err, nil
-	}
-
-	result, _, err := client.V0alpha2Api.SubmitSelfServiceRegistrationFlow(ctx).Flow(flow.Id).SubmitSelfServiceRegistrationFlowBody(
-		ory.SubmitSelfServiceRegistrationFlowWithPasswordMethodBodyAsSubmitSelfServiceRegistrationFlowBody(&ory.SubmitSelfServiceRegistrationFlowWithPasswordMethodBody{
-			Method:   "password",
-			Password: password,
-			Traits:   map[string]interface{}{"username": username},
-		}),
-	).Execute()
-
-	if err != nil {
-		parsedKratosError, parsingErr := parseKratosRegistrationFormError(err)
-		if parsingErr != nil {
-			logger.Log.Debugf("error parsing kratos error: %v", parsingErr)
-			return nil, "", err, nil
-		} else {
-			return nil, "", err, parsedKratosError
-		}
-	}
-
-	return result.SessionToken, result.Identity.Id, nil, nil
-}
-
-func PerformLogin(username string, password string, ctx context.Context) (*string, error) {
-	flow, _, err := client.V0alpha2Api.InitializeSelfServiceLoginFlowWithoutBrowser(ctx).Execute()
-	if err != nil {
-		return nil, err
-	}
-
-	result, _, err := client.V0alpha2Api.SubmitSelfServiceLoginFlow(ctx).Flow(flow.Id).SubmitSelfServiceLoginFlowBody(
-		ory.SubmitSelfServiceLoginFlowWithPasswordMethodBodyAsSubmitSelfServiceLoginFlowBody(&ory.SubmitSelfServiceLoginFlowWithPasswordMethodBody{
-			Method:             "password",
-			Password:           password,
-			PasswordIdentifier: username,
-		}),
-	).Execute()
-
-	if err != nil {
-		return nil, err
-	}
-	if result == nil {
-		return nil, errors.New("unknown error occured during login")
-	}
-
-	return result.SessionToken, nil
-}
-
-func VerifyToken(token string, ctx context.Context) (*ory.Session, error) {
-	flow, _, err := client.V0alpha2Api.ToSession(ctx).XSessionToken(token).Execute()
-	if err != nil {
-		return nil, err
-	}
-
-	if flow == nil {
-		return nil, nil
-	}
-
-	return flow, nil
-}
-
-func DeleteUser(identityId string, ctx context.Context) error {
-	result, err := client.V0alpha2Api.AdminDeleteIdentity(ctx, identityId).Execute()
-	if err != nil {
-		return err
-	}
-	if result == nil {
-		return fmt.Errorf("unknown error occured during user deletion %v", identityId)
-	}
-
-	if result.StatusCode < 200 || result.StatusCode > 299 {
-		return fmt.Errorf("user deletion %v returned bad status %d", identityId, result.StatusCode)
-	} else {
-		return nil
-	}
-}
-
-func AnyUserExists(ctx context.Context) (bool, error) {
-	request := client.V0alpha2Api.AdminListIdentities(ctx)
-	request.PerPage(1)
-
-	if result, _, err := request.Execute(); err != nil {
-		return false, err
-	} else {
-		return len(result) > 0, nil
-	}
-}
-
-func Logout(token string, ctx context.Context) error {
-	logoutRequest := client.V0alpha2Api.SubmitSelfServiceLogoutFlowWithoutBrowser(ctx)
-	logoutRequest = logoutRequest.SubmitSelfServiceLogoutFlowWithoutBrowserBody(ory.SubmitSelfServiceLogoutFlowWithoutBrowserBody{
-		SessionToken: token,
-	})
-	if response, err := logoutRequest.Execute(); err != nil {
-		return err
-	} else if response == nil || response.StatusCode < 200 || response.StatusCode > 299 {
-		return errors.New("unknown error occured during logout")
-	}
-
-	return nil
-}
-
-func getKratosClient(url string, adminUrl string) *ory.APIClient {
-	conf := ory.NewConfiguration()
-	conf.Servers = ory.ServerConfigurations{{URL: url}}
-
-	// this ensures kratos client uses the admin url for admin actions (any new admin action we use will have to be added here)
-	conf.OperationServers = map[string]ory.ServerConfigurations{
-		"V0alpha2ApiService.AdminDeleteIdentity": {{URL: adminUrl}},
-		"V0alpha2ApiService.AdminListIdentities": {{URL: adminUrl}},
-	}
-
-	cj, _ := cookiejar.New(nil)
-	conf.HTTPClient = &http.Client{Jar: cj}
-	return ory.NewAPIClient(conf)
-}
-
-// returns map of form value key to error message
-func parseKratosRegistrationFormError(err error) (map[string][]ory.UiText, error) {
-	var openApiError *ory.GenericOpenAPIError
-	if errors.As(err, &openApiError) {
-		var registrationFlowModel *ory.SelfServiceRegistrationFlow
-		if jsonErr := json.Unmarshal(openApiError.Body(), &registrationFlowModel); jsonErr != nil {
-			return nil, jsonErr
-		} else {
-			formMessages := registrationFlowModel.Ui.Nodes
-			parsedMessages := make(map[string][]ory.UiText)
-
-			for _, message := range formMessages {
-				if len(message.Messages) > 0 {
-					if _, ok := parsedMessages[message.Group]; !ok {
-						parsedMessages[message.Group] = make([]ory.UiText, 0)
-					}
-					parsedMessages[message.Group] = append(parsedMessages[message.Group], message.Messages...)
-				}
-			}
-			return parsedMessages, nil
-		}
-	} else {
-		return nil, errors.New("error is not a generic openapi error")
-	}
-}